From c03617daf9f29fdbb2172fb248d6609264fb33c6 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Tue, 30 Jun 2020 16:25:44 +0000 Subject: [PATCH 001/209] Setup vs code and theia/gitpod for debugging rust" --- .gitpod.Dockerfile | 7 +++++++ .gitpod.yml | 4 ++++ .theia/launch.json | 16 ++++++++++++++++ .theia/settings.json | 12 ++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 .theia/launch.json create mode 100644 .theia/settings.json diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile index 546e02fd1..0a54e9d39 100644 --- a/.gitpod.Dockerfile +++ b/.gitpod.Dockerfile @@ -11,4 +11,11 @@ RUN rm -rf ~/.rustup && \ curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \ rustup target add wasm32-unknown-unknown +RUN sudo apt-get -q update \ + && sudo apt-get install -yq \ + libpython3.6 \ + rust-lldb \ + && sudo rm -rf /var/lib/apt/lists/* +ENV RUST_LLDB=/usr/bin/lldb-8 + USER root diff --git a/.gitpod.yml b/.gitpod.yml index 527c76d39..7f2eea913 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,2 +1,6 @@ image: file: .gitpod.Dockerfile + +vscode: + extensions: + - vadimcn.vscode-lldb@1.5.3:vTh/rWhvJ5nQpeAVsD20QA== \ No newline at end of file diff --git a/.theia/launch.json b/.theia/launch.json new file mode 100644 index 000000000..699c72ee9 --- /dev/null +++ b/.theia/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug Rust Code", + //"preLaunchTask": "cargo", + "program": "${workspaceFolder}/target/debug/rustpython", + "cwd": "${workspaceFolder}", + //"valuesFormatting": "parseText" + } + ] +} diff --git a/.theia/settings.json b/.theia/settings.json new file mode 100644 index 000000000..61f198875 --- /dev/null +++ b/.theia/settings.json @@ -0,0 +1,12 @@ +{ + "cpp.buildConfigurations": [ + { + "name": "", + "directory": "" + }, + { + "name": "", + "directory": "" + } + ] +} \ No newline at end of file From c4d83ab03cf3cd7b01f9fd0fee194c374b0b3b71 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Mon, 6 Jul 2020 18:25:10 +0000 Subject: [PATCH 002/209] cleaned up --- .theia/settings.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.theia/settings.json b/.theia/settings.json index 61f198875..83db8fc48 100644 --- a/.theia/settings.json +++ b/.theia/settings.json @@ -4,9 +4,5 @@ "name": "", "directory": "" }, - { - "name": "", - "directory": "" - } ] } \ No newline at end of file From 50ffa8ff10122d06a4d16a4ee78c644cdb6c9d07 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 14 Jul 2020 21:24:23 -0500 Subject: [PATCH 003/209] fp.read(0) == b'', not read_full --- vm/src/stdlib/io.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 09396fdbd..26daec8d0 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -85,7 +85,7 @@ impl BufferedIO { let mut buffer = Vec::new(); //for a defined number of bytes, i.e. bytes != -1 - if bytes > 0 { + if bytes >= 0 { let mut handle = self.cursor.clone().take(bytes as u64); //read handle into buffer From 3c241cd91e6ee68f028fef64fce6f3aa022a4146 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 14 Jul 2020 21:24:41 -0500 Subject: [PATCH 004/209] Allow struct.pack to take a bytes spec --- vm/src/stdlib/pystruct.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index b10767fb3..c4a386390 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -436,8 +436,18 @@ mod _struct { } #[pyfunction] - fn pack(fmt: PyStringRef, args: Args, vm: &VirtualMachine) -> PyResult> { - let format_spec = FormatSpec::parse(fmt.as_str()).map_err(|e| new_struct_error(vm, e))?; + fn pack( + fmt: Either, + args: Args, + vm: &VirtualMachine, + ) -> PyResult> { + // FIXME: the given fmt must be parsed as ascii string + // https://github.com/RustPython/RustPython/pull/1792#discussion_r387340905 + let parsed = match fmt { + Either::A(string) => FormatSpec::parse(string.as_str()), + Either::B(bytes) => FormatSpec::parse(std::str::from_utf8(&bytes).unwrap()), + }; + let format_spec = parsed.map_err(|e| new_struct_error(vm, e))?; format_spec.pack(args.as_ref(), vm) } From dd4abd082ebc34da9db393fd75176ddbdf5b9c87 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 14 Jul 2020 21:24:49 -0500 Subject: [PATCH 005/209] Add zlib.compressobj() --- vm/src/stdlib/zlib.rs | 139 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 5 deletions(-) diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index b14ca606d..908e17043 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -10,7 +10,9 @@ use crate::vm::VirtualMachine; use adler32::RollingAdler32 as Adler32; use crc32fast::Hasher as Crc32; use crossbeam_utils::atomic::AtomicCell; -use flate2::{write::ZlibEncoder, Compression, Decompress, FlushDecompress, Status}; +use flate2::{ + write::ZlibEncoder, Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status, +}; use libz_sys as libz; use parking_lot::Mutex; @@ -34,14 +36,16 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "adler32" => ctx.new_function(zlib_adler32), "compress" => ctx.new_function(zlib_compress), "decompress" => ctx.new_function(zlib_decompress), - // "compressobj" => ctx.new_function(zlib_compressobj), + "compressobj" => ctx.new_function(zlib_compressobj), "decompressobj" => ctx.new_function(zlib_decompressobj), + "Compress" => PyCompress::make_class(ctx), "Decompress" => PyDecompress::make_class(ctx), "error" => zlib_error, "Z_DEFAULT_COMPRESSION" => ctx.new_int(libz::Z_DEFAULT_COMPRESSION), "Z_NO_COMPRESSION" => ctx.new_int(libz::Z_NO_COMPRESSION), "Z_BEST_SPEED" => ctx.new_int(libz::Z_BEST_SPEED), "Z_BEST_COMPRESSION" => ctx.new_int(libz::Z_BEST_COMPRESSION), + "DEFLATED" => ctx.new_int(libz::Z_DEFLATED), "DEF_BUF_SIZE" => ctx.new_int(DEF_BUF_SIZE), "MAX_WBITS" => ctx.new_int(MAX_WBITS), }) @@ -118,7 +122,7 @@ fn zlib_decompress( let mut buf = Vec::new(); // TODO: maybe deduplicate this with the Decompress.{decompress,flush} - 'outer: for chunk in data.chunks(libc::c_uint::max_value() as usize) { + 'outer: for chunk in data.chunks(CHUNKSIZE) { // if this is the final chunk, finish it let flush = if d.total_in() == (data.len() - chunk.len()) as u64 { FlushDecompress::Finish @@ -227,7 +231,7 @@ impl PyDecompress { let mut buf = Vec::new(); let mut stream_end = false; - 'outer: for chunk in data.chunks(libc::c_uint::max_value() as usize) { + 'outer: for chunk in data.chunks(CHUNKSIZE) { // if this is the final chunk, finish it let flush = if d.total_in() - orig_in == (data.len() - chunk.len()) as u64 { FlushDecompress::Finish @@ -292,7 +296,7 @@ impl PyDecompress { let mut buf = Vec::new(); let mut stream_end = false; - 'outer: for chunk in data.chunks(libc::c_uint::max_value() as usize) { + 'outer: for chunk in data.chunks(CHUNKSIZE) { // if this is the final chunk, finish it let flush = if d.total_in() - orig_in == (data.len() - chunk.len()) as u64 { FlushDecompress::Finish @@ -338,6 +342,131 @@ struct DecompressArgs { max_length: usize, } +fn zlib_compressobj( + level: OptionalArg, + // only DEFLATED is valid right now, it's w/e + _method: OptionalArg, + wbits: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let (header, wbits) = header_from_wbits(wbits); + let level = level.unwrap_or(-1); + + let level = match level { + -1 => libz::Z_DEFAULT_COMPRESSION as u32, + n @ 0..=9 => n as u32, + _ => return Err(vm.new_value_error("invalid initialization option".to_owned())), + }; + let compress = Compress::new_with_window_bits(Compression::new(level), header, wbits); + Ok(PyCompress { + inner: Mutex::new(CompressInner { + compress, + unconsumed: Vec::new(), + }), + }) +} + +#[derive(Debug)] +struct CompressInner { + compress: Compress, + unconsumed: Vec, +} + +#[pyclass] +#[derive(Debug)] +struct PyCompress { + inner: Mutex, +} + +impl PyValue for PyCompress { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("zlib", "Compress") + } +} + +#[pyimpl] +impl PyCompress { + #[pymethod] + fn compress(&self, data: PyBytesLike, vm: &VirtualMachine) -> PyResult> { + let mut inner = self.inner.lock(); + data.with_ref(|b| inner.compress(b, vm)) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult> { + self.inner.lock().flush(vm) + } + + #[pymethod] + #[pymethod(magic)] + #[pymethod(name = "__deepcopy__")] + fn copy(&self) -> Self { + todo!("") + } +} + +const CHUNKSIZE: usize = libc::c_uint::MAX as usize; + +impl CompressInner { + fn save_unconsumed_input(&mut self, data: &[u8], orig_in: u64) { + let leftover = &data[(self.compress.total_in() - orig_in) as usize..]; + self.unconsumed.extend_from_slice(leftover); + } + + fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { + let orig_in = self.compress.total_in(); + let unconsumed = std::mem::take(&mut self.unconsumed); + let mut buf = Vec::new(); + + 'outer: for chunk in unconsumed.chunks(CHUNKSIZE).chain(data.chunks(CHUNKSIZE)) { + loop { + buf.reserve(DEF_BUF_SIZE); + match self + .compress + .compress_vec(chunk, &mut buf, FlushCompress::None) + { + Ok(_) if buf.len() == buf.capacity() => continue, + Ok(Status::StreamEnd) => break 'outer, + Ok(_) => break, + Err(_) => { + self.save_unconsumed_input(data, orig_in); + return Err(zlib_error("error while compressing", vm)); + } + } + } + } + self.save_unconsumed_input(data, orig_in); + + buf.shrink_to_fit(); + Ok(buf) + } + + // TODO: flush mode (FlushDecompress) parameter + fn flush(&mut self, vm: &VirtualMachine) -> PyResult> { + let data = std::mem::take(&mut self.unconsumed); + let mut data_it = data.chunks(CHUNKSIZE); + let mut buf = Vec::new(); + + loop { + let chunk = data_it.next().unwrap_or(&[]); + if buf.len() == buf.capacity() { + buf.reserve(DEF_BUF_SIZE); + } + match self + .compress + .compress_vec(chunk, &mut buf, FlushCompress::Finish) + { + Ok(Status::StreamEnd) => break, + Ok(_) => continue, + Err(_) => return Err(zlib_error("error while compressing", vm)), + } + } + + buf.shrink_to_fit(); + Ok(buf) + } +} + fn zlib_error(message: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_exception_msg(vm.class("zlib", "error"), message.to_owned()) } From e30e2cd278f540ab2ab7f8d0af2de04d7e6557f9 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 15 Jul 2020 12:53:57 -0500 Subject: [PATCH 006/209] Update some num-* crates --- Cargo.lock | 20 ++++++++++---------- bytecode/Cargo.toml | 2 +- parser/Cargo.toml | 2 +- vm/Cargo.toml | 4 ++-- vm/src/obj/objint.rs | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 156c88a12..e810a1fd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -964,9 +964,9 @@ checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] name = "num-bigint" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +checksum = "b7f3fc75e3697059fb1bc465e3d8cca6cf92f56854f201158b3f9c77d5a3cfa0" dependencies = [ "autocfg 1.0.0", "num-integer", @@ -987,9 +987,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ "autocfg 1.0.0", "num-traits", @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00" +checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f" dependencies = [ "autocfg 1.0.0", "num-integer", @@ -1008,9 +1008,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +checksum = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138" dependencies = [ "autocfg 1.0.0", "num-bigint", @@ -1020,9 +1020,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ "autocfg 1.0.0", ] diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml index 289f9fc6a..f46fd4e79 100644 --- a/bytecode/Cargo.toml +++ b/bytecode/Cargo.toml @@ -12,7 +12,7 @@ license = "MIT" bincode = "1.1" bitflags = "1.1" lz4-compress = "0.1.1" -num-bigint = { version = "0.2", features = ["serde"] } +num-bigint = { version = "0.3", features = ["serde"] } num-complex = { version = "0.2", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } itertools = "0.8" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index d4c0c7e7a..8d3aa4b09 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -14,7 +14,7 @@ lalrpop="0.17" [dependencies] lalrpop-util="0.17" log="0.4.1" -num-bigint = "0.2" +num-bigint = "0.3" num-traits = "0.2" unic-emoji-char = "0.9" unic-ucd-ident = "0.9" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index a65bc2408..50cbc9213 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -27,10 +27,10 @@ blake2 = "0.8" volatile = "0.2" num-complex = { version = "0.2.2", features = ["serde"] } -num-bigint = { version = "0.2.4", features = ["serde"] } +num-bigint = { version = "0.3", features = ["serde"] } num-traits = "0.2.8" num-integer = "0.1.41" -num-rational = "0.2.2" +num-rational = "0.3" num-iter = "0.1.39" rand = { version = "0.7", features = ["wasm-bindgen"] } rand_core = "0.5" diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 615e8a2f6..d1b73f7dd 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -124,7 +124,7 @@ fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { objfloat::float_pow(v1, v2, vm).into_pyobject(vm) } else { Ok(if let Some(v2) = int2.to_u64() { - vm.ctx.new_int(int1.pow(v2)) + vm.ctx.new_int(Pow::pow(int1, v2)) } else if int1.is_one() { vm.ctx.new_int(1) } else if int1.is_zero() { @@ -519,7 +519,7 @@ impl PyInt { #[pymethod(name = "__sizeof__")] fn sizeof(&self) -> usize { - size_of::() + ((self.value.bits() + 7) & !7) / 8 + size_of::() + (((self.value.bits() + 7) & !7) / 8) as usize } #[pymethod(name = "as_integer_ratio")] @@ -531,7 +531,7 @@ impl PyInt { } #[pymethod] - fn bit_length(&self) -> usize { + fn bit_length(&self) -> u64 { self.value.bits() } From a9a5f0662348c5ee10313b2d3d20acda7955e2f3 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 15 Jul 2020 12:54:51 -0500 Subject: [PATCH 007/209] Rewrite the pwd module to use nix instead of the pwd crate --- Cargo.lock | 73 ---------------------- vm/Cargo.toml | 3 - vm/src/stdlib/pwd.rs | 143 +++++++++++++++++++++++++------------------ 3 files changed, 82 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e810a1fd8..385b861fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,28 +104,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -[[package]] -name = "backtrace" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ed64ae6d9ebfd9893193c4b2532b1292ec97bd8271c9d7d0fa90cd78a34cba" -dependencies = [ - "backtrace-sys", - "cfg-if", - "libc", - "rustc-demangle", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "base64" version = "0.11.0" @@ -532,28 +510,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" -[[package]] -name = "failure" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" -dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", - "synstructure", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -1259,16 +1215,6 @@ dependencies = [ "unicode-xid 0.2.0", ] -[[package]] -name = "pwd" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd32d8bece608e144ca20251e714ed107cdecdabb20c2d383cfc687825106a5" -dependencies = [ - "failure", - "libc", -] - [[package]] name = "python3-sys" version = "0.2.1" @@ -1524,12 +1470,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.2.3" @@ -1685,7 +1625,6 @@ dependencies = [ "openssl-sys", "parking_lot", "paste", - "pwd", "rand 0.7.3", "rand_core 0.5.1", "regex", @@ -2001,18 +1940,6 @@ dependencies = [ "unicode-xid 0.2.0", ] -[[package]] -name = "synstructure" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" -dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", - "unicode-xid 0.2.0", -] - [[package]] name = "term" version = "0.5.2" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 50cbc9213..66c08aa9b 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -92,9 +92,6 @@ unic-ucd-ident = "0.9" flame = { version = "0.2", optional = true } flamer = { version = "0.3", optional = true } -[target.'cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))'.dependencies] -pwd = "1" - [target.'cfg(unix)'.dependencies] exitcode = "1.1.2" uname = "0.1.1" diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index 5c2872e11..a44870363 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -1,52 +1,54 @@ -use pwd::Passwd; - +use super::os::convert_nix_error; +use crate::obj::objint::PyIntRef; use crate::obj::objstr::PyStringRef; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use std::convert::TryFrom; +use std::ptr::NonNull; -impl PyValue for Passwd { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.class("pwd", "struct_passwd") +use nix::unistd::{self, User}; + +#[pystruct_sequence(name = "pwd.struct_passwd")] +struct Passwd { + pw_name: String, + pw_passwd: String, + pw_uid: u32, + pw_gid: u32, + pw_gecos: String, + pw_dir: String, + pw_shell: String, +} + +impl From for Passwd { + fn from(user: User) -> Self { + // this is just a pain... + let cstr_lossy = |s: std::ffi::CString| { + s.into_string() + .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) + }; + let pathbuf_lossy = |p: std::path::PathBuf| { + p.into_os_string() + .into_string() + .unwrap_or_else(|s| s.to_string_lossy().into_owned()) + }; + Passwd { + pw_name: user.name, + pw_passwd: cstr_lossy(user.passwd), + pw_uid: user.uid.as_raw(), + pw_gid: user.gid.as_raw(), + pw_gecos: cstr_lossy(user.gecos), + pw_dir: pathbuf_lossy(user.dir), + pw_shell: pathbuf_lossy(user.shell), + } } } -type PasswdRef = PyRef; - -impl PasswdRef { - fn pw_name(self) -> String { - self.name.clone() - } - - fn pw_passwd(self) -> Option { - self.passwd.clone() - } - - fn pw_uid(self) -> u32 { - self.uid - } - - fn pw_gid(self) -> u32 { - self.gid - } - - fn pw_gecos(self) -> Option { - self.gecos.clone() - } - - fn pw_dir(self) -> String { - self.dir.clone() - } - - fn pw_shell(self) -> String { - self.shell.clone() - } -} - -fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { - match Passwd::from_name(name.as_str()) { - Ok(Some(passwd)) => Ok(passwd), - _ => { +fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + match User::from_name(name.as_str()).map_err(|e| convert_nix_error(vm, e))? { + Some(user) => Ok(Passwd::from(user) + .into_struct_sequence(vm, vm.try_class("pwd", "struct_passwd")?)? + .into_object()), + None => { let name_repr = vm.to_repr(name.as_object())?; let message = vm.new_str(format!("getpwnam(): name not found: {}", name_repr)); Err(vm.new_key_error(message)) @@ -54,32 +56,51 @@ fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -fn pwd_getpwuid(uid: u32, vm: &VirtualMachine) -> PyResult { - match Passwd::from_uid(uid) { - Some(passwd) => Ok(passwd), - _ => { - let message = vm.new_str(format!("getpwuid(): uid not found: {}", uid)); +fn pwd_getpwuid(uid: PyIntRef, vm: &VirtualMachine) -> PyResult { + let uid_t = libc::uid_t::try_from(uid.as_bigint()).map(unistd::Uid::from_raw); + let user = match uid_t { + Ok(uid) => User::from_uid(uid).map_err(|e| convert_nix_error(vm, e))?, + Err(_) => None, + }; + match user { + Some(user) => Ok(Passwd::from(user) + .into_struct_sequence(vm, vm.try_class("pwd", "struct_passwd")?)? + .into_object()), + None => { + let message = vm.new_str(format!("getpwuid(): uid not found: {}", uid.as_bigint())); Err(vm.new_key_error(message)) } } } +// TODO: maybe merge this functionality into nix? +fn pwd_getpwall(vm: &VirtualMachine) -> PyResult { + // setpwent, getpwent, etc are not thread safe. Could use fgetpwent_r, but this is easier + static GETPWALL: parking_lot::Mutex<()> = parking_lot::const_mutex(()); + let _guard = GETPWALL.lock(); + let mut list = Vec::new(); + let cls = vm.try_class("pwd", "struct_passwd")?; + + unsafe { libc::setpwent() }; + while let Some(ptr) = NonNull::new(unsafe { libc::getpwent() }) { + let user = User::from(unsafe { ptr.as_ref() }); + let passwd = Passwd::from(user) + .into_struct_sequence(vm, cls.clone())? + .into_object(); + list.push(passwd); + } + unsafe { libc::endpwent() }; + + Ok(vm.ctx.new_list(list)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let passwd_type = py_class!(ctx, "struct_passwd", ctx.object(), { - "pw_name" => ctx.new_readonly_getset("pw_name", PasswdRef::pw_name), - "pw_passwd" => ctx.new_readonly_getset("pw_passwd", PasswdRef::pw_passwd), - "pw_uid" => ctx.new_readonly_getset("pw_uid", PasswdRef::pw_uid), - "pw_gid" => ctx.new_readonly_getset("pw_gid", PasswdRef::pw_gid), - "pw_gecos" => ctx.new_readonly_getset("pw_gecos", PasswdRef::pw_gecos), - "pw_dir" => ctx.new_readonly_getset("pw_dir", PasswdRef::pw_dir), - "pw_shell" => ctx.new_readonly_getset("pw_shell", PasswdRef::pw_shell), - }); - py_module!(vm, "pwd", { - "struct_passwd" => passwd_type, - "getpwnam" => ctx.new_function(pwd_getpwnam), - "getpwuid" => ctx.new_function(pwd_getpwuid), + "struct_passwd" => Passwd::make_class(ctx), + "getpwnam" => named_function!(ctx, pwd, getpwnam), + "getpwuid" => named_function!(ctx, pwd, getpwuid), + "getpwall" => named_function!(ctx, pwd, getpwall), }) } From 5e86f91fe95615bdbc03714f94150372a20f7849 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 15 Jul 2020 12:55:15 -0500 Subject: [PATCH 008/209] Add test_pwd.py from CPython 3.8.2 --- Lib/test/test_pwd.py | 112 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 Lib/test/test_pwd.py diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py new file mode 100644 index 000000000..c13a7c929 --- /dev/null +++ b/Lib/test/test_pwd.py @@ -0,0 +1,112 @@ +import sys +import unittest +from test import support + +pwd = support.import_module('pwd') + +@unittest.skipUnless(hasattr(pwd, 'getpwall'), 'Does not have getpwall()') +class PwdTest(unittest.TestCase): + + def test_values(self): + entries = pwd.getpwall() + + for e in entries: + self.assertEqual(len(e), 7) + self.assertEqual(e[0], e.pw_name) + self.assertIsInstance(e.pw_name, str) + self.assertEqual(e[1], e.pw_passwd) + self.assertIsInstance(e.pw_passwd, str) + self.assertEqual(e[2], e.pw_uid) + self.assertIsInstance(e.pw_uid, int) + self.assertEqual(e[3], e.pw_gid) + self.assertIsInstance(e.pw_gid, int) + self.assertEqual(e[4], e.pw_gecos) + self.assertIsInstance(e.pw_gecos, str) + self.assertEqual(e[5], e.pw_dir) + self.assertIsInstance(e.pw_dir, str) + self.assertEqual(e[6], e.pw_shell) + self.assertIsInstance(e.pw_shell, str) + + # The following won't work, because of duplicate entries + # for one uid + # self.assertEqual(pwd.getpwuid(e.pw_uid), e) + # instead of this collect all entries for one uid + # and check afterwards (done in test_values_extended) + + def test_values_extended(self): + entries = pwd.getpwall() + entriesbyname = {} + entriesbyuid = {} + + if len(entries) > 1000: # Huge passwd file (NIS?) -- skip this test + self.skipTest('passwd file is huge; extended test skipped') + + for e in entries: + entriesbyname.setdefault(e.pw_name, []).append(e) + entriesbyuid.setdefault(e.pw_uid, []).append(e) + + # check whether the entry returned by getpwuid() + # for each uid is among those from getpwall() for this uid + for e in entries: + if not e[0] or e[0] == '+': + continue # skip NIS entries etc. + self.assertIn(pwd.getpwnam(e.pw_name), entriesbyname[e.pw_name]) + self.assertIn(pwd.getpwuid(e.pw_uid), entriesbyuid[e.pw_uid]) + + def test_errors(self): + self.assertRaises(TypeError, pwd.getpwuid) + self.assertRaises(TypeError, pwd.getpwuid, 3.14) + self.assertRaises(TypeError, pwd.getpwnam) + self.assertRaises(TypeError, pwd.getpwnam, 42) + self.assertRaises(TypeError, pwd.getpwall, 42) + + # try to get some errors + bynames = {} + byuids = {} + for (n, p, u, g, gecos, d, s) in pwd.getpwall(): + bynames[n] = u + byuids[u] = n + + allnames = list(bynames.keys()) + namei = 0 + fakename = allnames[namei] + while fakename in bynames: + chars = list(fakename) + for i in range(len(chars)): + if chars[i] == 'z': + chars[i] = 'A' + break + elif chars[i] == 'Z': + continue + else: + chars[i] = chr(ord(chars[i]) + 1) + break + else: + namei = namei + 1 + try: + fakename = allnames[namei] + except IndexError: + # should never happen... if so, just forget it + break + fakename = ''.join(chars) + + self.assertRaises(KeyError, pwd.getpwnam, fakename) + + # In some cases, byuids isn't a complete list of all users in the + # system, so if we try to pick a value not in byuids (via a perturbing + # loop, say), pwd.getpwuid() might still be able to find data for that + # uid. Using sys.maxint may provoke the same problems, but hopefully + # it will be a more repeatable failure. + fakeuid = sys.maxsize + self.assertNotIn(fakeuid, byuids) + self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + + # -1 shouldn't be a valid uid because it has a special meaning in many + # uid-related functions + self.assertRaises(KeyError, pwd.getpwuid, -1) + # should be out of uid_t range + self.assertRaises(KeyError, pwd.getpwuid, 2**128) + self.assertRaises(KeyError, pwd.getpwuid, -2**128) + +if __name__ == "__main__": + unittest.main() From 72a78ff542339faf86405e704ca25816347afa10 Mon Sep 17 00:00:00 2001 From: qodot Date: Thu, 16 Jul 2020 22:50:22 +0900 Subject: [PATCH 009/209] Implement lt, le, gt, ge method of array --- vm/src/stdlib/array.rs | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 3b90d38f8..5eaf7e2a6 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -412,6 +412,78 @@ impl PyArray { } } + #[pymethod(name = "__lt__")] + fn lt(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let lhs = class_or_notimplemented!(vm, Self, lhs); + let rhs = class_or_notimplemented!(vm, Self, rhs); + let lhs = lhs.borrow_value(); + let rhs = rhs.borrow_value(); + + for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { + let lt = objbool::boolval(vm, vm._lt(a?, b?)?)?; + + if lt { + return Ok(vm.new_bool(true)); + } + } + + Ok(vm.new_bool(lhs.len() < rhs.len())) + } + + #[pymethod(name = "__le__")] + fn le(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let lhs = class_or_notimplemented!(vm, Self, lhs); + let rhs = class_or_notimplemented!(vm, Self, rhs); + let lhs = lhs.borrow_value(); + let rhs = rhs.borrow_value(); + + for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { + let le = objbool::boolval(vm, vm._le(a?, b?)?)?; + + if le { + return Ok(vm.new_bool(true)); + } + } + + Ok(vm.new_bool(lhs.len() <= rhs.len())) + } + + #[pymethod(name = "__gt__")] + fn gt(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let lhs = class_or_notimplemented!(vm, Self, lhs); + let rhs = class_or_notimplemented!(vm, Self, rhs); + let lhs = lhs.borrow_value(); + let rhs = rhs.borrow_value(); + + for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { + let gt = objbool::boolval(vm, vm._gt(a?, b?)?)?; + + if gt { + return Ok(vm.new_bool(true)); + } + } + + Ok(vm.new_bool(lhs.len() > rhs.len())) + } + + #[pymethod(name = "__ge__")] + fn ge(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let lhs = class_or_notimplemented!(vm, Self, lhs); + let rhs = class_or_notimplemented!(vm, Self, rhs); + let lhs = lhs.borrow_value(); + let rhs = rhs.borrow_value(); + + for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { + let ge = objbool::boolval(vm, vm._ge(a?, b?)?)?; + + if ge { + return Ok(vm.new_bool(true)); + } + } + + Ok(vm.new_bool(lhs.len() >= rhs.len())) + } + #[pymethod(name = "__len__")] fn len(&self) -> usize { self.borrow_value().len() From 40a5e64bf8c1bf1e26855d50f37e59d3b8115395 Mon Sep 17 00:00:00 2001 From: qodot Date: Fri, 17 Jul 2020 10:18:23 +0900 Subject: [PATCH 010/209] Fix rustfmt and clippy followed message, - using `Result.or_else(|x| Err(y))`, which is more succinctly expressed as `map_err(|x| y)` --- vm/src/obj/objbyteinner.rs | 6 +++--- vm/src/stdlib/os.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 730e44e77..02139637b 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -115,11 +115,11 @@ impl ByteInnerNewOptions { let bytes = vm.invoke(&bytes_method?, vec![])?; return PyByteInner::try_from_object(vm, bytes); } - let elements = vm.extract_elements(&obj).or_else(|_| { - Err(vm.new_type_error(format!( + let elements = vm.extract_elements(&obj).map_err(|_| { + vm.new_type_error(format!( "cannot convert '{}' object to bytes", obj.class().name - ))) + )) })?; let mut data_bytes = vec![]; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b045f1d7e..07c4cb3e3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1118,7 +1118,7 @@ fn os_set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<( } Ok(()) }; - _set_flag().or_else(|err| Err(convert_nix_error(vm, err))) + _set_flag().map_err(|err| convert_nix_error(vm, err)) } #[cfg(unix)] @@ -1128,10 +1128,10 @@ fn os_pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { let (rfd, wfd) = pipe().map_err(|err| convert_nix_error(vm, err))?; os_set_inheritable(rfd.into(), false, vm) .and_then(|_| os_set_inheritable(wfd.into(), false, vm)) - .or_else(|err| { + .map_err(|err| { let _ = close(rfd); let _ = close(wfd); - Err(err) + err })?; Ok((rfd, wfd)) } From c4378eb2752a88d5283f4eea72728c49e96b1099 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 15 Jul 2020 19:07:39 -0500 Subject: [PATCH 011/209] Fix compilation on wasi --- vm/src/stdlib/os.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 07c4cb3e3..b66a284d4 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -138,6 +138,11 @@ impl OutputMode { use std::os::unix::ffi::OsStringExt; Ok(vm.ctx.new_bytes(path.into_os_string().into_vec())) } + #[cfg(target_os = "wasi")] + { + use std::os::wasi::ffi::OsStringExt; + Ok(vm.ctx.new_bytes(path.into_os_string().into_vec())) + } #[cfg(windows)] { path_as_string(path).map(|s| vm.ctx.new_bytes(s.into_bytes())) @@ -1504,6 +1509,7 @@ struct UtimeArgs { _follow_symlinks: FollowSymlinks, } +#[cfg(not(target_os = "wasi"))] fn os_utime(args: UtimeArgs, vm: &VirtualMachine) -> PyResult<()> { let parse_tup = |tup: PyTupleRef| -> Option<(i64, i64)> { let tup = tup.as_slice(); @@ -1931,6 +1937,7 @@ struct PyTerminalSize { lines: usize, } +#[cfg(not(target_os = "wasi"))] fn os_get_terminal_size(fd: OptionalArg, vm: &VirtualMachine) -> PyResult { let (columns, lines) = { #[cfg(unix)] @@ -2073,6 +2080,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "symlink", os_symlink, None, Some(false), None), // truncate Some None None SupportFunc::new(vm, "unlink", os_remove, Some(false), Some(false), None), + #[cfg(not(target_os = "wasi"))] SupportFunc::new(vm, "utime", os_utime, Some(false), Some(false), Some(false)), ]; #[cfg(unix)] @@ -2114,7 +2122,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "link" => ctx.new_function(os_link), "kill" => ctx.new_function(os_kill), "strerror" => ctx.new_function(os_strerror), - "get_terminal_size" => ctx.new_function(os_get_terminal_size), "O_RDONLY" => ctx.new_int(libc::O_RDONLY), "O_WRONLY" => ctx.new_int(libc::O_WRONLY), @@ -2161,6 +2168,11 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "supports_follow_symlinks" => supports_follow_symlinks.into_object(), }); + #[cfg(not(target_os = "wasi"))] + extend_module!(vm, module, { + "get_terminal_size" => ctx.new_function(os_get_terminal_size), + }); + extend_module_platform_specific(&vm, &module); module From 7169541232fb9948092805c5004e2c321fc8306f Mon Sep 17 00:00:00 2001 From: Minjun Shin Date: Sat, 18 Jul 2020 05:22:24 +0900 Subject: [PATCH 012/209] Added CPython 3.8.3's test___future__.py. Works perfectly fine, out of the box --- Lib/test/test___future__.py | 61 +++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 Lib/test/test___future__.py diff --git a/Lib/test/test___future__.py b/Lib/test/test___future__.py new file mode 100644 index 000000000..559a1873a --- /dev/null +++ b/Lib/test/test___future__.py @@ -0,0 +1,61 @@ +import unittest +import __future__ + +GOOD_SERIALS = ("alpha", "beta", "candidate", "final") + +features = __future__.all_feature_names + +class FutureTest(unittest.TestCase): + + def test_names(self): + # Verify that all_feature_names appears correct. + given_feature_names = features[:] + for name in dir(__future__): + obj = getattr(__future__, name, None) + if obj is not None and isinstance(obj, __future__._Feature): + self.assertTrue( + name in given_feature_names, + "%r should have been in all_feature_names" % name + ) + given_feature_names.remove(name) + self.assertEqual(len(given_feature_names), 0, + "all_feature_names has too much: %r" % given_feature_names) + + def test_attributes(self): + for feature in features: + value = getattr(__future__, feature) + + optional = value.getOptionalRelease() + mandatory = value.getMandatoryRelease() + + a = self.assertTrue + e = self.assertEqual + def check(t, name): + a(isinstance(t, tuple), "%s isn't tuple" % name) + e(len(t), 5, "%s isn't 5-tuple" % name) + (major, minor, micro, level, serial) = t + a(isinstance(major, int), "%s major isn't int" % name) + a(isinstance(minor, int), "%s minor isn't int" % name) + a(isinstance(micro, int), "%s micro isn't int" % name) + a(isinstance(level, str), + "%s level isn't string" % name) + a(level in GOOD_SERIALS, + "%s level string has unknown value" % name) + a(isinstance(serial, int), "%s serial isn't int" % name) + + check(optional, "optional") + if mandatory is not None: + check(mandatory, "mandatory") + a(optional < mandatory, + "optional not less than mandatory, and mandatory not None") + + a(hasattr(value, "compiler_flag"), + "feature is missing a .compiler_flag attr") + # Make sure the compile accepts the flag. + compile("", "", "exec", value.compiler_flag) + a(isinstance(getattr(value, "compiler_flag"), int), + ".compiler_flag isn't int") + + +if __name__ == "__main__": + unittest.main() From ce2fc345ad938b107e681dfff7c03898468bf9b2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Jul 2020 09:02:51 +0900 Subject: [PATCH 013/209] enhance github actions cache --- .github/workflows/ci.yaml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7bd1504fc..69a06575f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,10 +25,11 @@ jobs: - name: Cache cargo dependencies uses: actions/cache@v2 with: - key: ${{ runner.os }}-rust_tests-${{ hashFiles('Cargo.lock') }} - path: target - restore-keys: | - ${{ runner.os }}-rust_tests- + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }} - name: run rust tests uses: actions-rs/cargo@v1 with: @@ -51,10 +52,11 @@ jobs: - name: Cache cargo dependencies uses: actions/cache@v2 with: - key: ${{ runner.os }}-snippets-${{ hashFiles('Cargo.lock') }} - path: target - restore-keys: | - ${{ runner.os }}-snippets- + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-release-${{ hashFiles('Cargo.lock') }} - name: build rustpython uses: actions-rs/cargo@v1 with: @@ -122,10 +124,13 @@ jobs: - name: Cache cargo dependencies uses: actions/cache@v2 with: + path: | + ~/.cargo/registry + ~/.cargo/git + target key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }} - path: target restore-keys: | - ${{ runner.os }}-wasm- + ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }} - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver From 35825676869863ba9ae917267fc3ccd7d390ded6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Jul 2020 09:03:12 +0900 Subject: [PATCH 014/209] skip platform independent tests for macOS --- .github/workflows/ci.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 69a06575f..4eaf4c247 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -79,7 +79,12 @@ jobs: run: target/release/rustpython -m test -v env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - if: runner.os != 'Windows' + if: runner.os == 'Linux' + - name: run cpython tests (macOS lightweight) + run: target/release/rustpython -m test -x test_argparse -x test_json -x test_bytes -x test_long -v + env: + RUSTPYTHONPATH: ${{ github.workspace }}/Lib + if: runner.os == 'macOS' format: name: Check Rust code with rustfmt and clippy @@ -131,6 +136,7 @@ jobs: key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }} restore-keys: | ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }} + ${{ runner.os }}-debug- - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver From ce0b511a3199737e35b09fa647539bd63f37d70a Mon Sep 17 00:00:00 2001 From: Antonio Yang Date: Sat, 18 Jul 2020 10:12:23 +0800 Subject: [PATCH 015/209] init nestied formatting test case --- tests/snippets/builtin_format.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/snippets/builtin_format.py b/tests/snippets/builtin_format.py index 46b627ad5..a30f300d4 100644 --- a/tests/snippets/builtin_format.py +++ b/tests/snippets/builtin_format.py @@ -27,3 +27,4 @@ assert f'{100:_}' == '100' assert f'{1024:_}' == '1_024' assert f'{65536:,}' == '65,536' assert f'{4294967296:,}' == '4,294,967,296' +assert 'F' == "{0:{base}}".format(15, base="X") From d71223b54603c2717c96bb487f2021aaaf2a7229 Mon Sep 17 00:00:00 2001 From: CPython dev <> Date: Sat, 18 Jul 2020 05:00:01 +0900 Subject: [PATCH 016/209] test_zlib from CPython 3.8.3 --- Lib/test/test_zlib.py | 922 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 922 insertions(+) create mode 100644 Lib/test/test_zlib.py diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py new file mode 100644 index 000000000..f828b4c73 --- /dev/null +++ b/Lib/test/test_zlib.py @@ -0,0 +1,922 @@ +import unittest +from test import support +import binascii +import copy +import pickle +import random +import sys +from test.support import bigmemtest, _1G, _4G + +zlib = support.import_module('zlib') + +requires_Compress_copy = unittest.skipUnless( + hasattr(zlib.compressobj(), "copy"), + 'requires Compress.copy()') +requires_Decompress_copy = unittest.skipUnless( + hasattr(zlib.decompressobj(), "copy"), + 'requires Decompress.copy()') + + +class VersionTestCase(unittest.TestCase): + + def test_library_version(self): + # Test that the major version of the actual library in use matches the + # major version that we were compiled against. We can't guarantee that + # the minor versions will match (even on the machine on which the module + # was compiled), and the API is stable between minor versions, so + # testing only the major versions avoids spurious failures. + self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0]) + + +class ChecksumTestCase(unittest.TestCase): + # checksum test cases + def test_crc32start(self): + self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0)) + self.assertTrue(zlib.crc32(b"abc", 0xffffffff)) + + def test_crc32empty(self): + self.assertEqual(zlib.crc32(b"", 0), 0) + self.assertEqual(zlib.crc32(b"", 1), 1) + self.assertEqual(zlib.crc32(b"", 432), 432) + + def test_adler32start(self): + self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1)) + self.assertTrue(zlib.adler32(b"abc", 0xffffffff)) + + def test_adler32empty(self): + self.assertEqual(zlib.adler32(b"", 0), 0) + self.assertEqual(zlib.adler32(b"", 1), 1) + self.assertEqual(zlib.adler32(b"", 432), 432) + + def test_penguins(self): + self.assertEqual(zlib.crc32(b"penguin", 0), 0x0e5c1a120) + self.assertEqual(zlib.crc32(b"penguin", 1), 0x43b6aa94) + self.assertEqual(zlib.adler32(b"penguin", 0), 0x0bcf02f6) + self.assertEqual(zlib.adler32(b"penguin", 1), 0x0bd602f7) + + self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0)) + self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1)) + + def test_crc32_adler32_unsigned(self): + foo = b'abcdefghijklmnop' + # explicitly test signed behavior + self.assertEqual(zlib.crc32(foo), 2486878355) + self.assertEqual(zlib.crc32(b'spam'), 1138425661) + self.assertEqual(zlib.adler32(foo+foo), 3573550353) + self.assertEqual(zlib.adler32(b'spam'), 72286642) + + def test_same_as_binascii_crc32(self): + foo = b'abcdefghijklmnop' + crc = 2486878355 + self.assertEqual(binascii.crc32(foo), crc) + self.assertEqual(zlib.crc32(foo), crc) + self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam')) + + +# Issue #10276 - check that inputs >=4 GiB are handled correctly. +class ChecksumBigBufferTestCase(unittest.TestCase): + + @bigmemtest(size=_4G + 4, memuse=1, dry_run=False) + def test_big_buffer(self, size): + data = b"nyan" * (_1G + 1) + self.assertEqual(zlib.crc32(data), 1044521549) + self.assertEqual(zlib.adler32(data), 2256789997) + + +class ExceptionTestCase(unittest.TestCase): + # make sure we generate some expected errors + def test_badlevel(self): + # specifying compression level out of range causes an error + # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib + # accepts 0 too) + self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10) + + def test_badargs(self): + self.assertRaises(TypeError, zlib.adler32) + self.assertRaises(TypeError, zlib.crc32) + self.assertRaises(TypeError, zlib.compress) + self.assertRaises(TypeError, zlib.decompress) + for arg in (42, None, '', 'abc', (), []): + self.assertRaises(TypeError, zlib.adler32, arg) + self.assertRaises(TypeError, zlib.crc32, arg) + self.assertRaises(TypeError, zlib.compress, arg) + self.assertRaises(TypeError, zlib.decompress, arg) + + def test_badcompressobj(self): + # verify failure on building compress object with bad params + self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0) + # specifying total bits too large causes an error + self.assertRaises(ValueError, + zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1) + + def test_baddecompressobj(self): + # verify failure on building decompress object with bad params + self.assertRaises(ValueError, zlib.decompressobj, -1) + + def test_decompressobj_badflush(self): + # verify failure on calling decompressobj.flush with bad params + self.assertRaises(ValueError, zlib.decompressobj().flush, 0) + self.assertRaises(ValueError, zlib.decompressobj().flush, -1) + + @support.cpython_only + def test_overflow(self): + with self.assertRaisesRegex(OverflowError, 'int too large'): + zlib.decompress(b'', 15, sys.maxsize + 1) + with self.assertRaisesRegex(OverflowError, 'int too large'): + zlib.decompressobj().decompress(b'', sys.maxsize + 1) + with self.assertRaisesRegex(OverflowError, 'int too large'): + zlib.decompressobj().flush(sys.maxsize + 1) + + +class BaseCompressTestCase(object): + def check_big_compress_buffer(self, size, compress_func): + _1M = 1024 * 1024 + # Generate 10 MiB worth of random, and expand it by repeating it. + # The assumption is that zlib's memory is not big enough to exploit + # such spread out redundancy. + data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little') + for i in range(10)]) + data = data * (size // len(data) + 1) + try: + compress_func(data) + finally: + # Release memory + data = None + + def check_big_decompress_buffer(self, size, decompress_func): + data = b'x' * size + try: + compressed = zlib.compress(data, 1) + finally: + # Release memory + data = None + data = decompress_func(compressed) + # Sanity check + try: + self.assertEqual(len(data), size) + self.assertEqual(len(data.strip(b'x')), 0) + finally: + data = None + + +class CompressTestCase(BaseCompressTestCase, unittest.TestCase): + # Test compression in one go (whole message compression) + def test_speech(self): + x = zlib.compress(HAMLET_SCENE) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + + def test_keywords(self): + x = zlib.compress(HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + with self.assertRaises(TypeError): + zlib.compress(data=HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x, + wbits=zlib.MAX_WBITS, + bufsize=zlib.DEF_BUF_SIZE), + HAMLET_SCENE) + + def test_speech128(self): + # compress more data + data = HAMLET_SCENE * 128 + x = zlib.compress(data) + self.assertEqual(zlib.compress(bytearray(data)), x) + for ob in x, bytearray(x): + self.assertEqual(zlib.decompress(ob), data) + + def test_incomplete_stream(self): + # A useful error message is given + x = zlib.compress(HAMLET_SCENE) + self.assertRaisesRegex(zlib.error, + "Error -5 while decompressing data: incomplete or truncated stream", + zlib.decompress, x[:-1]) + + # Memory use of the following functions takes into account overallocation + + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) + def test_big_compress_buffer(self, size): + compress = lambda s: zlib.compress(s, 1) + self.check_big_compress_buffer(size, compress) + + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) + def test_big_decompress_buffer(self, size): + self.check_big_decompress_buffer(size, zlib.decompress) + + @bigmemtest(size=_4G, memuse=1) + def test_large_bufsize(self, size): + # Test decompress(bufsize) parameter greater than the internal limit + data = HAMLET_SCENE * 10 + compressed = zlib.compress(data, 1) + self.assertEqual(zlib.decompress(compressed, 15, size), data) + + def test_custom_bufsize(self): + data = HAMLET_SCENE * 10 + compressed = zlib.compress(data, 1) + self.assertEqual(zlib.decompress(compressed, 15, CustomInt()), data) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=_4G + 100, memuse=4) + def test_64bit_compress(self, size): + data = b'x' * size + try: + comp = zlib.compress(data, 0) + self.assertEqual(zlib.decompress(comp), data) + finally: + comp = data = None + + +class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): + # Test compression object + def test_pair(self): + # straightforward compress/decompress objects + datasrc = HAMLET_SCENE * 128 + datazip = zlib.compress(datasrc) + # should compress both bytes and bytearray data + for data in (datasrc, bytearray(datasrc)): + co = zlib.compressobj() + x1 = co.compress(data) + x2 = co.flush() + self.assertRaises(zlib.error, co.flush) # second flush should not work + self.assertEqual(x1 + x2, datazip) + for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): + dco = zlib.decompressobj() + y1 = dco.decompress(v1 + v2) + y2 = dco.flush() + self.assertEqual(data, y1 + y2) + self.assertIsInstance(dco.unconsumed_tail, bytes) + self.assertIsInstance(dco.unused_data, bytes) + + def test_keywords(self): + level = 2 + method = zlib.DEFLATED + wbits = -12 + memLevel = 9 + strategy = zlib.Z_FILTERED + co = zlib.compressobj(level=level, + method=method, + wbits=wbits, + memLevel=memLevel, + strategy=strategy, + zdict=b"") + do = zlib.decompressobj(wbits=wbits, zdict=b"") + with self.assertRaises(TypeError): + co.compress(data=HAMLET_SCENE) + with self.assertRaises(TypeError): + do.decompress(data=zlib.compress(HAMLET_SCENE)) + x = co.compress(HAMLET_SCENE) + co.flush() + y = do.decompress(x, max_length=len(HAMLET_SCENE)) + do.flush() + self.assertEqual(HAMLET_SCENE, y) + + def test_compressoptions(self): + # specify lots of options to compressobj() + level = 2 + method = zlib.DEFLATED + wbits = -12 + memLevel = 9 + strategy = zlib.Z_FILTERED + co = zlib.compressobj(level, method, wbits, memLevel, strategy) + x1 = co.compress(HAMLET_SCENE) + x2 = co.flush() + dco = zlib.decompressobj(wbits) + y1 = dco.decompress(x1 + x2) + y2 = dco.flush() + self.assertEqual(HAMLET_SCENE, y1 + y2) + + def test_compressincremental(self): + # compress object in steps, decompress object as one-shot + data = HAMLET_SCENE * 128 + co = zlib.compressobj() + bufs = [] + for i in range(0, len(data), 256): + bufs.append(co.compress(data[i:i+256])) + bufs.append(co.flush()) + combuf = b''.join(bufs) + + dco = zlib.decompressobj() + y1 = dco.decompress(b''.join(bufs)) + y2 = dco.flush() + self.assertEqual(data, y1 + y2) + + def test_decompinc(self, flush=False, source=None, cx=256, dcx=64): + # compress object in steps, decompress object in steps + source = source or HAMLET_SCENE + data = source * 128 + co = zlib.compressobj() + bufs = [] + for i in range(0, len(data), cx): + bufs.append(co.compress(data[i:i+cx])) + bufs.append(co.flush()) + combuf = b''.join(bufs) + + decombuf = zlib.decompress(combuf) + # Test type of return value + self.assertIsInstance(decombuf, bytes) + + self.assertEqual(data, decombuf) + + dco = zlib.decompressobj() + bufs = [] + for i in range(0, len(combuf), dcx): + bufs.append(dco.decompress(combuf[i:i+dcx])) + self.assertEqual(b'', dco.unconsumed_tail, ######## + "(A) uct should be b'': not %d long" % + len(dco.unconsumed_tail)) + self.assertEqual(b'', dco.unused_data) + if flush: + bufs.append(dco.flush()) + else: + while True: + chunk = dco.decompress(b'') + if chunk: + bufs.append(chunk) + else: + break + self.assertEqual(b'', dco.unconsumed_tail, ######## + "(B) uct should be b'': not %d long" % + len(dco.unconsumed_tail)) + self.assertEqual(b'', dco.unused_data) + self.assertEqual(data, b''.join(bufs)) + # Failure means: "decompressobj with init options failed" + + def test_decompincflush(self): + self.test_decompinc(flush=True) + + def test_decompimax(self, source=None, cx=256, dcx=64): + # compress in steps, decompress in length-restricted steps + source = source or HAMLET_SCENE + # Check a decompression object with max_length specified + data = source * 128 + co = zlib.compressobj() + bufs = [] + for i in range(0, len(data), cx): + bufs.append(co.compress(data[i:i+cx])) + bufs.append(co.flush()) + combuf = b''.join(bufs) + self.assertEqual(data, zlib.decompress(combuf), + 'compressed data failure') + + dco = zlib.decompressobj() + bufs = [] + cb = combuf + while cb: + #max_length = 1 + len(cb)//10 + chunk = dco.decompress(cb, dcx) + self.assertFalse(len(chunk) > dcx, + 'chunk too big (%d>%d)' % (len(chunk), dcx)) + bufs.append(chunk) + cb = dco.unconsumed_tail + bufs.append(dco.flush()) + self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') + + def test_decompressmaxlen(self, flush=False): + # Check a decompression object with max_length specified + data = HAMLET_SCENE * 128 + co = zlib.compressobj() + bufs = [] + for i in range(0, len(data), 256): + bufs.append(co.compress(data[i:i+256])) + bufs.append(co.flush()) + combuf = b''.join(bufs) + self.assertEqual(data, zlib.decompress(combuf), + 'compressed data failure') + + dco = zlib.decompressobj() + bufs = [] + cb = combuf + while cb: + max_length = 1 + len(cb)//10 + chunk = dco.decompress(cb, max_length) + self.assertFalse(len(chunk) > max_length, + 'chunk too big (%d>%d)' % (len(chunk),max_length)) + bufs.append(chunk) + cb = dco.unconsumed_tail + if flush: + bufs.append(dco.flush()) + else: + while chunk: + chunk = dco.decompress(b'', max_length) + self.assertFalse(len(chunk) > max_length, + 'chunk too big (%d>%d)' % (len(chunk),max_length)) + bufs.append(chunk) + self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') + + def test_decompressmaxlenflush(self): + self.test_decompressmaxlen(flush=True) + + def test_maxlenmisc(self): + # Misc tests of max_length + dco = zlib.decompressobj() + self.assertRaises(ValueError, dco.decompress, b"", -1) + self.assertEqual(b'', dco.unconsumed_tail) + + def test_maxlen_large(self): + # Sizes up to sys.maxsize should be accepted, although zlib is + # internally limited to expressing sizes with unsigned int + data = HAMLET_SCENE * 10 + self.assertGreater(len(data), zlib.DEF_BUF_SIZE) + compressed = zlib.compress(data, 1) + dco = zlib.decompressobj() + self.assertEqual(dco.decompress(compressed, sys.maxsize), data) + + def test_maxlen_custom(self): + data = HAMLET_SCENE * 10 + compressed = zlib.compress(data, 1) + dco = zlib.decompressobj() + self.assertEqual(dco.decompress(compressed, CustomInt()), data[:100]) + + def test_clear_unconsumed_tail(self): + # Issue #12050: calling decompress() without providing max_length + # should clear the unconsumed_tail attribute. + cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc" + dco = zlib.decompressobj() + ddata = dco.decompress(cdata, 1) + ddata += dco.decompress(dco.unconsumed_tail) + self.assertEqual(dco.unconsumed_tail, b"") + + def test_flushes(self): + # Test flush() with the various options, using all the + # different levels in order to provide more variations. + sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', + 'Z_PARTIAL_FLUSH'] + + ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) + # Z_BLOCK has a known failure prior to 1.2.5.3 + if ver >= (1, 2, 5, 3): + sync_opt.append('Z_BLOCK') + + sync_opt = [getattr(zlib, opt) for opt in sync_opt + if hasattr(zlib, opt)] + data = HAMLET_SCENE * 8 + + for sync in sync_opt: + for level in range(10): + try: + obj = zlib.compressobj( level ) + a = obj.compress( data[:3000] ) + b = obj.flush( sync ) + c = obj.compress( data[3000:] ) + d = obj.flush() + except: + print("Error for flush mode={}, level={}" + .format(sync, level)) + raise + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj + + @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), + 'requires zlib.Z_SYNC_FLUSH') + def test_odd_flush(self): + # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1 + import random + # Testing on 17K of "random" data + + # Create compressor and decompressor objects + co = zlib.compressobj(zlib.Z_BEST_COMPRESSION) + dco = zlib.decompressobj() + + # Try 17K of data + # generate random data stream + try: + # In 2.3 and later, WichmannHill is the RNG of the bug report + gen = random.WichmannHill() + except AttributeError: + try: + # 2.2 called it Random + gen = random.Random() + except AttributeError: + # others might simply have a single RNG + gen = random + gen.seed(1) + data = genblock(1, 17 * 1024, generator=gen) + + # compress, sync-flush, and decompress + first = co.compress(data) + second = co.flush(zlib.Z_SYNC_FLUSH) + expanded = dco.decompress(first + second) + + # if decompressed data is different from the input data, choke. + self.assertEqual(expanded, data, "17K random source doesn't match") + + def test_empty_flush(self): + # Test that calling .flush() on unused objects works. + # (Bug #1083110 -- calling .flush() on decompress objects + # caused a core dump.) + + co = zlib.compressobj(zlib.Z_BEST_COMPRESSION) + self.assertTrue(co.flush()) # Returns a zlib header + dco = zlib.decompressobj() + self.assertEqual(dco.flush(), b"") # Returns nothing + + def test_dictionary(self): + h = HAMLET_SCENE + # Build a simulated dictionary out of the words in HAMLET. + words = h.split() + random.shuffle(words) + zdict = b''.join(words) + # Use it to compress HAMLET. + co = zlib.compressobj(zdict=zdict) + cd = co.compress(h) + co.flush() + # Verify that it will decompress with the dictionary. + dco = zlib.decompressobj(zdict=zdict) + self.assertEqual(dco.decompress(cd) + dco.flush(), h) + # Verify that it fails when not given the dictionary. + dco = zlib.decompressobj() + self.assertRaises(zlib.error, dco.decompress, cd) + + def test_dictionary_streaming(self): + # This simulates the reuse of a compressor object for compressing + # several separate data streams. + co = zlib.compressobj(zdict=HAMLET_SCENE) + do = zlib.decompressobj(zdict=HAMLET_SCENE) + piece = HAMLET_SCENE[1000:1500] + d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH) + d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH) + d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH) + self.assertEqual(do.decompress(d0), piece) + self.assertEqual(do.decompress(d1), piece[100:]) + self.assertEqual(do.decompress(d2), piece[:-100]) + + def test_decompress_incomplete_stream(self): + # This is 'foo', deflated + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' + # For the record + self.assertEqual(zlib.decompress(x), b'foo') + self.assertRaises(zlib.error, zlib.decompress, x[:-5]) + # Omitting the stream end works with decompressor objects + # (see issue #8672). + dco = zlib.decompressobj() + y = dco.decompress(x[:-5]) + y += dco.flush() + self.assertEqual(y, b'foo') + + def test_decompress_eof(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.decompress(x[-5:]) + self.assertTrue(dco.eof) + dco.flush() + self.assertTrue(dco.eof) + + def test_decompress_eof_incomplete_stream(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.flush() + self.assertFalse(dco.eof) + + def test_decompress_unused_data(self): + # Repeated calls to decompress() after EOF should accumulate data in + # dco.unused_data, instead of just storing the arg to the last call. + source = b'abcdefghijklmnopqrstuvwxyz' + remainder = b'0123456789' + y = zlib.compress(source) + x = y + remainder + for maxlen in 0, 1000: + for step in 1, 2, len(y), len(x): + dco = zlib.decompressobj() + data = b'' + for i in range(0, len(x), step): + if i < len(y): + self.assertEqual(dco.unused_data, b'') + if maxlen == 0: + data += dco.decompress(x[i : i + step]) + self.assertEqual(dco.unconsumed_tail, b'') + else: + data += dco.decompress( + dco.unconsumed_tail + x[i : i + step], maxlen) + data += dco.flush() + self.assertTrue(dco.eof) + self.assertEqual(data, source) + self.assertEqual(dco.unconsumed_tail, b'') + self.assertEqual(dco.unused_data, remainder) + + # issue27164 + def test_decompress_raw_with_dictionary(self): + zdict = b'abcdefghijklmnopqrstuvwxyz' + co = zlib.compressobj(wbits=-zlib.MAX_WBITS, zdict=zdict) + comp = co.compress(zdict) + co.flush() + dco = zlib.decompressobj(wbits=-zlib.MAX_WBITS, zdict=zdict) + uncomp = dco.decompress(comp) + dco.flush() + self.assertEqual(zdict, uncomp) + + def test_flush_with_freed_input(self): + # Issue #16411: decompressor accesses input to last decompress() call + # in flush(), even if this object has been freed in the meanwhile. + input1 = b'abcdefghijklmnopqrstuvwxyz' + input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM' + data = zlib.compress(input1) + dco = zlib.decompressobj() + dco.decompress(data, 1) + del data + data = zlib.compress(input2) + self.assertEqual(dco.flush(), input1[1:]) + + @bigmemtest(size=_4G, memuse=1) + def test_flush_large_length(self, size): + # Test flush(length) parameter greater than internal limit UINT_MAX + input = HAMLET_SCENE * 10 + data = zlib.compress(input, 1) + dco = zlib.decompressobj() + dco.decompress(data, 1) + self.assertEqual(dco.flush(size), input[1:]) + + def test_flush_custom_length(self): + input = HAMLET_SCENE * 10 + data = zlib.compress(input, 1) + dco = zlib.decompressobj() + dco.decompress(data, 1) + self.assertEqual(dco.flush(CustomInt()), input[1:]) + + @requires_Compress_copy + def test_compresscopy(self): + # Test copying a compression object + data0 = HAMLET_SCENE + data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii") + for func in lambda c: c.copy(), copy.copy, copy.deepcopy: + c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION) + bufs0 = [] + bufs0.append(c0.compress(data0)) + + c1 = func(c0) + bufs1 = bufs0[:] + + bufs0.append(c0.compress(data0)) + bufs0.append(c0.flush()) + s0 = b''.join(bufs0) + + bufs1.append(c1.compress(data1)) + bufs1.append(c1.flush()) + s1 = b''.join(bufs1) + + self.assertEqual(zlib.decompress(s0),data0+data0) + self.assertEqual(zlib.decompress(s1),data0+data1) + + @requires_Compress_copy + def test_badcompresscopy(self): + # Test copying a compression object in an inconsistent state + c = zlib.compressobj() + c.compress(HAMLET_SCENE) + c.flush() + self.assertRaises(ValueError, c.copy) + self.assertRaises(ValueError, copy.copy, c) + self.assertRaises(ValueError, copy.deepcopy, c) + + @requires_Decompress_copy + def test_decompresscopy(self): + # Test copying a decompression object + data = HAMLET_SCENE + comp = zlib.compress(data) + # Test type of return value + self.assertIsInstance(comp, bytes) + + for func in lambda c: c.copy(), copy.copy, copy.deepcopy: + d0 = zlib.decompressobj() + bufs0 = [] + bufs0.append(d0.decompress(comp[:32])) + + d1 = func(d0) + bufs1 = bufs0[:] + + bufs0.append(d0.decompress(comp[32:])) + s0 = b''.join(bufs0) + + bufs1.append(d1.decompress(comp[32:])) + s1 = b''.join(bufs1) + + self.assertEqual(s0,s1) + self.assertEqual(s0,data) + + @requires_Decompress_copy + def test_baddecompresscopy(self): + # Test copying a compression object in an inconsistent state + data = zlib.compress(HAMLET_SCENE) + d = zlib.decompressobj() + d.decompress(data) + d.flush() + self.assertRaises(ValueError, d.copy) + self.assertRaises(ValueError, copy.copy, d) + self.assertRaises(ValueError, copy.deepcopy, d) + + def test_compresspickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises((TypeError, pickle.PicklingError)): + pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto) + + def test_decompresspickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises((TypeError, pickle.PicklingError)): + pickle.dumps(zlib.decompressobj(), proto) + + # Memory use of the following functions takes into account overallocation + + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) + def test_big_compress_buffer(self, size): + c = zlib.compressobj(1) + compress = lambda s: c.compress(s) + c.flush() + self.check_big_compress_buffer(size, compress) + + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) + def test_big_decompress_buffer(self, size): + d = zlib.decompressobj() + decompress = lambda s: d.decompress(s) + d.flush() + self.check_big_decompress_buffer(size, decompress) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=_4G + 100, memuse=4) + def test_64bit_compress(self, size): + data = b'x' * size + co = zlib.compressobj(0) + do = zlib.decompressobj() + try: + comp = co.compress(data) + co.flush() + uncomp = do.decompress(comp) + do.flush() + self.assertEqual(uncomp, data) + finally: + comp = uncomp = data = None + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=_4G + 100, memuse=3) + def test_large_unused_data(self, size): + data = b'abcdefghijklmnop' + unused = b'x' * size + comp = zlib.compress(data) + unused + do = zlib.decompressobj() + try: + uncomp = do.decompress(comp) + do.flush() + self.assertEqual(unused, do.unused_data) + self.assertEqual(uncomp, data) + finally: + unused = comp = do = None + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=_4G + 100, memuse=5) + def test_large_unconsumed_tail(self, size): + data = b'x' * size + do = zlib.decompressobj() + try: + comp = zlib.compress(data, 0) + uncomp = do.decompress(comp, 1) + do.flush() + self.assertEqual(uncomp, data) + self.assertEqual(do.unconsumed_tail, b'') + finally: + comp = uncomp = data = None + + def test_wbits(self): + # wbits=0 only supported since zlib v1.2.3.5 + # Register "1.2.3" as "1.2.3.0" + # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" + v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.') + if len(v) < 4: + v.append('0') + elif not v[-1].isnumeric(): + v[-1] = '0' + + v = tuple(map(int, v)) + supports_wbits_0 = v >= (1, 2, 3, 5) + + co = zlib.compressobj(level=1, wbits=15) + zlib15 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(zlib15, 15), HAMLET_SCENE) + if supports_wbits_0: + self.assertEqual(zlib.decompress(zlib15, 0), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib15, 32 + 15), HAMLET_SCENE) + with self.assertRaisesRegex(zlib.error, 'invalid window size'): + zlib.decompress(zlib15, 14) + dco = zlib.decompressobj(wbits=32 + 15) + self.assertEqual(dco.decompress(zlib15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=14) + with self.assertRaisesRegex(zlib.error, 'invalid window size'): + dco.decompress(zlib15) + + co = zlib.compressobj(level=1, wbits=9) + zlib9 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(zlib9, 9), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib9, 15), HAMLET_SCENE) + if supports_wbits_0: + self.assertEqual(zlib.decompress(zlib9, 0), HAMLET_SCENE) + self.assertEqual(zlib.decompress(zlib9, 32 + 9), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=32 + 9) + self.assertEqual(dco.decompress(zlib9), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=-15) + deflate15 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(deflate15, -15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=-15) + self.assertEqual(dco.decompress(deflate15), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=-9) + deflate9 = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(deflate9, -9), HAMLET_SCENE) + self.assertEqual(zlib.decompress(deflate9, -15), HAMLET_SCENE) + dco = zlib.decompressobj(wbits=-9) + self.assertEqual(dco.decompress(deflate9), HAMLET_SCENE) + + co = zlib.compressobj(level=1, wbits=16 + 15) + gzip = co.compress(HAMLET_SCENE) + co.flush() + self.assertEqual(zlib.decompress(gzip, 16 + 15), HAMLET_SCENE) + self.assertEqual(zlib.decompress(gzip, 32 + 15), HAMLET_SCENE) + dco = zlib.decompressobj(32 + 15) + self.assertEqual(dco.decompress(gzip), HAMLET_SCENE) + + +def genblock(seed, length, step=1024, generator=random): + """length-byte stream of random data from a seed (in step-byte blocks).""" + if seed is not None: + generator.seed(seed) + randint = generator.randint + if length < step or step < 2: + step = length + blocks = bytes() + for i in range(0, length, step): + blocks += bytes(randint(0, 255) for x in range(step)) + return blocks + + + +def choose_lines(source, number, seed=None, generator=random): + """Return a list of number lines randomly chosen from the source""" + if seed is not None: + generator.seed(seed) + sources = source.split('\n') + return [generator.choice(sources) for n in range(number)] + + + +HAMLET_SCENE = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + + +class CustomInt: + def __index__(self): + return 100 + + +if __name__ == "__main__": + unittest.main() From 915078e32f5e091557f44cb7b387e752c1d0cc25 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 18 Jul 2020 05:12:25 +0900 Subject: [PATCH 017/209] mark failing tests --- Lib/test/test_zlib.py | 66 +++++++++++++++++++++++++++++++++++++++++++ vm/src/stdlib/zlib.rs | 13 +++++---- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index f828b4c73..4097b8eaf 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -19,6 +19,8 @@ requires_Decompress_copy = unittest.skipUnless( class VersionTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_library_version(self): # Test that the major version of the actual library in use matches the # major version that we were compiled against. We can't guarantee that @@ -29,6 +31,8 @@ class VersionTestCase(unittest.TestCase): class ChecksumTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure # checksum test cases def test_crc32start(self): self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0)) @@ -39,6 +43,8 @@ class ChecksumTestCase(unittest.TestCase): self.assertEqual(zlib.crc32(b"", 1), 1) self.assertEqual(zlib.crc32(b"", 432), 432) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_adler32start(self): self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1)) self.assertTrue(zlib.adler32(b"abc", 0xffffffff)) @@ -102,6 +108,7 @@ class ExceptionTestCase(unittest.TestCase): self.assertRaises(TypeError, zlib.compress, arg) self.assertRaises(TypeError, zlib.decompress, arg) + @unittest.skip('TODO: RUSTPYTHON') def test_badcompressobj(self): # verify failure on building compress object with bad params self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0) @@ -109,6 +116,7 @@ class ExceptionTestCase(unittest.TestCase): self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1) + @unittest.skip('TODO: RUSTPYTHON') def test_baddecompressobj(self): # verify failure on building decompress object with bad params self.assertRaises(ValueError, zlib.decompressobj, -1) @@ -165,6 +173,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): x = zlib.compress(HAMLET_SCENE) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_keywords(self): x = zlib.compress(HAMLET_SCENE, level=3) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) @@ -175,6 +185,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 @@ -183,6 +195,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_incomplete_stream(self): # A useful error message is given x = zlib.compress(HAMLET_SCENE) @@ -201,6 +215,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): def test_big_decompress_buffer(self, size): self.check_big_decompress_buffer(size, zlib.decompress) + # TODO: RUSTPYTHON + @unittest.expectedFailure @bigmemtest(size=_4G, memuse=1) def test_large_bufsize(self, size): # Test decompress(bufsize) parameter greater than the internal limit @@ -208,6 +224,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): compressed = zlib.compress(data, 1) self.assertEqual(zlib.decompress(compressed, 15, size), data) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_custom_bufsize(self): data = HAMLET_SCENE * 10 compressed = zlib.compress(data, 1) @@ -225,6 +243,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure # Test compression object def test_pair(self): # straightforward compress/decompress objects @@ -245,6 +265,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertIsInstance(dco.unconsumed_tail, bytes) self.assertIsInstance(dco.unused_data, bytes) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_keywords(self): level = 2 method = zlib.DEFLATED @@ -266,6 +288,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y = do.decompress(x, max_length=len(HAMLET_SCENE)) + do.flush() self.assertEqual(HAMLET_SCENE, y) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compressoptions(self): # specify lots of options to compressobj() level = 2 @@ -281,6 +305,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y2 = dco.flush() self.assertEqual(HAMLET_SCENE, y1 + y2) + @unittest.skip('TODO: RUSTPYTHON') def test_compressincremental(self): # compress object in steps, decompress object as one-shot data = HAMLET_SCENE * 128 @@ -296,6 +321,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y2 = dco.flush() self.assertEqual(data, y1 + y2) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompinc(self, flush=False, source=None, cx=256, dcx=64): # compress object in steps, decompress object in steps source = source or HAMLET_SCENE @@ -337,9 +364,13 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertEqual(data, b''.join(bufs)) # Failure means: "decompressobj with init options failed" + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompincflush(self): self.test_decompinc(flush=True) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompimax(self, source=None, cx=256, dcx=64): # compress in steps, decompress in length-restricted steps source = source or HAMLET_SCENE @@ -367,6 +398,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): bufs.append(dco.flush()) self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressmaxlen(self, flush=False): # Check a decompression object with max_length specified data = HAMLET_SCENE * 128 @@ -399,15 +432,20 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): bufs.append(chunk) self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressmaxlenflush(self): self.test_decompressmaxlen(flush=True) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_maxlenmisc(self): # Misc tests of max_length dco = zlib.decompressobj() self.assertRaises(ValueError, dco.decompress, b"", -1) self.assertEqual(b'', dco.unconsumed_tail) + @unittest.skip('TODO: RUSTPYTHON') def test_maxlen_large(self): # Sizes up to sys.maxsize should be accepted, although zlib is # internally limited to expressing sizes with unsigned int @@ -417,6 +455,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco = zlib.decompressobj() self.assertEqual(dco.decompress(compressed, sys.maxsize), data) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_maxlen_custom(self): data = HAMLET_SCENE * 10 compressed = zlib.compress(data, 1) @@ -432,6 +472,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): ddata += dco.decompress(dco.unconsumed_tail) self.assertEqual(dco.unconsumed_tail, b"") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. @@ -498,6 +540,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # if decompressed data is different from the input data, choke. self.assertEqual(expanded, data, "17K random source doesn't match") + @unittest.skip('TODO: RUSTPYTHON') def test_empty_flush(self): # Test that calling .flush() on unused objects works. # (Bug #1083110 -- calling .flush() on decompress objects @@ -508,6 +551,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco = zlib.decompressobj() self.assertEqual(dco.flush(), b"") # Returns nothing + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dictionary(self): h = HAMLET_SCENE # Build a simulated dictionary out of the words in HAMLET. @@ -524,6 +569,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco = zlib.decompressobj() self.assertRaises(zlib.error, dco.decompress, cd) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dictionary_streaming(self): # This simulates the reuse of a compressor object for compressing # several separate data streams. @@ -537,6 +584,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertEqual(do.decompress(d1), piece[100:]) self.assertEqual(do.decompress(d2), piece[:-100]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_incomplete_stream(self): # This is 'foo', deflated x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' @@ -550,6 +599,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y += dco.flush() self.assertEqual(y, b'foo') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_eof(self): x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' dco = zlib.decompressobj() @@ -561,6 +612,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.flush() self.assertTrue(dco.eof) + @unittest.skip('TODO: RUSTPYTHON') def test_decompress_eof_incomplete_stream(self): x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' dco = zlib.decompressobj() @@ -570,6 +622,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.flush() self.assertFalse(dco.eof) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_unused_data(self): # Repeated calls to decompress() after EOF should accumulate data in # dco.unused_data, instead of just storing the arg to the last call. @@ -596,6 +650,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertEqual(dco.unconsumed_tail, b'') self.assertEqual(dco.unused_data, remainder) + # TODO: RUSTPYTHON + @unittest.expectedFailure # issue27164 def test_decompress_raw_with_dictionary(self): zdict = b'abcdefghijklmnopqrstuvwxyz' @@ -605,6 +661,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): uncomp = dco.decompress(comp) + dco.flush() self.assertEqual(zdict, uncomp) + @unittest.skip('TODO: RUSTPYTHON') def test_flush_with_freed_input(self): # Issue #16411: decompressor accesses input to last decompress() call # in flush(), even if this object has been freed in the meanwhile. @@ -617,6 +674,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): data = zlib.compress(input2) self.assertEqual(dco.flush(), input1[1:]) + @unittest.skip('TODO: RUSTPYTHON') @bigmemtest(size=_4G, memuse=1) def test_flush_large_length(self, size): # Test flush(length) parameter greater than internal limit UINT_MAX @@ -626,6 +684,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.decompress(data, 1) self.assertEqual(dco.flush(size), input[1:]) + @unittest.skip('TODO: RUSTPYTHON') def test_flush_custom_length(self): input = HAMLET_SCENE * 10 data = zlib.compress(input, 1) @@ -715,18 +774,21 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation + @unittest.skip('TODO: RUSTPYTHON') @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): c = zlib.compressobj(1) compress = lambda s: c.compress(s) + c.flush() self.check_big_compress_buffer(size, compress) + @unittest.skip('TODO: RUSTPYTHON') @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): d = zlib.decompressobj() decompress = lambda s: d.decompress(s) + d.flush() self.check_big_decompress_buffer(size, decompress) + @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=4) def test_64bit_compress(self, size): @@ -740,6 +802,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): finally: comp = uncomp = data = None + @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=3) def test_large_unused_data(self, size): @@ -754,6 +817,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): finally: unused = comp = do = None + @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=5) def test_large_unconsumed_tail(self, size): @@ -767,6 +831,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): finally: comp = uncomp = data = None + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 # Register "1.2.3" as "1.2.3.0" diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index 908e17043..b7ee1bdce 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -397,12 +397,13 @@ impl PyCompress { self.inner.lock().flush(vm) } - #[pymethod] - #[pymethod(magic)] - #[pymethod(name = "__deepcopy__")] - fn copy(&self) -> Self { - todo!("") - } + // TODO: This is optional feature of Compress + // #[pymethod] + // #[pymethod(magic)] + // #[pymethod(name = "__deepcopy__")] + // fn copy(&self) -> Self { + // todo!("") + // } } const CHUNKSIZE: usize = libc::c_uint::MAX as usize; From ec42059dcb3d4b17254b044dd21d81221966e728 Mon Sep 17 00:00:00 2001 From: Antonio Yang Date: Sat, 18 Jul 2020 21:11:02 +0800 Subject: [PATCH 018/209] build spec of format when parse --- vm/src/format.rs | 81 ++++++++++++++++++++++++++++++++++---------- vm/src/obj/objint.rs | 2 +- vm/src/obj/objstr.rs | 8 +++-- 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index c6da20f73..c0e39044d 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,3 +1,6 @@ +use crate::function::PyFuncArgs; +use crate::obj::objint::PyInt; +use crate::obj::objstr::PyString; use num_bigint::{BigInt, Sign}; use num_traits::cast::ToPrimitive; use num_traits::Signed; @@ -549,6 +552,7 @@ pub enum FormatParseError { UnmatchedBracket, MissingStartBracket, UnescapedStartBracketInLiteral, + InvalidFormatSpecifier, } impl FromStr for FormatSpec { @@ -659,37 +663,80 @@ impl FormatString { } } - fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> { - let mut chars = text.chars(); - if chars.next() != Some('{') { - return Err(FormatParseError::MissingStartBracket); - } + fn parse_spec<'a>( + text: &'a str, + args: &'a PyFuncArgs, + ) -> Result<(FormatPart, &'a str), FormatParseError> { + let mut nested = false; + let mut end_bracket_pos = None; + let mut spec_template = String::new(); + let mut left = String::new(); - // Get remaining characters after opening bracket. - let cur_text = chars.as_str(); - // Find the matching bracket and parse the text within for a spec - match cur_text.find('}') { - Some(position) => { - let (left, right) = cur_text.split_at(position); - let format_part = FormatString::parse_part_in_brackets(left)?; - Ok((format_part, &right[1..])) + // There may be one layer nesting brackets in spec + for (idx, c) in text.chars().enumerate() { + if idx == 0 { + if c != '{' { + return Err(FormatParseError::MissingStartBracket); + } + } else if c == '{' { + if nested { + return Err(FormatParseError::InvalidFormatSpecifier); + } else { + nested = true; + continue; + } + } else if c == '}' { + if nested { + nested = false; + if let Some(obj) = args.kwargs.get(&spec_template) { + if let Some(s) = (*obj).clone().payload::() { + left.push_str(s.as_str()); + } else if let Some(i) = (*obj).clone().payload::() { + left.push_str(&PyInt::repr(i)); + } else { + return Err(FormatParseError::InvalidFormatSpecifier); + } + } else { + // CPython return KeyError here + return Err(FormatParseError::InvalidFormatSpecifier); + } + continue; + } else { + end_bracket_pos = Some(idx); + break; + } + } else if nested { + spec_template.push(c); + } else { + left.push(c); } - None => Err(FormatParseError::UnmatchedBracket), + } + if let Some(pos) = end_bracket_pos { + let (_, right) = text.split_at(pos); + let format_part = FormatString::parse_part_in_brackets(&left)?; + Ok((format_part, &right[1..])) + } else { + Err(FormatParseError::UnmatchedBracket) } } } -impl FromStr for FormatString { +pub trait FromTemplate<'a>: Sized { + type Err; + fn from_str(s: &'a str, arg: &'a PyFuncArgs) -> Result; +} + +impl<'a> FromTemplate<'a> for FormatString { type Err = FormatParseError; - fn from_str(text: &str) -> Result { + fn from_str(text: &'a str, args: &'a PyFuncArgs) -> Result { let mut cur_text: &str = text; let mut parts: Vec = Vec::new(); while !cur_text.is_empty() { // Try to parse both literals and bracketed format parts util we // run out of text cur_text = FormatString::parse_literal(cur_text) - .or_else(|_| FormatString::parse_spec(cur_text)) + .or_else(|_| FormatString::parse_spec(cur_text, &args)) .map(|(part, new_text)| { parts.push(part); new_text diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 9e07ea3e6..780b7ebd2 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -498,7 +498,7 @@ impl PyInt { } #[pymethod(name = "__repr__")] - fn repr(&self) -> String { + pub(crate) fn repr(&self) -> String { self.value.to_string() } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a2dcf88ca..d9998a50d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -27,7 +27,9 @@ use crate::cformat::{ CFormatPart, CFormatPreconversor, CFormatQuantity, CFormatSpec, CFormatString, CFormatType, CNumberType, }; -use crate::format::{FormatParseError, FormatPart, FormatPreconversor, FormatSpec, FormatString}; +use crate::format::{ + FormatParseError, FormatPart, FormatPreconversor, FormatSpec, FormatString, FromTemplate, +}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyhash; use crate::pyobject::{ @@ -618,7 +620,7 @@ impl PyString { ))); } let format_string_text = borrow_value(zelf); - match FormatString::from_str(format_string_text) { + match FormatString::from_str(format_string_text, &args) { Ok(format_string) => perform_format(vm, &format_string, &args), Err(err) => match err { FormatParseError::UnmatchedBracket => { @@ -644,7 +646,7 @@ impl PyString { let zelf = &args.args[0]; let format_string_text = borrow_value(zelf); - match FormatString::from_str(format_string_text) { + match FormatString::from_str(format_string_text, &args) { Ok(format_string) => perform_format_map(vm, &format_string, &args.args[1]), Err(err) => match err { FormatParseError::UnmatchedBracket => { From bf9eb7a1b04b6817dd06f2e0c7a25216c0b37323 Mon Sep 17 00:00:00 2001 From: Antonio Yang Date: Sat, 18 Jul 2020 21:26:48 +0800 Subject: [PATCH 019/209] update testcase --- vm/src/format.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index c0e39044d..86f7b08db 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -881,13 +881,16 @@ mod tests { ], }); - assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected); + assert_eq!( + FormatString::from_str("abcd{1}:{key}", &PyFuncArgs::default()), + expected + ); } #[test] fn test_format_parse_fail() { assert_eq!( - FormatString::from_str("{s"), + FormatString::from_str("{s", &PyFuncArgs::default()), Err(FormatParseError::UnmatchedBracket) ); } @@ -902,7 +905,10 @@ mod tests { ], }); - assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected); + assert_eq!( + FormatString::from_str("{{{key}}}ddfe", &PyFuncArgs::default()), + expected + ); } #[test] From f0fcd5d9fb0aa58b892ed38302d0eeb1eb7c9bdc Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 11:58:16 +0300 Subject: [PATCH 020/209] Add test_abstract_numbers from CPython 3.8 --- Lib/test/test_abstract_numbers.py | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Lib/test/test_abstract_numbers.py diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py new file mode 100644 index 000000000..2e06f0d16 --- /dev/null +++ b/Lib/test/test_abstract_numbers.py @@ -0,0 +1,44 @@ +"""Unit tests for numbers.py.""" + +import math +import operator +import unittest +from numbers import Complex, Real, Rational, Integral + +class TestNumbers(unittest.TestCase): + def test_int(self): + self.assertTrue(issubclass(int, Integral)) + self.assertTrue(issubclass(int, Complex)) + + self.assertEqual(7, int(7).real) + self.assertEqual(0, int(7).imag) + self.assertEqual(7, int(7).conjugate()) + self.assertEqual(-7, int(-7).conjugate()) + self.assertEqual(7, int(7).numerator) + self.assertEqual(1, int(7).denominator) + + def test_float(self): + self.assertFalse(issubclass(float, Rational)) + self.assertTrue(issubclass(float, Real)) + + self.assertEqual(7.3, float(7.3).real) + self.assertEqual(0, float(7.3).imag) + self.assertEqual(7.3, float(7.3).conjugate()) + self.assertEqual(-7.3, float(-7.3).conjugate()) + + def test_complex(self): + self.assertFalse(issubclass(complex, Real)) + self.assertTrue(issubclass(complex, Complex)) + + c1, c2 = complex(3, 2), complex(4,1) + # XXX: This is not ideal, but see the comment in math_trunc(). + self.assertRaises(TypeError, math.trunc, c1) + self.assertRaises(TypeError, operator.mod, c1, c2) + self.assertRaises(TypeError, divmod, c1, c2) + self.assertRaises(TypeError, operator.floordiv, c1, c2) + self.assertRaises(TypeError, float, c1) + self.assertRaises(TypeError, int, c1) + + +if __name__ == "__main__": + unittest.main() From c9584f530e7060214e499a60c6c8f80a09fa4c41 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 12:25:48 +0300 Subject: [PATCH 021/209] Advance importlib code to CPython 3.8 --- Lib/importlib/__init__.py | 11 +- Lib/importlib/_bootstrap_external.py | 10 -- Lib/importlib/abc.py | 77 +++++++- Lib/importlib/resources.py | 259 +++++++++++++++++++++++++++ Lib/importlib/util.py | 18 +- 5 files changed, 349 insertions(+), 26 deletions(-) create mode 100644 Lib/importlib/resources.py diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e0..0c73c505f 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -48,8 +48,8 @@ else: sys.modules['importlib._bootstrap_external'] = _bootstrap_external # To simplify imports in test code -_w_long = _bootstrap_external._w_long -_r_long = _bootstrap_external._r_long +_pack_uint32 = _bootstrap_external._pack_uint32 +_unpack_uint32 = _bootstrap_external._unpack_uint32 # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) @@ -79,7 +79,8 @@ def find_loader(name, path=None): This function is deprecated in favor of importlib.util.find_spec(). """ - warnings.warn('Use importlib.util.find_spec() instead.', + warnings.warn('Deprecated since Python 3.4. ' + 'Use importlib.util.find_spec() instead.', DeprecationWarning, stacklevel=2) try: loader = sys.modules[name].__loader__ @@ -136,7 +137,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: @@ -163,6 +164,8 @@ def reload(module): pkgpath = None target = module spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target) + if spec is None: + raise ModuleNotFoundError(f"spec not found for the module {name!r}", name=name) _bootstrap._exec(spec, module) # The module may have replaced itself in sys.modules! return sys.modules[name] diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index d60cb7691..f8ff5f4f2 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -26,16 +26,6 @@ _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) -def _w_long(x): - """Convert a 32-bit integer to little-endian.""" - return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') - - -def _r_long(int_bytes): - """Convert 4 bytes in little-endian to an integer.""" - return int.from_bytes(int_bytes, 'little') - - def _make_relax_case(): if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index daff681e6..4b2d3de6d 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,6 +13,7 @@ try: except ImportError as exc: _frozen_importlib_external = _bootstrap_external import abc +import warnings def _register(abstract_cls, *classes): @@ -34,6 +35,8 @@ class Finder(metaclass=abc.ABCMeta): reimplementations of the import system. Otherwise, finder implementations should derive from the more specific MetaPathFinder or PathEntryFinder ABCs. + + Deprecated since Python 3.3 """ @abc.abstractmethod @@ -57,11 +60,16 @@ class MetaPathFinder(Finder): If no module is found, return None. The fullname is a str and the path is a list of strings or None. - This method is deprecated in favor of finder.find_spec(). If find_spec() - exists then backwards-compatible functionality is provided for this - method. + This method is deprecated since Python 3.4 in favor of + finder.find_spec(). If find_spec() exists then backwards-compatible + functionality is provided for this method. """ + warnings.warn("MetaPathFinder.find_module() is deprecated since Python " + "3.4 in favor of MetaPathFinder.find_spec() " + "(available since 3.4)", + DeprecationWarning, + stacklevel=2) if not hasattr(self, 'find_spec'): return None found = self.find_spec(fullname, path) @@ -94,10 +102,15 @@ class PathEntryFinder(Finder): The portion will be discarded if another path entry finder locates the module as a normal module or package. - This method is deprecated in favor of finder.find_spec(). If find_spec() - is provided than backwards-compatible functionality is provided. - + This method is deprecated since Python 3.4 in favor of + finder.find_spec(). If find_spec() is provided than backwards-compatible + functionality is provided. """ + warnings.warn("PathEntryFinder.find_loader() is deprecated since Python " + "3.4 in favor of PathEntryFinder.find_spec() " + "(available since 3.4)", + DeprecationWarning, + stacklevel=2) if not hasattr(self, 'find_spec'): return None, [] found = self.find_spec(fullname) @@ -180,7 +193,7 @@ class ResourceLoader(Loader): def get_data(self, path): """Abstract method which when implemented should return the bytes for the specified path. The path must be a str.""" - raise IOError + raise OSError class InspectLoader(Loader): @@ -302,7 +315,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" if self.path_stats.__func__ is SourceLoader.path_stats: - raise IOError + raise OSError return int(self.path_stats(path)['mtime']) def path_stats(self, path): @@ -313,7 +326,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo - 'size' (optional) is the size in bytes of the source code. """ if self.path_mtime.__func__ is SourceLoader.path_mtime: - raise IOError + raise OSError return {'mtime': self.path_mtime(path)} def set_data(self, path, data): @@ -327,3 +340,49 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo """ _register(SourceLoader, machinery.SourceFileLoader) + + +class ResourceReader(metaclass=abc.ABCMeta): + + """Abstract base class to provide resource-reading support. + + Loaders that support resource reading are expected to implement + the ``get_resource_reader(fullname)`` method and have it either return None + or an object compatible with this ABC. + """ + + @abc.abstractmethod + def open_resource(self, resource): + """Return an opened, file-like object for binary reading. + + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + + If the resource cannot be found, FileNotFoundError is raised. + """ + raise FileNotFoundError + + @abc.abstractmethod + def resource_path(self, resource): + """Return the file system path to the specified resource. + + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + + If the resource does not exist on the file system, raise + FileNotFoundError. + """ + raise FileNotFoundError + + @abc.abstractmethod + def is_resource(self, name): + """Return True if the named 'name' is consider a resource.""" + raise FileNotFoundError + + @abc.abstractmethod + def contents(self): + """Return an iterable of strings over the contents of the package.""" + return [] + + +_register(ResourceReader, machinery.SourceFileLoader) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py new file mode 100644 index 000000000..fc3a1c9ca --- /dev/null +++ b/Lib/importlib/resources.py @@ -0,0 +1,259 @@ +import os +import tempfile + +from . import abc as resources_abc +from contextlib import contextmanager, suppress +from importlib import import_module +from importlib.abc import ResourceLoader +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401 +from typing import cast +from typing.io import BinaryIO, TextIO +from zipimport import ZipImportError + + +__all__ = [ + 'Package', + 'Resource', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', + ] + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] + + +def _get_package(package) -> ModuleType: + """Take a package name or module object and return the module. + + If a name, the module is imported. If the passed or imported module + object is not a package, raise an exception. + """ + if hasattr(package, '__spec__'): + if package.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format( + package.__spec__.name)) + else: + return package + else: + module = import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format(package)) + else: + return module + + +def _normalize_path(path) -> str: + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + parent, file_name = os.path.split(path) + if parent: + raise ValueError('{!r} must be only a file name'.format(path)) + else: + return file_name + + +def _get_resource_reader( + package: ModuleType) -> Optional[resources_abc.ResourceReader]: + # Return the package's loader if it's a ResourceReader. We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + if hasattr(spec.loader, 'get_resource_reader'): + return cast(resources_abc.ResourceReader, + spec.loader.get_resource_reader(spec.name)) + return None + + +def _check_location(package): + if package.__spec__.origin is None or not package.__spec__.has_location: + raise FileNotFoundError(f'Package has no location {package!r}') + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + _check_location(package) + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return BytesIO(data) + + +def open_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return TextIOWrapper(reader.open_resource(resource), encoding, errors) + _check_location(package) + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='r', encoding=encoding, errors=errors) + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return TextIOWrapper(BytesIO(data), encoding, errors) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + resource = _normalize_path(resource) + package = _get_package(package) + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +@contextmanager +def path(package: Package, resource: Resource) -> Iterator[Path]: + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + try: + yield Path(reader.resource_path(resource)) + return + except FileNotFoundError: + pass + else: + _check_location(package) + # Fall-through for both the lack of resource_path() *and* if + # resource_path() raises FileNotFoundError. + package_directory = Path(package.__spec__.origin).parent + file_path = package_directory / resource + if file_path.exists(): + yield file_path + else: + with open_binary(package, resource) as fp: + data = fp.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() + try: + os.write(fd, data) + os.close(fd) + yield Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass + + +def is_resource(package: Package, name: str) -> bool: + """True if 'name' is a resource inside 'package'. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + reader = _get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + try: + package_contents = set(contents(package)) + except (NotADirectoryError, FileNotFoundError): + return False + if name not in package_contents: + return False + # Just because the given file_name lives as an entry in the package's + # contents doesn't necessarily mean it's a resource. Directories are not + # resources, so let's try to find out if it's a directory or not. + path = Path(package.__spec__.origin).parent / name + return path.is_file() + + +def contents(package: Package) -> Iterable[str]: + """Return an iterable of entries in 'package'. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.contents() + # Is the package a namespace package? By definition, namespace packages + # cannot have resources. We could use _check_location() and catch the + # exception, but that's extra work, so just inline the check. + elif package.__spec__.origin is None or not package.__spec__.has_location: + return () + else: + package_directory = Path(package.__spec__.origin).parent + return os.listdir(package_directory) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 6bdf0d445..201e0f4cb 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,18 +5,25 @@ from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER +from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache from ._bootstrap_external import spec_from_file_location from contextlib import contextmanager +import _imp import functools import sys import types import warnings +def source_hash(source_bytes): + "Return the hash of *source_bytes* as used in hash-based pyc files." + return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + + def resolve_name(name, package): """Resolve a relative module name to an absolute one.""" if not name.startswith('.'): @@ -84,11 +91,16 @@ def find_spec(name, package=None): if fullname not in sys.modules: parent_name = fullname.rpartition('.')[0] if parent_name: - # Use builtins.__import__() in case someone replaced it. parent = __import__(parent_name, fromlist=['__path__']) - return _find_spec(fullname, parent.__path__) + try: + parent_path = parent.__path__ + except AttributeError as e: + raise ModuleNotFoundError( + f"__path__ attribute not found on {parent_name!r} " + f"while trying to find {fullname!r}", name=fullname) from e else: - return _find_spec(fullname, None) + parent_path = None + return _find_spec(fullname, parent_path) else: module = sys.modules[fullname] if module is None: From d14f796b1a395d0191bd49d30fe33fb990b993b8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 13:23:56 +0300 Subject: [PATCH 022/209] Raise error if sys.module value is None --- vm/src/frame.rs | 2 +- vm/src/import.rs | 14 ++++++++++++-- vm/src/stdlib/imp.rs | 10 ++++++++-- vm/src/vm.rs | 22 ++++++++++++++++++---- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 8426b101c..35fef5315 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -780,7 +780,7 @@ impl ExecutingFrame<'_> { // Load attribute, and transform any error into import error. let obj = vm .get_attribute(module, name) - .map_err(|_| vm.new_import_error(format!("cannot import name '{}'", name)))?; + .map_err(|_| vm.new_import_error(format!("cannot import name '{}'", name), name))?; self.push_value(obj); Ok(None) } diff --git a/vm/src/import.rs b/vm/src/import.rs index d865c2060..c36c15827 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -60,7 +60,12 @@ pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { vm.state .frozen .get(module_name) - .ok_or_else(|| vm.new_import_error(format!("Cannot import frozen module {}", module_name))) + .ok_or_else(|| { + vm.new_import_error( + format!("Cannot import frozen module {}", module_name), + module_name, + ) + }) .and_then(|frozen| import_codeobj(vm, module_name, frozen.code.clone(), false)) } @@ -68,7 +73,12 @@ pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { vm.state .stdlib_inits .get(module_name) - .ok_or_else(|| vm.new_import_error(format!("Cannot import bultin module {}", module_name))) + .ok_or_else(|| { + vm.new_import_error( + format!("Cannot import bultin module {}", module_name), + module_name, + ) + }) .and_then(|make_module_func| { let module = make_module_func(vm); let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 92ec0bc2b..c9625004e 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -62,7 +62,10 @@ fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult PyResult PyBaseExceptionRef { + pub fn new_import_error(&self, msg: String, name: &str) -> PyBaseExceptionRef { let import_error = self.ctx.exceptions.import_error.clone(); - self.new_exception_msg(import_error, msg) + let exc = self.new_exception_msg(import_error, msg); + self.set_attr(exc.as_object(), "name", self.new_str(name.to_owned())) + .unwrap(); + exc } pub fn new_runtime_error(&self, msg: String) -> PyBaseExceptionRef { @@ -699,11 +702,22 @@ impl VirtualMachine { }; match cached_module { - Some(module) => Ok(module), + Some(cached_module) => { + if self.is_none(&cached_module) { + Err(self.new_import_error( + format!("import of {} halted; None in sys.modules", module), + module, + )) + } else { + Ok(cached_module) + } + } None => { let import_func = self .get_attribute(self.builtins.clone(), "__import__") - .map_err(|_| self.new_import_error("__import__ not found".to_owned()))?; + .map_err(|_| { + self.new_import_error("__import__ not found".to_owned(), module) + })?; let (locals, globals) = if let Some(frame) = self.current_frame() { ( From d14c5a62a9cbb295dab82530109681975fe72950 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 13:31:12 +0300 Subject: [PATCH 023/209] Advance py_compile code to CPython 3.8 --- Lib/py_compile.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 11c5b505c..8e9dd57a5 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -3,6 +3,7 @@ This module has intimate knowledge of the format of .pyc files. """ +import enum import importlib._bootstrap_external import importlib.machinery import importlib.util @@ -11,7 +12,7 @@ import os.path import sys import traceback -__all__ = ["compile", "main", "PyCompileError"] +__all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"] class PyCompileError(Exception): @@ -62,7 +63,21 @@ class PyCompileError(Exception): return self.msg -def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): +class PycInvalidationMode(enum.Enum): + TIMESTAMP = 1 + CHECKED_HASH = 2 + UNCHECKED_HASH = 3 + + +def _get_default_invalidation_mode(): + if os.environ.get('SOURCE_DATE_EPOCH'): + return PycInvalidationMode.CHECKED_HASH + else: + return PycInvalidationMode.TIMESTAMP + + +def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, + invalidation_mode=None): """Byte-compile one Python source file to Python bytecode. :param file: The source file name. @@ -79,6 +94,7 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): :param optimize: The optimization level for the compiler. Valid values are -1, 0, 1 and 2. A value of -1 means to use the optimization level of the current interpreter, as given by -O command line options. + :param invalidation_mode: :return: Path to the resulting byte compiled file. @@ -103,6 +119,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): the resulting file would be regular and thus not the same type of file as it was previously. """ + if invalidation_mode is None: + invalidation_mode = _get_default_invalidation_mode() if cfile is None: if optimize >= 0: optimization = optimize if optimize >= 1 else '' @@ -136,9 +154,17 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): os.makedirs(dirname) except FileExistsError: pass - source_stats = loader.path_stats(file) - bytecode = importlib._bootstrap_external._code_to_bytecode( + if invalidation_mode == PycInvalidationMode.TIMESTAMP: + source_stats = loader.path_stats(file) + bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( code, source_stats['mtime'], source_stats['size']) + else: + source_hash = importlib.util.source_hash(source_bytes) + bytecode = importlib._bootstrap_external._code_to_hash_pyc( + code, + source_hash, + (invalidation_mode == PycInvalidationMode.CHECKED_HASH), + ) mode = importlib._bootstrap_external._calc_mode(file) importlib._bootstrap_external._write_atomic(cfile, bytecode, mode) return cfile From ca66b027e66b434f993f4ad0e12282a05401a32c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 13:40:52 +0300 Subject: [PATCH 024/209] Add test_imp from CPython 3.8 --- Lib/test/test_imp.py | 458 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 Lib/test/test_imp.py diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py new file mode 100644 index 000000000..fe394dc50 --- /dev/null +++ b/Lib/test/test_imp.py @@ -0,0 +1,458 @@ +import importlib +import importlib.util +import os +import os.path +import py_compile +import sys +from test import support +from test.support import script_helper +import unittest +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import imp +import _imp + + +def requires_load_dynamic(meth): + """Decorator to skip a test if not running under CPython or lacking + imp.load_dynamic().""" + meth = support.cpython_only(meth) + return unittest.skipIf(not hasattr(imp, 'load_dynamic'), + 'imp.load_dynamic() required')(meth) + + +class LockTests(unittest.TestCase): + + """Very basic test of import lock functions.""" + + def verify_lock_state(self, expected): + self.assertEqual(imp.lock_held(), expected, + "expected imp.lock_held() to be %r" % expected) + def testLock(self): + LOOPS = 50 + + # The import lock may already be held, e.g. if the test suite is run + # via "import test.autotest". + lock_held_at_start = imp.lock_held() + self.verify_lock_state(lock_held_at_start) + + for i in range(LOOPS): + imp.acquire_lock() + self.verify_lock_state(True) + + for i in range(LOOPS): + imp.release_lock() + + # The original state should be restored now. + self.verify_lock_state(lock_held_at_start) + + if not lock_held_at_start: + try: + imp.release_lock() + except RuntimeError: + pass + else: + self.fail("release_lock() without lock should raise " + "RuntimeError") + +class ImportTests(unittest.TestCase): + def setUp(self): + mod = importlib.import_module('test.encoded_modules') + self.test_strings = mod.test_strings + self.test_path = mod.__path__ + + def test_import_encoded_module(self): + for modname, encoding, teststr in self.test_strings: + mod = importlib.import_module('test.encoded_modules.' + 'module_' + modname) + self.assertEqual(teststr, mod.test) + + def test_find_module_encoding(self): + for mod, encoding, _ in self.test_strings: + with imp.find_module('module_' + mod, self.test_path)[0] as fd: + self.assertEqual(fd.encoding, encoding) + + path = [os.path.dirname(__file__)] + with self.assertRaises(SyntaxError): + imp.find_module('badsyntax_pep3120', path) + + def test_issue1267(self): + for mod, encoding, _ in self.test_strings: + fp, filename, info = imp.find_module('module_' + mod, + self.test_path) + with fp: + self.assertNotEqual(fp, None) + self.assertEqual(fp.encoding, encoding) + self.assertEqual(fp.tell(), 0) + self.assertEqual(fp.readline(), '# test %s encoding\n' + % encoding) + + fp, filename, info = imp.find_module("tokenize") + with fp: + self.assertNotEqual(fp, None) + self.assertEqual(fp.encoding, "utf-8") + self.assertEqual(fp.tell(), 0) + self.assertEqual(fp.readline(), + '"""Tokenization help for Python programs.\n') + + def test_issue3594(self): + temp_mod_name = 'test_imp_helper' + sys.path.insert(0, '.') + try: + with open(temp_mod_name + '.py', 'w') as file: + file.write("# coding: cp1252\nu = 'test.test_imp'\n") + file, filename, info = imp.find_module(temp_mod_name) + file.close() + self.assertEqual(file.encoding, 'cp1252') + finally: + del sys.path[0] + support.unlink(temp_mod_name + '.py') + support.unlink(temp_mod_name + '.pyc') + + def test_issue5604(self): + # Test cannot cover imp.load_compiled function. + # Martin von Loewis note what shared library cannot have non-ascii + # character because init_xxx function cannot be compiled + # and issue never happens for dynamic modules. + # But sources modified to follow generic way for processing paths. + + # the return encoding could be uppercase or None + fs_encoding = sys.getfilesystemencoding() + + # covers utf-8 and Windows ANSI code pages + # one non-space symbol from every page + # (http://en.wikipedia.org/wiki/Code_page) + known_locales = { + 'utf-8' : b'\xc3\xa4', + 'cp1250' : b'\x8C', + 'cp1251' : b'\xc0', + 'cp1252' : b'\xc0', + 'cp1253' : b'\xc1', + 'cp1254' : b'\xc0', + 'cp1255' : b'\xe0', + 'cp1256' : b'\xe0', + 'cp1257' : b'\xc0', + 'cp1258' : b'\xc0', + } + + if sys.platform == 'darwin': + self.assertEqual(fs_encoding, 'utf-8') + # Mac OS X uses the Normal Form D decomposition + # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html + special_char = b'a\xcc\x88' + else: + special_char = known_locales.get(fs_encoding) + + if not special_char: + self.skipTest("can't run this test with %s as filesystem encoding" + % fs_encoding) + decoded_char = special_char.decode(fs_encoding) + temp_mod_name = 'test_imp_helper_' + decoded_char + test_package_name = 'test_imp_helper_package_' + decoded_char + init_file_name = os.path.join(test_package_name, '__init__.py') + try: + # if the curdir is not in sys.path the test fails when run with + # ./python ./Lib/test/regrtest.py test_imp + sys.path.insert(0, os.curdir) + with open(temp_mod_name + '.py', 'w') as file: + file.write('a = 1\n') + file, filename, info = imp.find_module(temp_mod_name) + with file: + self.assertIsNotNone(file) + self.assertTrue(filename[:-3].endswith(temp_mod_name)) + self.assertEqual(info[0], '.py') + self.assertEqual(info[1], 'r') + self.assertEqual(info[2], imp.PY_SOURCE) + + mod = imp.load_module(temp_mod_name, file, filename, info) + self.assertEqual(mod.a, 1) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') + self.assertEqual(mod.a, 1) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + if not sys.dont_write_bytecode: + mod = imp.load_compiled( + temp_mod_name, + imp.cache_from_source(temp_mod_name + '.py')) + self.assertEqual(mod.a, 1) + + if not os.path.exists(test_package_name): + os.mkdir(test_package_name) + with open(init_file_name, 'w') as file: + file.write('b = 2\n') + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + package = imp.load_package(test_package_name, test_package_name) + self.assertEqual(package.b, 2) + finally: + del sys.path[0] + for ext in ('.py', '.pyc'): + support.unlink(temp_mod_name + ext) + support.unlink(init_file_name + ext) + support.rmtree(test_package_name) + support.rmtree('__pycache__') + + def test_issue9319(self): + path = os.path.dirname(__file__) + self.assertRaises(SyntaxError, + imp.find_module, "badsyntax_pep3120", [path]) + + def test_load_from_source(self): + # Verify that the imp module can correctly load and find .py files + # XXX (ncoghlan): It would be nice to use support.CleanImport + # here, but that breaks because the os module registers some + # handlers in copy_reg on import. Since CleanImport doesn't + # revert that registration, the module is left in a broken + # state after reversion. Reinitialising the module contents + # and just reverting os.environ to its previous state is an OK + # workaround + orig_path = os.path + orig_getenv = os.getenv + with support.EnvironmentVarGuard(): + x = imp.find_module("os") + self.addCleanup(x[0].close) + new_os = imp.load_module("os", *x) + self.assertIs(os, new_os) + self.assertIs(orig_path, new_os.path) + self.assertIsNot(orig_getenv, new_os.getenv) + + @requires_load_dynamic + def test_issue15828_load_extensions(self): + # Issue 15828 picked up that the adapter between the old imp API + # and importlib couldn't handle C extensions + example = "_heapq" + x = imp.find_module(example) + file_ = x[0] + if file_ is not None: + self.addCleanup(file_.close) + mod = imp.load_module(example, *x) + self.assertEqual(mod.__name__, example) + + @requires_load_dynamic + def test_issue16421_multiple_modules_in_one_dll(self): + # Issue 16421: loading several modules from the same compiled file fails + m = '_testimportmultiple' + fileobj, pathname, description = imp.find_module(m) + fileobj.close() + mod0 = imp.load_dynamic(m, pathname) + mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname) + mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname) + self.assertEqual(mod0.__name__, m) + self.assertEqual(mod1.__name__, '_testimportmultiple_foo') + self.assertEqual(mod2.__name__, '_testimportmultiple_bar') + with self.assertRaises(ImportError): + imp.load_dynamic('nonexistent', pathname) + + @requires_load_dynamic + def test_load_dynamic_ImportError_path(self): + # Issue #1559549 added `name` and `path` attributes to ImportError + # in order to provide better detail. Issue #10854 implemented those + # attributes on import failures of extensions on Windows. + path = 'bogus file path' + name = 'extension' + with self.assertRaises(ImportError) as err: + imp.load_dynamic(name, path) + self.assertIn(path, err.exception.path) + self.assertEqual(name, err.exception.name) + + @requires_load_dynamic + def test_load_module_extension_file_is_None(self): + # When loading an extension module and the file is None, open one + # on the behalf of imp.load_dynamic(). + # Issue #15902 + name = '_testimportmultiple' + found = imp.find_module(name) + if found[0] is not None: + found[0].close() + if found[2][2] != imp.C_EXTENSION: + self.skipTest("found module doesn't appear to be a C extension") + imp.load_module(name, None, *found[1:]) + + @requires_load_dynamic + def test_issue24748_load_module_skips_sys_modules_check(self): + name = 'test.imp_dummy' + try: + del sys.modules[name] + except KeyError: + pass + try: + module = importlib.import_module(name) + spec = importlib.util.find_spec('_testmultiphase') + module = imp.load_dynamic(name, spec.origin) + self.assertEqual(module.__name__, name) + self.assertEqual(module.__spec__.name, name) + self.assertEqual(module.__spec__.origin, spec.origin) + self.assertRaises(AttributeError, getattr, module, 'dummy_name') + self.assertEqual(module.int_const, 1969) + self.assertIs(sys.modules[name], module) + finally: + try: + del sys.modules[name] + except KeyError: + pass + + @unittest.skipIf(sys.dont_write_bytecode, + "test meaningful only when writing bytecode") + def test_bug7732(self): + with support.temp_cwd(): + source = support.TESTFN + '.py' + os.mkdir(source) + self.assertRaisesRegex(ImportError, '^No module', + imp.find_module, support.TESTFN, ["."]) + + def test_multiple_calls_to_get_data(self): + # Issue #18755: make sure multiple calls to get_data() can succeed. + loader = imp._LoadSourceCompatibility('imp', imp.__file__, + open(imp.__file__)) + loader.get_data(imp.__file__) # File should be closed + loader.get_data(imp.__file__) # Will need to create a newly opened file + + def test_load_source(self): + # Create a temporary module since load_source(name) modifies + # sys.modules[name] attributes like __loader___ + modname = f"tmp{__name__}" + mod = type(sys.modules[__name__])(modname) + with support.swap_item(sys.modules, modname, mod): + with self.assertRaisesRegex(ValueError, 'embedded null'): + imp.load_source(modname, __file__ + "\0") + + @support.cpython_only + def test_issue31315(self): + # There shouldn't be an assertion failure in imp.create_dynamic(), + # when spec.name is not a string. + create_dynamic = support.get_attribute(imp, 'create_dynamic') + class BadSpec: + name = None + origin = 'foo' + with self.assertRaises(TypeError): + create_dynamic(BadSpec()) + + def test_issue_35321(self): + # Both _frozen_importlib and _frozen_importlib_external + # should have a spec origin of "frozen" and + # no need to clean up imports in this case. + + import _frozen_importlib_external + self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen") + + import _frozen_importlib + self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") + + def test_source_hash(self): + self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab') + self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9') + + def test_pyc_invalidation_mode_from_cmdline(self): + cases = [ + ([], "default"), + (["--check-hash-based-pycs", "default"], "default"), + (["--check-hash-based-pycs", "always"], "always"), + (["--check-hash-based-pycs", "never"], "never"), + ] + for interp_args, expected in cases: + args = interp_args + [ + "-c", + "import _imp; print(_imp.check_hash_based_pycs)", + ] + res = script_helper.assert_python_ok(*args) + self.assertEqual(res.out.strip().decode('utf-8'), expected) + + def test_find_and_load_checked_pyc(self): + # issue 34056 + with support.temp_cwd(): + with open('mymod.py', 'wb') as fp: + fp.write(b'x = 42\n') + py_compile.compile( + 'mymod.py', + doraise=True, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + file, path, description = imp.find_module('mymod', path=['.']) + mod = imp.load_module('mymod', file, path, description) + self.assertEqual(mod.x, 42) + + +class ReloadTests(unittest.TestCase): + + """Very basic tests to make sure that imp.reload() operates just like + reload().""" + + def test_source(self): + # XXX (ncoghlan): It would be nice to use test.support.CleanImport + # here, but that breaks because the os module registers some + # handlers in copy_reg on import. Since CleanImport doesn't + # revert that registration, the module is left in a broken + # state after reversion. Reinitialising the module contents + # and just reverting os.environ to its previous state is an OK + # workaround + with support.EnvironmentVarGuard(): + import os + imp.reload(os) + + def test_extension(self): + with support.CleanImport('time'): + import time + imp.reload(time) + + def test_builtin(self): + with support.CleanImport('marshal'): + import marshal + imp.reload(marshal) + + def test_with_deleted_parent(self): + # see #18681 + from html import parser + html = sys.modules.pop('html') + def cleanup(): + sys.modules['html'] = html + self.addCleanup(cleanup) + with self.assertRaisesRegex(ImportError, 'html'): + imp.reload(parser) + + +class PEP3147Tests(unittest.TestCase): + """Tests of PEP 3147.""" + + tag = imp.get_tag() + + @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag not be None') + def test_cache_from_source(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyc file (i.e. under __pycache__). + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + self.assertEqual(imp.cache_from_source(path, True), expect) + + @unittest.skipUnless(sys.implementation.cache_tag is not None, + 'requires sys.implementation.cache_tag to not be ' + 'None') + def test_source_from_cache(self): + # Given the path to a PEP 3147 defined .pyc file, return the path to + # its source. This tests the good path. + path = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + self.assertEqual(imp.source_from_cache(path), expect) + + +class NullImporterTests(unittest.TestCase): + @unittest.skipIf(support.TESTFN_UNENCODABLE is None, + "Need an undecodeable filename") + def test_unencodeable(self): + name = support.TESTFN_UNENCODABLE + os.mkdir(name) + try: + self.assertRaises(ImportError, imp.NullImporter, name) + finally: + os.rmdir(name) + + +if __name__ == "__main__": + unittest.main() From aebac26cf269bb6ec72affcb4232397263845a3f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 13:45:51 +0300 Subject: [PATCH 025/209] Add imp.py from CPython 3.8 --- Lib/imp.py | 345 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 Lib/imp.py diff --git a/Lib/imp.py b/Lib/imp.py new file mode 100644 index 000000000..31f8c7663 --- /dev/null +++ b/Lib/imp.py @@ -0,0 +1,345 @@ +"""This module provides the components needed to build your own __import__ +function. Undocumented functions are obsolete. + +In most cases it is preferred you consider using the importlib module's +functionality over this module. + +""" +# (Probably) need to stay in _imp +from _imp import (lock_held, acquire_lock, release_lock, + get_frozen_object, is_frozen_package, + init_frozen, is_builtin, is_frozen, + _fix_co_filename) +try: + from _imp import create_dynamic +except ImportError: + # Platform doesn't support dynamic loading. + create_dynamic = None + +from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name +from importlib._bootstrap_external import SourcelessFileLoader + +from importlib import machinery +from importlib import util +import importlib +import os +import sys +import tokenize +import types +import warnings + +warnings.warn("the imp module is deprecated in favour of importlib; " + "see the module's documentation for alternative uses", + DeprecationWarning, stacklevel=2) + +# DEPRECATED +SEARCH_ERROR = 0 +PY_SOURCE = 1 +PY_COMPILED = 2 +C_EXTENSION = 3 +PY_RESOURCE = 4 +PKG_DIRECTORY = 5 +C_BUILTIN = 6 +PY_FROZEN = 7 +PY_CODERESOURCE = 8 +IMP_HOOK = 9 + + +def new_module(name): + """**DEPRECATED** + + Create a new module. + + The module is not entered into sys.modules. + + """ + return types.ModuleType(name) + + +def get_magic(): + """**DEPRECATED** + + Return the magic number for .pyc files. + """ + return util.MAGIC_NUMBER + + +def get_tag(): + """Return the magic tag for .pyc files.""" + return sys.implementation.cache_tag + + +def cache_from_source(path, debug_override=None): + """**DEPRECATED** + + Given the path to a .py file, return the path to its .pyc file. + + The .py file does not need to exist; this simply returns the path to the + .pyc file calculated as if the .py file were imported. + + If debug_override is not None, then it must be a boolean and is used in + place of sys.flags.optimize. + + If sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + return util.cache_from_source(path, debug_override) + + +def source_from_cache(path): + """**DEPRECATED** + + Given the path to a .pyc. file, return the path to its .py file. + + The .pyc file does not need to exist; this simply returns the path to + the .py file calculated to correspond to the .pyc file. If path does + not conform to PEP 3147 format, ValueError will be raised. If + sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + return util.source_from_cache(path) + + +def get_suffixes(): + """**DEPRECATED**""" + extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] + source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] + + return extensions + source + bytecode + + +class NullImporter: + + """**DEPRECATED** + + Null import object. + + """ + + def __init__(self, path): + if path == '': + raise ImportError('empty pathname', path='') + elif os.path.isdir(path): + raise ImportError('existing directory', path=path) + + def find_module(self, fullname): + """Always returns None.""" + return None + + +class _HackedGetData: + + """Compatibility support for 'file' arguments of various load_*() + functions.""" + + def __init__(self, fullname, path, file=None): + super().__init__(fullname, path) + self.file = file + + def get_data(self, path): + """Gross hack to contort loader to deal w/ load_*()'s bad API.""" + if self.file and path == self.path: + # The contract of get_data() requires us to return bytes. Reopen the + # file in binary mode if needed. + if not self.file.closed: + file = self.file + if 'b' not in file.mode: + file.close() + if self.file.closed: + self.file = file = open(self.path, 'rb') + + with file: + return file.read() + else: + return super().get_data(path) + + +class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): + + """Compatibility support for implementing load_source().""" + + +def load_source(name, pathname, file=None): + loader = _LoadSourceCompatibility(name, pathname, file) + spec = util.spec_from_file_location(name, pathname, loader=loader) + if name in sys.modules: + module = _exec(spec, sys.modules[name]) + else: + module = _load(spec) + # To allow reloading to potentially work, use a non-hacked loader which + # won't rely on a now-closed file object. + module.__loader__ = machinery.SourceFileLoader(name, pathname) + module.__spec__.loader = module.__loader__ + return module + + +class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): + + """Compatibility support for implementing load_compiled().""" + + +def load_compiled(name, pathname, file=None): + """**DEPRECATED**""" + loader = _LoadCompiledCompatibility(name, pathname, file) + spec = util.spec_from_file_location(name, pathname, loader=loader) + if name in sys.modules: + module = _exec(spec, sys.modules[name]) + else: + module = _load(spec) + # To allow reloading to potentially work, use a non-hacked loader which + # won't rely on a now-closed file object. + module.__loader__ = SourcelessFileLoader(name, pathname) + module.__spec__.loader = module.__loader__ + return module + + +def load_package(name, path): + """**DEPRECATED**""" + if os.path.isdir(path): + extensions = (machinery.SOURCE_SUFFIXES[:] + + machinery.BYTECODE_SUFFIXES[:]) + for extension in extensions: + init_path = os.path.join(path, '__init__' + extension) + if os.path.exists(init_path): + path = init_path + break + else: + raise ValueError('{!r} is not a package'.format(path)) + spec = util.spec_from_file_location(name, path, + submodule_search_locations=[]) + if name in sys.modules: + return _exec(spec, sys.modules[name]) + else: + return _load(spec) + + +def load_module(name, file, filename, details): + """**DEPRECATED** + + Load a module, given information returned by find_module(). + + The module name must include the full package name, if any. + + """ + suffix, mode, type_ = details + if mode and (not mode.startswith(('r', 'U')) or '+' in mode): + raise ValueError('invalid file open mode {!r}'.format(mode)) + elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: + msg = 'file object required for import (type code {})'.format(type_) + raise ValueError(msg) + elif type_ == PY_SOURCE: + return load_source(name, filename, file) + elif type_ == PY_COMPILED: + return load_compiled(name, filename, file) + elif type_ == C_EXTENSION and load_dynamic is not None: + if file is None: + with open(filename, 'rb') as opened_file: + return load_dynamic(name, filename, opened_file) + else: + return load_dynamic(name, filename, file) + elif type_ == PKG_DIRECTORY: + return load_package(name, filename) + elif type_ == C_BUILTIN: + return init_builtin(name) + elif type_ == PY_FROZEN: + return init_frozen(name) + else: + msg = "Don't know how to import {} (type code {})".format(name, type_) + raise ImportError(msg, name=name) + + +def find_module(name, path=None): + """**DEPRECATED** + + Search for a module. + + If path is omitted or None, search for a built-in, frozen or special + module and continue search in sys.path. The module name cannot + contain '.'; to search for a submodule of a package, pass the + submodule name and the package's __path__. + + """ + if not isinstance(name, str): + raise TypeError("'name' must be a str, not {}".format(type(name))) + elif not isinstance(path, (type(None), list)): + # Backwards-compatibility + raise RuntimeError("'path' must be None or a list, " + "not {}".format(type(path))) + + if path is None: + if is_builtin(name): + return None, None, ('', '', C_BUILTIN) + elif is_frozen(name): + return None, None, ('', '', PY_FROZEN) + else: + path = sys.path + + for entry in path: + package_directory = os.path.join(entry, name) + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: + package_file_name = '__init__' + suffix + file_path = os.path.join(package_directory, package_file_name) + if os.path.isfile(file_path): + return None, package_directory, ('', '', PKG_DIRECTORY) + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. + else: + raise ImportError(_ERR_MSG.format(name), name=name) + + encoding = None + if 'b' not in mode: + with open(file_path, 'rb') as file: + encoding = tokenize.detect_encoding(file.readline)[0] + file = open(file_path, mode, encoding=encoding) + return file, file_path, (suffix, mode, type_) + + +def reload(module): + """**DEPRECATED** + + Reload the module and return it. + + The module must have been successfully imported before. + + """ + return importlib.reload(module) + + +def init_builtin(name): + """**DEPRECATED** + + Load and return a built-in module by name, or None is such module doesn't + exist + """ + try: + return _builtin_from_name(name) + except ImportError: + return None + + +if create_dynamic: + def load_dynamic(name, path, file=None): + """**DEPRECATED** + + Load an extension module. + """ + import importlib.machinery + loader = importlib.machinery.ExtensionFileLoader(name, path) + + # Issue #24748: Skip the sys.modules check in _load_module_shim; + # always load new extension + spec = importlib.machinery.ModuleSpec( + name=name, loader=loader, origin=path) + return _load(spec) + +else: + load_dynamic = None From f970f4fabc182b01157a0a1af7c25c605849c1e5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 16:04:08 +0300 Subject: [PATCH 026/209] Add global import lock --- vm/src/stdlib/imp.rs | 30 +++++++++++++++++++++--------- vm/src/stdlib/thread.rs | 2 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index c9625004e..1cbc24b85 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,28 +1,34 @@ use crate::import; +use crate::obj::objbytes::PyBytesRef; use crate::obj::objcode::PyCode; use crate::obj::objmodule::PyModuleRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; +use crate::stdlib::thread::RawRMutex; use crate::vm::VirtualMachine; +static IMP_LOCK: RawRMutex = RawRMutex::INIT; + fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_list(vec![])) } -fn imp_acquire_lock(_vm: &VirtualMachine) -> PyResult<()> { - // TODO - Ok(()) +fn imp_acquire_lock(_vm: &VirtualMachine) { + IMP_LOCK.lock() } -fn imp_release_lock(_vm: &VirtualMachine) -> PyResult<()> { - // TODO - Ok(()) +fn imp_release_lock(vm: &VirtualMachine) -> PyResult<()> { + if !IMP_LOCK.is_locked() { + Err(vm.new_runtime_error("Global import lock not held".to_owned())) + } else { + IMP_LOCK.unlock(); + Ok(()) + } } -fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { - // TODO - Ok(()) +fn imp_lock_held(_vm: &VirtualMachine) -> bool { + IMP_LOCK.is_locked() } fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { @@ -90,6 +96,11 @@ fn imp_fix_co_filename(_code: PyObjectRef, _path: PyStringRef) { // TODO: } +fn imp_source_hash(_key: u64, _source: PyBytesRef, vm: &VirtualMachine) -> PyResult { + // TODO: + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "_imp", { @@ -105,5 +116,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "init_frozen" => ctx.new_function(imp_init_frozen), "is_frozen_package" => ctx.new_function(imp_is_frozen_package), "_fix_co_filename" => ctx.new_function(imp_fix_co_filename), + "source_hash" => ctx.new_function(imp_source_hash) }) } diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index ff6707ac5..69da935b1 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -145,7 +145,7 @@ impl PyLock { } } -type RawRMutex = RawReentrantMutex; +pub type RawRMutex = RawReentrantMutex; #[pyclass(name = "RLock")] struct PyRLock { mu: RawRMutex, From 389f27070d7df4a2da3b2c1c278d58dd91b3624e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 16:04:41 +0300 Subject: [PATCH 027/209] Mark unsupported imp tests --- Lib/test/test_imp.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index fe394dc50..b42f14afb 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -57,17 +57,20 @@ class LockTests(unittest.TestCase): "RuntimeError") class ImportTests(unittest.TestCase): - def setUp(self): - mod = importlib.import_module('test.encoded_modules') - self.test_strings = mod.test_strings - self.test_path = mod.__path__ + # TODO: RustPython + # def setUp(self): + # mod = importlib.import_module('test.encoded_modules') + # self.test_strings = mod.test_strings + # self.test_path = mod.__path__ + @unittest.skip("TODO: RUSTPYTHON") def test_import_encoded_module(self): for modname, encoding, teststr in self.test_strings: mod = importlib.import_module('test.encoded_modules.' 'module_' + modname) self.assertEqual(teststr, mod.test) + @unittest.skip("TODO: RUSTPYTHON") def test_find_module_encoding(self): for mod, encoding, _ in self.test_strings: with imp.find_module('module_' + mod, self.test_path)[0] as fd: @@ -77,6 +80,7 @@ class ImportTests(unittest.TestCase): with self.assertRaises(SyntaxError): imp.find_module('badsyntax_pep3120', path) + @unittest.skip("TODO: RUSTPYTHON") def test_issue1267(self): for mod, encoding, _ in self.test_strings: fp, filename, info = imp.find_module('module_' + mod, @@ -96,6 +100,7 @@ class ImportTests(unittest.TestCase): self.assertEqual(fp.readline(), '"""Tokenization help for Python programs.\n') + @unittest.skip("TODO: RUSTPYTHON") def test_issue3594(self): temp_mod_name = 'test_imp_helper' sys.path.insert(0, '.') @@ -110,6 +115,7 @@ class ImportTests(unittest.TestCase): support.unlink(temp_mod_name + '.py') support.unlink(temp_mod_name + '.pyc') + @unittest.skip("TODO: RUSTPYTHON") def test_issue5604(self): # Test cannot cover imp.load_compiled function. # Martin von Loewis note what shared library cannot have non-ascii @@ -197,11 +203,13 @@ class ImportTests(unittest.TestCase): support.rmtree(test_package_name) support.rmtree('__pycache__') + @unittest.skip("TODO: RUSTPYTHON") def test_issue9319(self): path = os.path.dirname(__file__) self.assertRaises(SyntaxError, imp.find_module, "badsyntax_pep3120", [path]) + @unittest.skip("TODO: RUSTPYTHON") def test_load_from_source(self): # Verify that the imp module can correctly load and find .py files # XXX (ncoghlan): It would be nice to use support.CleanImport @@ -305,6 +313,7 @@ class ImportTests(unittest.TestCase): self.assertRaisesRegex(ImportError, '^No module', imp.find_module, support.TESTFN, ["."]) + @unittest.skip("TODO: RUSTPYTHON") def test_multiple_calls_to_get_data(self): # Issue #18755: make sure multiple calls to get_data() can succeed. loader = imp._LoadSourceCompatibility('imp', imp.__file__, @@ -312,6 +321,7 @@ class ImportTests(unittest.TestCase): loader.get_data(imp.__file__) # File should be closed loader.get_data(imp.__file__) # Will need to create a newly opened file + @unittest.skip("TODO: RUSTPYTHON") def test_load_source(self): # Create a temporary module since load_source(name) modifies # sys.modules[name] attributes like __loader___ @@ -343,10 +353,13 @@ class ImportTests(unittest.TestCase): import _frozen_importlib self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_source_hash(self): self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab') self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9') + @unittest.skip("TODO: RUSTPYTHON") def test_pyc_invalidation_mode_from_cmdline(self): cases = [ ([], "default"), @@ -362,6 +375,7 @@ class ImportTests(unittest.TestCase): res = script_helper.assert_python_ok(*args) self.assertEqual(res.out.strip().decode('utf-8'), expected) + @unittest.skip("TODO: RUSTPYTHON") def test_find_and_load_checked_pyc(self): # issue 34056 with support.temp_cwd(): From a98c15ddfd589b8251e75bc29299ccd5a1319c00 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 16:13:44 +0300 Subject: [PATCH 028/209] Add test_py_compile from CPython 3.8 --- Lib/test/bad_coding2.py | 2 + Lib/test/test_py_compile.py | 211 ++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 Lib/test/bad_coding2.py create mode 100644 Lib/test/test_py_compile.py diff --git a/Lib/test/bad_coding2.py b/Lib/test/bad_coding2.py new file mode 100644 index 000000000..bb2bb7e1e --- /dev/null +++ b/Lib/test/bad_coding2.py @@ -0,0 +1,2 @@ +#coding: utf8 +print('我') diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py new file mode 100644 index 000000000..f86abe26f --- /dev/null +++ b/Lib/test/test_py_compile.py @@ -0,0 +1,211 @@ +import functools +import importlib.util +import os +import py_compile +import shutil +import stat +import sys +import tempfile +import unittest + +from test import support + + +def without_source_date_epoch(fxn): + """Runs function with SOURCE_DATE_EPOCH unset.""" + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with support.EnvironmentVarGuard() as env: + env.unset('SOURCE_DATE_EPOCH') + return fxn(*args, **kwargs) + return wrapper + + +def with_source_date_epoch(fxn): + """Runs function with SOURCE_DATE_EPOCH set.""" + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with support.EnvironmentVarGuard() as env: + env['SOURCE_DATE_EPOCH'] = '123456789' + return fxn(*args, **kwargs) + return wrapper + + +# Run tests with SOURCE_DATE_EPOCH set or unset explicitly. +class SourceDateEpochTestMeta(type(unittest.TestCase)): + def __new__(mcls, name, bases, dct, *, source_date_epoch): + cls = super().__new__(mcls, name, bases, dct) + + for attr in dir(cls): + if attr.startswith('test_'): + meth = getattr(cls, attr) + if source_date_epoch: + wrapper = with_source_date_epoch(meth) + else: + wrapper = without_source_date_epoch(meth) + setattr(cls, attr, wrapper) + + return cls + + +class PyCompileTestsBase: + + def setUp(self): + self.directory = tempfile.mkdtemp() + self.source_path = os.path.join(self.directory, '_test.py') + self.pyc_path = self.source_path + 'c' + self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cwd_drive = os.path.splitdrive(os.getcwd())[0] + # In these tests we compute relative paths. When using Windows, the + # current working directory path and the 'self.source_path' might be + # on different drives. Therefore we need to switch to the drive where + # the temporary source file lives. + drive = os.path.splitdrive(self.source_path)[0] + if drive: + os.chdir(drive) + with open(self.source_path, 'w') as file: + file.write('x = 123\n') + + def tearDown(self): + shutil.rmtree(self.directory) + if self.cwd_drive: + os.chdir(self.cwd_drive) + + def test_absolute_path(self): + py_compile.compile(self.source_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + + def test_do_not_overwrite_symlinks(self): + # In the face of a cfile argument being a symlink, bail out. + # Issue #17222 + try: + os.symlink(self.pyc_path + '.actual', self.pyc_path) + except (NotImplementedError, OSError): + self.skipTest('need to be able to create a symlink for a file') + else: + assert os.path.islink(self.pyc_path) + with self.assertRaises(FileExistsError): + py_compile.compile(self.source_path, self.pyc_path) + + @unittest.skipIf(not os.path.exists(os.devnull) or os.path.isfile(os.devnull), + 'requires os.devnull and for it to be a non-regular file') + def test_do_not_overwrite_nonregular_files(self): + # In the face of a cfile argument being a non-regular file, bail out. + # Issue #17222 + with self.assertRaises(FileExistsError): + py_compile.compile(self.source_path, os.devnull) + + def test_cache_path(self): + py_compile.compile(self.source_path) + self.assertTrue(os.path.exists(self.cache_path)) + + def test_cwd(self): + with support.change_cwd(self.directory): + py_compile.compile(os.path.basename(self.source_path), + os.path.basename(self.pyc_path)) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + + def test_relative_path(self): + py_compile.compile(os.path.relpath(self.source_path), + os.path.relpath(self.pyc_path)) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + + @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, + 'non-root user required') + @unittest.skipIf(os.name == 'nt', + 'cannot control directory permissions on Windows') + def test_exceptions_propagate(self): + # Make sure that exceptions raised thanks to issues with writing + # bytecode. + # http://bugs.python.org/issue17244 + mode = os.stat(self.directory) + os.chmod(self.directory, stat.S_IREAD) + try: + with self.assertRaises(IOError): + py_compile.compile(self.source_path, self.pyc_path) + finally: + os.chmod(self.directory, mode.st_mode) + + def test_bad_coding(self): + bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') + with support.captured_stderr(): + self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) + self.assertFalse(os.path.exists( + importlib.util.cache_from_source(bad_coding))) + + def test_source_date_epoch(self): + py_compile.compile(self.source_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + with open(self.pyc_path, 'rb') as fp: + flags = importlib._bootstrap_external._classify_pyc( + fp.read(), 'test', {}) + if os.environ.get('SOURCE_DATE_EPOCH'): + expected_flags = 0b11 + else: + expected_flags = 0b00 + + self.assertEqual(flags, expected_flags) + + @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O') + def test_double_dot_no_clobber(self): + # http://bugs.python.org/issue22966 + # py_compile foo.bar.py -> __pycache__/foo.cpython-34.pyc + weird_path = os.path.join(self.directory, 'foo.bar.py') + cache_path = importlib.util.cache_from_source(weird_path) + pyc_path = weird_path + 'c' + head, tail = os.path.split(cache_path) + penultimate_tail = os.path.basename(head) + self.assertEqual( + os.path.join(penultimate_tail, tail), + os.path.join( + '__pycache__', + 'foo.bar.{}.pyc'.format(sys.implementation.cache_tag))) + with open(weird_path, 'w') as file: + file.write('x = 123\n') + py_compile.compile(weird_path) + self.assertTrue(os.path.exists(cache_path)) + self.assertFalse(os.path.exists(pyc_path)) + + def test_optimization_path(self): + # Specifying optimized bytecode should lead to a path reflecting that. + self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2)) + + def test_invalidation_mode(self): + py_compile.compile( + self.source_path, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + with open(self.cache_path, 'rb') as fp: + flags = importlib._bootstrap_external._classify_pyc( + fp.read(), 'test', {}) + self.assertEqual(flags, 0b11) + py_compile.compile( + self.source_path, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + with open(self.cache_path, 'rb') as fp: + flags = importlib._bootstrap_external._classify_pyc( + fp.read(), 'test', {}) + self.assertEqual(flags, 0b1) + + +class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, + unittest.TestCase, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=True): + pass + + +class PyCompileTestsWithoutSourceEpoch(PyCompileTestsBase, + unittest.TestCase, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=False): + pass + + +if __name__ == "__main__": + unittest.main() From 0cdb3d1eb083b1c7418f9795131e8030760f577f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 16:14:17 +0300 Subject: [PATCH 029/209] Mark unsupported py_compile tests --- Lib/test/test_py_compile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index f86abe26f..912f0f0ea 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -174,6 +174,7 @@ class PyCompileTestsBase: # Specifying optimized bytecode should lead to a path reflecting that. self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2)) + @unittest.skip("TODO: RUSTPYTHON") def test_invalidation_mode(self): py_compile.compile( self.source_path, @@ -193,6 +194,7 @@ class PyCompileTestsBase: self.assertEqual(flags, 0b1) +@unittest.skip("TODO: RUSTPYTHON") class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, unittest.TestCase, metaclass=SourceDateEpochTestMeta, From e6d413b30b47d9a6e3bf51798e1e505a302a1536 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Jun 2020 16:16:33 +0300 Subject: [PATCH 030/209] Add test_importlib from CPython 3.8 --- Lib/test/test_importlib/__init__.py | 5 + Lib/test/test_importlib/__main__.py | 4 + Lib/test/test_importlib/abc.py | 93 ++ Lib/test/test_importlib/builtin/__init__.py | 5 + Lib/test/test_importlib/builtin/__main__.py | 4 + .../test_importlib/builtin/test_finder.py | 90 ++ .../test_importlib/builtin/test_loader.py | 108 ++ Lib/test/test_importlib/data01/__init__.py | 0 Lib/test/test_importlib/data01/binary.file | Bin 0 -> 4 bytes .../data01/subdirectory/__init__.py | 0 .../data01/subdirectory/binary.file | Bin 0 -> 4 bytes Lib/test/test_importlib/data01/utf-16.file | Bin 0 -> 44 bytes Lib/test/test_importlib/data01/utf-8.file | 1 + Lib/test/test_importlib/data02/__init__.py | 0 .../test_importlib/data02/one/__init__.py | 0 .../test_importlib/data02/one/resource1.txt | 1 + .../test_importlib/data02/two/__init__.py | 0 .../test_importlib/data02/two/resource2.txt | 1 + Lib/test/test_importlib/data03/__init__.py | 0 .../data03/namespace/portion1/__init__.py | 0 .../data03/namespace/portion2/__init__.py | 0 .../data03/namespace/resource1.txt | 0 Lib/test/test_importlib/extension/__init__.py | 5 + Lib/test/test_importlib/extension/__main__.py | 4 + .../extension/test_case_sensitivity.py | 46 + .../test_importlib/extension/test_finder.py | 44 + .../test_importlib/extension/test_loader.py | 300 +++++ .../extension/test_path_hook.py | 31 + Lib/test/test_importlib/frozen/__init__.py | 5 + Lib/test/test_importlib/frozen/__main__.py | 4 + Lib/test/test_importlib/frozen/test_finder.py | 84 ++ Lib/test/test_importlib/frozen/test_loader.py | 225 ++++ Lib/test/test_importlib/import_/__init__.py | 5 + Lib/test/test_importlib/import_/__main__.py | 4 + .../test_importlib/import_/test___loader__.py | 75 ++ .../import_/test___package__.py | 163 +++ Lib/test/test_importlib/import_/test_api.py | 119 ++ .../test_importlib/import_/test_caching.py | 93 ++ .../test_importlib/import_/test_fromlist.py | 175 +++ .../test_importlib/import_/test_meta_path.py | 125 ++ .../test_importlib/import_/test_packages.py | 110 ++ Lib/test/test_importlib/import_/test_path.py | 278 +++++ .../import_/test_relative_imports.py | 232 ++++ .../namespace_pkgs/both_portions/foo/one.py | 1 + .../namespace_pkgs/both_portions/foo/two.py | 1 + .../namespace_pkgs/missing_directory.zip | Bin 0 -> 515 bytes .../module_and_namespace_package/a_test.py | 1 + .../module_and_namespace_package/a_test/empty | 0 .../namespace_pkgs/nested_portion1.zip | Bin 0 -> 556 bytes .../not_a_namespace_pkg/foo/__init__.py | 0 .../not_a_namespace_pkg/foo/one.py | 1 + .../namespace_pkgs/portion1/foo/one.py | 1 + .../namespace_pkgs/portion2/foo/two.py | 1 + .../project1/parent/child/one.py | 1 + .../project2/parent/child/two.py | 1 + .../project3/parent/child/three.py | 1 + .../namespace_pkgs/top_level_portion1.zip | Bin 0 -> 332 bytes Lib/test/test_importlib/source/__init__.py | 5 + Lib/test/test_importlib/source/__main__.py | 4 + .../source/test_case_sensitivity.py | 85 ++ .../test_importlib/source/test_file_loader.py | 797 +++++++++++++ Lib/test/test_importlib/source/test_finder.py | 236 ++++ .../test_importlib/source/test_path_hook.py | 41 + .../source/test_source_encoding.py | 175 +++ Lib/test/test_importlib/test_abc.py | 1001 +++++++++++++++++ Lib/test/test_importlib/test_api.py | 461 ++++++++ Lib/test/test_importlib/test_lazy.py | 145 +++ Lib/test/test_importlib/test_locks.py | 152 +++ .../test_importlib/test_namespace_pkgs.py | 343 ++++++ Lib/test/test_importlib/test_open.py | 72 ++ Lib/test/test_importlib/test_path.py | 39 + Lib/test/test_importlib/test_read.py | 62 + Lib/test/test_importlib/test_resource.py | 166 +++ Lib/test/test_importlib/test_spec.py | 819 ++++++++++++++ Lib/test/test_importlib/test_util.py | 884 +++++++++++++++ Lib/test/test_importlib/test_windows.py | 109 ++ Lib/test/test_importlib/util.py | 569 ++++++++++ Lib/test/test_importlib/zipdata01/__init__.py | 0 .../test_importlib/zipdata01/ziptestdata.zip | Bin 0 -> 876 bytes Lib/test/test_importlib/zipdata02/__init__.py | 0 .../test_importlib/zipdata02/ziptestdata.zip | Bin 0 -> 698 bytes 81 files changed, 8613 insertions(+) create mode 100644 Lib/test/test_importlib/__init__.py create mode 100644 Lib/test/test_importlib/__main__.py create mode 100644 Lib/test/test_importlib/abc.py create mode 100644 Lib/test/test_importlib/builtin/__init__.py create mode 100644 Lib/test/test_importlib/builtin/__main__.py create mode 100644 Lib/test/test_importlib/builtin/test_finder.py create mode 100644 Lib/test/test_importlib/builtin/test_loader.py create mode 100644 Lib/test/test_importlib/data01/__init__.py create mode 100644 Lib/test/test_importlib/data01/binary.file create mode 100644 Lib/test/test_importlib/data01/subdirectory/__init__.py create mode 100644 Lib/test/test_importlib/data01/subdirectory/binary.file create mode 100644 Lib/test/test_importlib/data01/utf-16.file create mode 100644 Lib/test/test_importlib/data01/utf-8.file create mode 100644 Lib/test/test_importlib/data02/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/__init__.py create mode 100644 Lib/test/test_importlib/data02/one/resource1.txt create mode 100644 Lib/test/test_importlib/data02/two/__init__.py create mode 100644 Lib/test/test_importlib/data02/two/resource2.txt create mode 100644 Lib/test/test_importlib/data03/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion1/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/portion2/__init__.py create mode 100644 Lib/test/test_importlib/data03/namespace/resource1.txt create mode 100644 Lib/test/test_importlib/extension/__init__.py create mode 100644 Lib/test/test_importlib/extension/__main__.py create mode 100644 Lib/test/test_importlib/extension/test_case_sensitivity.py create mode 100644 Lib/test/test_importlib/extension/test_finder.py create mode 100644 Lib/test/test_importlib/extension/test_loader.py create mode 100644 Lib/test/test_importlib/extension/test_path_hook.py create mode 100644 Lib/test/test_importlib/frozen/__init__.py create mode 100644 Lib/test/test_importlib/frozen/__main__.py create mode 100644 Lib/test/test_importlib/frozen/test_finder.py create mode 100644 Lib/test/test_importlib/frozen/test_loader.py create mode 100644 Lib/test/test_importlib/import_/__init__.py create mode 100644 Lib/test/test_importlib/import_/__main__.py create mode 100644 Lib/test/test_importlib/import_/test___loader__.py create mode 100644 Lib/test/test_importlib/import_/test___package__.py create mode 100644 Lib/test/test_importlib/import_/test_api.py create mode 100644 Lib/test/test_importlib/import_/test_caching.py create mode 100644 Lib/test/test_importlib/import_/test_fromlist.py create mode 100644 Lib/test/test_importlib/import_/test_meta_path.py create mode 100644 Lib/test/test_importlib/import_/test_packages.py create mode 100644 Lib/test/test_importlib/import_/test_path.py create mode 100644 Lib/test/test_importlib/import_/test_relative_imports.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/missing_directory.zip create mode 100644 Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty create mode 100644 Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip create mode 100644 Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py create mode 100644 Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip create mode 100644 Lib/test/test_importlib/source/__init__.py create mode 100644 Lib/test/test_importlib/source/__main__.py create mode 100644 Lib/test/test_importlib/source/test_case_sensitivity.py create mode 100644 Lib/test/test_importlib/source/test_file_loader.py create mode 100644 Lib/test/test_importlib/source/test_finder.py create mode 100644 Lib/test/test_importlib/source/test_path_hook.py create mode 100644 Lib/test/test_importlib/source/test_source_encoding.py create mode 100644 Lib/test/test_importlib/test_abc.py create mode 100644 Lib/test/test_importlib/test_api.py create mode 100644 Lib/test/test_importlib/test_lazy.py create mode 100644 Lib/test/test_importlib/test_locks.py create mode 100644 Lib/test/test_importlib/test_namespace_pkgs.py create mode 100644 Lib/test/test_importlib/test_open.py create mode 100644 Lib/test/test_importlib/test_path.py create mode 100644 Lib/test/test_importlib/test_read.py create mode 100644 Lib/test/test_importlib/test_resource.py create mode 100644 Lib/test/test_importlib/test_spec.py create mode 100644 Lib/test/test_importlib/test_util.py create mode 100644 Lib/test/test_importlib/test_windows.py create mode 100644 Lib/test/test_importlib/util.py create mode 100644 Lib/test/test_importlib/zipdata01/__init__.py create mode 100644 Lib/test/test_importlib/zipdata01/ziptestdata.zip create mode 100644 Lib/test/test_importlib/zipdata02/__init__.py create mode 100644 Lib/test/test_importlib/zipdata02/ziptestdata.zip diff --git a/Lib/test/test_importlib/__init__.py b/Lib/test/test_importlib/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/__main__.py b/Lib/test/test_importlib/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/abc.py b/Lib/test/test_importlib/abc.py new file mode 100644 index 000000000..5d4b95876 --- /dev/null +++ b/Lib/test/test_importlib/abc.py @@ -0,0 +1,93 @@ +import abc + + +class FinderTests(metaclass=abc.ABCMeta): + + """Basic tests for a finder to pass.""" + + @abc.abstractmethod + def test_module(self): + # Test importing a top-level module. + pass + + @abc.abstractmethod + def test_package(self): + # Test importing a package. + pass + + @abc.abstractmethod + def test_module_in_package(self): + # Test importing a module contained within a package. + # A value for 'path' should be used if for a meta_path finder. + pass + + @abc.abstractmethod + def test_package_in_package(self): + # Test importing a subpackage. + # A value for 'path' should be used if for a meta_path finder. + pass + + @abc.abstractmethod + def test_package_over_module(self): + # Test that packages are chosen over modules. + pass + + @abc.abstractmethod + def test_failure(self): + # Test trying to find a module that cannot be handled. + pass + + +class LoaderTests(metaclass=abc.ABCMeta): + + @abc.abstractmethod + def test_module(self): + """A module should load without issue. + + After the loader returns the module should be in sys.modules. + + Attributes to verify: + + * __file__ + * __loader__ + * __name__ + * No __path__ + + """ + pass + + @abc.abstractmethod + def test_package(self): + """Loading a package should work. + + After the loader returns the module should be in sys.modules. + + Attributes to verify: + + * __name__ + * __file__ + * __package__ + * __path__ + * __loader__ + + """ + pass + + @abc.abstractmethod + def test_lacking_parent(self): + """A loader should not be dependent on it's parent package being + imported.""" + pass + + @abc.abstractmethod + def test_state_after_failure(self): + """If a module is already in sys.modules and a reload fails + (e.g. a SyntaxError), the module should be in the state it was before + the reload began.""" + pass + + @abc.abstractmethod + def test_unloadable(self): + """Test ImportError is raised when the loader is asked to load a module + it can't.""" + pass diff --git a/Lib/test/test_importlib/builtin/__init__.py b/Lib/test/test_importlib/builtin/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/builtin/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/builtin/__main__.py b/Lib/test/test_importlib/builtin/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/builtin/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py new file mode 100644 index 000000000..084f3de6b --- /dev/null +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -0,0 +1,90 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import sys +import unittest + + +@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') +class FindSpecTests(abc.FinderTests): + + """Test find_spec() for built-in modules.""" + + def test_module(self): + # Common case. + with util.uncache(util.BUILTINS.good_name): + found = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name) + self.assertTrue(found) + self.assertEqual(found.origin, 'built-in') + + # Built-in modules cannot be a package. + test_package = None + + # Built-in modules cannot be in a package. + test_module_in_package = None + + # Built-in modules cannot be a package. + test_package_in_package = None + + # Built-in modules cannot be a package. + test_package_over_module = None + + def test_failure(self): + name = 'importlib' + assert name not in sys.builtin_module_names + spec = self.machinery.BuiltinImporter.find_spec(name) + self.assertIsNone(spec) + + def test_ignore_path(self): + # The value for 'path' should always trigger a failed import. + with util.uncache(util.BUILTINS.good_name): + spec = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name, + ['pkg']) + self.assertIsNone(spec) + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, machinery=machinery) + + +@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') +class FinderTests(abc.FinderTests): + + """Test find_module() for built-in modules.""" + + def test_module(self): + # Common case. + with util.uncache(util.BUILTINS.good_name): + found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name) + self.assertTrue(found) + self.assertTrue(hasattr(found, 'load_module')) + + # Built-in modules cannot be a package. + test_package = test_package_in_package = test_package_over_module = None + + # Built-in modules cannot be in a package. + test_module_in_package = None + + def test_failure(self): + assert 'importlib' not in sys.builtin_module_names + loader = self.machinery.BuiltinImporter.find_module('importlib') + self.assertIsNone(loader) + + def test_ignore_path(self): + # The value for 'path' should always trigger a failed import. + with util.uncache(util.BUILTINS.good_name): + loader = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name, + ['pkg']) + self.assertIsNone(loader) + + +(Frozen_FinderTests, + Source_FinderTests + ) = util.test_both(FinderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py new file mode 100644 index 000000000..b1349ec5d --- /dev/null +++ b/Lib/test/test_importlib/builtin/test_loader.py @@ -0,0 +1,108 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import sys +import types +import unittest + +@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') +class LoaderTests(abc.LoaderTests): + + """Test load_module() for built-in modules.""" + + def setUp(self): + self.verification = {'__name__': 'errno', '__package__': '', + '__loader__': self.machinery.BuiltinImporter} + + def verify(self, module): + """Verify that the module matches against what it should have.""" + self.assertIsInstance(module, types.ModuleType) + for attr, value in self.verification.items(): + self.assertEqual(getattr(module, attr), value) + self.assertIn(module.__name__, sys.modules) + + def load_module(self, name): + return self.machinery.BuiltinImporter.load_module(name) + + def test_module(self): + # Common case. + with util.uncache(util.BUILTINS.good_name): + module = self.load_module(util.BUILTINS.good_name) + self.verify(module) + + # Built-in modules cannot be a package. + test_package = test_lacking_parent = None + + # No way to force an import failure. + test_state_after_failure = None + + def test_module_reuse(self): + # Test that the same module is used in a reload. + with util.uncache(util.BUILTINS.good_name): + module1 = self.load_module(util.BUILTINS.good_name) + module2 = self.load_module(util.BUILTINS.good_name) + self.assertIs(module1, module2) + + def test_unloadable(self): + name = 'dssdsdfff' + assert name not in sys.builtin_module_names + with self.assertRaises(ImportError) as cm: + self.load_module(name) + self.assertEqual(cm.exception.name, name) + + def test_already_imported(self): + # Using the name of a module already imported but not a built-in should + # still fail. + module_name = 'builtin_reload_test' + assert module_name not in sys.builtin_module_names + with util.uncache(module_name): + module = types.ModuleType(module_name) + sys.modules[module_name] = module + with self.assertRaises(ImportError) as cm: + self.load_module(module_name) + self.assertEqual(cm.exception.name, module_name) + + +(Frozen_LoaderTests, + Source_LoaderTests + ) = util.test_both(LoaderTests, machinery=machinery) + + +@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') +class InspectLoaderTests: + + """Tests for InspectLoader methods for BuiltinImporter.""" + + def test_get_code(self): + # There is no code object. + result = self.machinery.BuiltinImporter.get_code(util.BUILTINS.good_name) + self.assertIsNone(result) + + def test_get_source(self): + # There is no source. + result = self.machinery.BuiltinImporter.get_source(util.BUILTINS.good_name) + self.assertIsNone(result) + + def test_is_package(self): + # Cannot be a package. + result = self.machinery.BuiltinImporter.is_package(util.BUILTINS.good_name) + self.assertFalse(result) + + @unittest.skipIf(util.BUILTINS.bad_name is None, 'all modules are built in') + def test_not_builtin(self): + # Modules not built-in should raise ImportError. + for meth_name in ('get_code', 'get_source', 'is_package'): + method = getattr(self.machinery.BuiltinImporter, meth_name) + with self.assertRaises(ImportError) as cm: + method(util.BUILTINS.bad_name) + + +(Frozen_InspectLoaderTests, + Source_InspectLoaderTests + ) = util.test_both(InspectLoaderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file new file mode 100644 index 0000000000000000000000000000000000000000..2cb772295ef4b480a8d83725bd5006a0236d8f68 GIT binary patch literal 44 ucmezW&x0YAAqNQa8FUyF7(y9B7~B|i84MZBfV^^`Xc15@g+Y;liva-T)Ce>H literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file new file mode 100644 index 000000000..1c0132ad9 --- /dev/null +++ b/Lib/test/test_importlib/data01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt new file mode 100644 index 000000000..61a813e40 --- /dev/null +++ b/Lib/test/test_importlib/data02/one/resource1.txt @@ -0,0 +1 @@ +one resource diff --git a/Lib/test/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt new file mode 100644 index 000000000..a80ce46ea --- /dev/null +++ b/Lib/test/test_importlib/data02/two/resource2.txt @@ -0,0 +1 @@ +two resource diff --git a/Lib/test/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/extension/__init__.py b/Lib/test/test_importlib/extension/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/extension/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/extension/__main__.py b/Lib/test/test_importlib/extension/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/extension/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py new file mode 100644 index 000000000..0dd9c8615 --- /dev/null +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -0,0 +1,46 @@ +from importlib import _bootstrap_external +from test import support +import unittest + +from .. import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + + +@unittest.skipIf(util.EXTENSIONS.filename is None, '_testcapi not available') +@util.case_insensitive_tests +class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): + + def find_module(self): + good_name = util.EXTENSIONS.name + bad_name = good_name.upper() + assert good_name != bad_name + finder = self.machinery.FileFinder(util.EXTENSIONS.path, + (self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES)) + return finder.find_module(bad_name) + + def test_case_sensitive(self): + with support.EnvironmentVarGuard() as env: + env.unset('PYTHONCASEOK') + self.caseok_env_changed(should_exist=False) + loader = self.find_module() + self.assertIsNone(loader) + + def test_case_insensitivity(self): + with support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + self.caseok_env_changed(should_exist=True) + loader = self.find_module() + self.assertTrue(hasattr(loader, 'load_module')) + + +(Frozen_ExtensionCaseSensitivity, + Source_ExtensionCaseSensitivity + ) = util.test_both(ExtensionModuleCaseSensitivityTest, importlib=importlib, + machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py new file mode 100644 index 000000000..c9b4a3772 --- /dev/null +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -0,0 +1,44 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import unittest +import warnings + + +class FinderTests(abc.FinderTests): + + """Test the finder for extension modules.""" + + def find_module(self, fullname): + importer = self.machinery.FileFinder(util.EXTENSIONS.path, + (self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES)) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return importer.find_module(fullname) + + def test_module(self): + self.assertTrue(self.find_module(util.EXTENSIONS.name)) + + # No extension module as an __init__ available for testing. + test_package = test_package_in_package = None + + # No extension module in a package available for testing. + test_module_in_package = None + + # Extension modules cannot be an __init__ for a package. + test_package_over_module = None + + def test_failure(self): + self.assertIsNone(self.find_module('asdfjkl;')) + + +(Frozen_FinderTests, + Source_FinderTests + ) = util.test_both(FinderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py new file mode 100644 index 000000000..9ad05fade --- /dev/null +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -0,0 +1,300 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import os.path +import sys +import types +import unittest +import importlib.util +import importlib +from test.support.script_helper import assert_python_failure + +class LoaderTests(abc.LoaderTests): + + """Test load_module() for extension modules.""" + + def setUp(self): + self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, + util.EXTENSIONS.file_path) + + def load_module(self, fullname): + return self.loader.load_module(fullname) + + def test_load_module_API(self): + # Test the default argument for load_module(). + self.loader.load_module() + self.loader.load_module(None) + with self.assertRaises(ImportError): + self.load_module('XXX') + + def test_equality(self): + other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, + util.EXTENSIONS.file_path) + self.assertEqual(self.loader, other) + + def test_inequality(self): + other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, + util.EXTENSIONS.file_path) + self.assertNotEqual(self.loader, other) + + def test_module(self): + with util.uncache(util.EXTENSIONS.name): + module = self.load_module(util.EXTENSIONS.name) + for attr, value in [('__name__', util.EXTENSIONS.name), + ('__file__', util.EXTENSIONS.file_path), + ('__package__', '')]: + self.assertEqual(getattr(module, attr), value) + self.assertIn(util.EXTENSIONS.name, sys.modules) + self.assertIsInstance(module.__loader__, + self.machinery.ExtensionFileLoader) + + # No extension module as __init__ available for testing. + test_package = None + + # No extension module in a package available for testing. + test_lacking_parent = None + + def test_module_reuse(self): + with util.uncache(util.EXTENSIONS.name): + module1 = self.load_module(util.EXTENSIONS.name) + module2 = self.load_module(util.EXTENSIONS.name) + self.assertIs(module1, module2) + + # No easy way to trigger a failure after a successful import. + test_state_after_failure = None + + def test_unloadable(self): + name = 'asdfjkl;' + with self.assertRaises(ImportError) as cm: + self.load_module(name) + self.assertEqual(cm.exception.name, name) + + def test_is_package(self): + self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) + for suffix in self.machinery.EXTENSION_SUFFIXES: + path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) + loader = self.machinery.ExtensionFileLoader('pkg', path) + self.assertTrue(loader.is_package('pkg')) + +(Frozen_LoaderTests, + Source_LoaderTests + ) = util.test_both(LoaderTests, machinery=machinery) + +class MultiPhaseExtensionModuleTests(abc.LoaderTests): + """Test loading extension modules with multi-phase initialization (PEP 489) + """ + + def setUp(self): + self.name = '_testmultiphase' + finder = self.machinery.FileFinder(None) + self.spec = importlib.util.find_spec(self.name) + assert self.spec + self.loader = self.machinery.ExtensionFileLoader( + self.name, self.spec.origin) + + # No extension module as __init__ available for testing. + test_package = None + + # No extension module in a package available for testing. + test_lacking_parent = None + + # Handling failure on reload is the up to the module. + test_state_after_failure = None + + def test_module(self): + '''Test loading an extension module''' + with util.uncache(self.name): + module = self.load_module() + for attr, value in [('__name__', self.name), + ('__file__', self.spec.origin), + ('__package__', '')]: + self.assertEqual(getattr(module, attr), value) + with self.assertRaises(AttributeError): + module.__path__ + self.assertIs(module, sys.modules[self.name]) + self.assertIsInstance(module.__loader__, + self.machinery.ExtensionFileLoader) + + def test_functionality(self): + '''Test basic functionality of stuff defined in an extension module''' + with util.uncache(self.name): + module = self.load_module() + self.assertIsInstance(module, types.ModuleType) + ex = module.Example() + self.assertEqual(ex.demo('abcd'), 'abcd') + self.assertEqual(ex.demo(), None) + with self.assertRaises(AttributeError): + ex.abc + ex.abc = 0 + self.assertEqual(ex.abc, 0) + self.assertEqual(module.foo(9, 9), 18) + self.assertIsInstance(module.Str(), str) + self.assertEqual(module.Str(1) + '23', '123') + with self.assertRaises(module.error): + raise module.error() + self.assertEqual(module.int_const, 1969) + self.assertEqual(module.str_const, 'something different') + + def test_reload(self): + '''Test that reload didn't re-set the module's attributes''' + with util.uncache(self.name): + module = self.load_module() + ex_class = module.Example + importlib.reload(module) + self.assertIs(ex_class, module.Example) + + def test_try_registration(self): + '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work''' + module = self.load_module() + with self.subTest('PyState_FindModule'): + self.assertEqual(module.call_state_registration_func(0), None) + with self.subTest('PyState_AddModule'): + with self.assertRaises(SystemError): + module.call_state_registration_func(1) + with self.subTest('PyState_RemoveModule'): + with self.assertRaises(SystemError): + module.call_state_registration_func(2) + + def load_module(self): + '''Load the module from the test extension''' + return self.loader.load_module(self.name) + + def load_module_by_name(self, fullname): + '''Load a module from the test extension by name''' + origin = self.spec.origin + loader = self.machinery.ExtensionFileLoader(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + def test_load_submodule(self): + '''Test loading a simulated submodule''' + module = self.load_module_by_name('pkg.' + self.name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, 'pkg.' + self.name) + self.assertEqual(module.str_const, 'something different') + + def test_load_short_name(self): + '''Test loading module with a one-character name''' + module = self.load_module_by_name('x') + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, 'x') + self.assertEqual(module.str_const, 'something different') + self.assertNotIn('x', sys.modules) + + def test_load_twice(self): + '''Test that 2 loads result in 2 module objects''' + module1 = self.load_module_by_name(self.name) + module2 = self.load_module_by_name(self.name) + self.assertIsNot(module1, module2) + + def test_unloadable(self): + '''Test nonexistent module''' + name = 'asdfjkl;' + with self.assertRaises(ImportError) as cm: + self.load_module_by_name(name) + self.assertEqual(cm.exception.name, name) + + def test_unloadable_nonascii(self): + '''Test behavior with nonexistent module with non-ASCII name''' + name = 'fo\xf3' + with self.assertRaises(ImportError) as cm: + self.load_module_by_name(name) + self.assertEqual(cm.exception.name, name) + + def test_nonmodule(self): + '''Test returning a non-module object from create works''' + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + '''Test creating a non-module object with methods defined''' + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + '''Test that NULL slots aren't a problem''' + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + def test_bad_modules(self): + '''Test SystemError is raised for misbehaving extensions''' + for name_base in [ + 'bad_slot_large', + 'bad_slot_negative', + 'create_int_with_state', + 'negative_size', + 'export_null', + 'export_uninitialized', + 'export_raise', + 'export_unreported_exception', + 'create_null', + 'create_raise', + 'create_unreported_exception', + 'nonmodule_with_exec_slots', + 'exec_err', + 'exec_raise', + 'exec_unreported_exception', + ]: + with self.subTest(name_base): + name = self.name + '_' + name_base + with self.assertRaises(SystemError): + self.load_module_by_name(name) + + def test_nonascii(self): + '''Test that modules with non-ASCII names can be loaded''' + # punycode behaves slightly differently in some-ASCII and no-ASCII + # cases, so test both + cases = [ + (self.name + '_zkou\u0161ka_na\u010dten\xed', 'Czech'), + ('\uff3f\u30a4\u30f3\u30dd\u30fc\u30c8\u30c6\u30b9\u30c8', + 'Japanese'), + ] + for name, lang in cases: + with self.subTest(name): + module = self.load_module_by_name(name) + self.assertEqual(module.__name__, name) + self.assertEqual(module.__doc__, "Module named in %s" % lang) + + @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'), + '--with-pydebug has to be enabled for this test') + def test_bad_traverse(self): + ''' Issue #32374: Test that traverse fails when accessing per-module + state before Py_mod_exec was executed. + (Multiphase initialization modules only) + ''' + script = """if True: + try: + from test import support + import importlib.util as util + spec = util.find_spec('_testmultiphase') + spec.name = '_testmultiphase_with_bad_traverse' + + with support.SuppressCrashReport(): + m = spec.loader.create_module(spec) + except: + # Prevent Python-level exceptions from + # ending the process with non-zero status + # (We are testing for a crash in C-code) + pass""" + assert_python_failure("-c", script) + + +(Frozen_MultiPhaseExtensionModuleTests, + Source_MultiPhaseExtensionModuleTests + ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py new file mode 100644 index 000000000..a4b5a64aa --- /dev/null +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -0,0 +1,31 @@ +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import unittest + + +class PathHookTests: + + """Test the path hook for extension modules.""" + # XXX Should it only succeed for pre-existing directories? + # XXX Should it only work for directories containing an extension module? + + def hook(self, entry): + return self.machinery.FileFinder.path_hook( + (self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES))(entry) + + def test_success(self): + # Path hook should handle a directory where a known extension module + # exists. + self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_module')) + + +(Frozen_PathHooksTests, + Source_PathHooksTests + ) = util.test_both(PathHookTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/frozen/__init__.py b/Lib/test/test_importlib/frozen/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/frozen/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/frozen/__main__.py b/Lib/test/test_importlib/frozen/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/frozen/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py new file mode 100644 index 000000000..519aa027d --- /dev/null +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -0,0 +1,84 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import unittest + + +class FindSpecTests(abc.FinderTests): + + """Test finding frozen modules.""" + + def find(self, name, path=None): + finder = self.machinery.FrozenImporter + return finder.find_spec(name, path) + + def test_module(self): + name = '__hello__' + spec = self.find(name) + self.assertEqual(spec.origin, 'frozen') + + def test_package(self): + spec = self.find('__phello__') + self.assertIsNotNone(spec) + + def test_module_in_package(self): + spec = self.find('__phello__.spam', ['__phello__']) + self.assertIsNotNone(spec) + + # No frozen package within another package to test with. + test_package_in_package = None + + # No easy way to test. + test_package_over_module = None + + def test_failure(self): + spec = self.find('') + self.assertIsNone(spec) + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, machinery=machinery) + + +class FinderTests(abc.FinderTests): + + """Test finding frozen modules.""" + + def find(self, name, path=None): + finder = self.machinery.FrozenImporter + return finder.find_module(name, path) + + def test_module(self): + name = '__hello__' + loader = self.find(name) + self.assertTrue(hasattr(loader, 'load_module')) + + def test_package(self): + loader = self.find('__phello__') + self.assertTrue(hasattr(loader, 'load_module')) + + def test_module_in_package(self): + loader = self.find('__phello__.spam', ['__phello__']) + self.assertTrue(hasattr(loader, 'load_module')) + + # No frozen package within another package to test with. + test_package_in_package = None + + # No easy way to test. + test_package_over_module = None + + def test_failure(self): + loader = self.find('') + self.assertIsNone(loader) + + +(Frozen_FinderTests, + Source_FinderTests + ) = util.test_both(FinderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py new file mode 100644 index 000000000..29ecff177 --- /dev/null +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -0,0 +1,225 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +from test.support import captured_stdout +import types +import unittest +import warnings + + +class ExecModuleTests(abc.LoaderTests): + + def exec_module(self, name): + with util.uncache(name), captured_stdout() as stdout: + spec = self.machinery.ModuleSpec( + name, self.machinery.FrozenImporter, origin='frozen', + is_package=self.machinery.FrozenImporter.is_package(name)) + module = types.ModuleType(name) + module.__spec__ = spec + assert not hasattr(module, 'initialized') + self.machinery.FrozenImporter.exec_module(module) + self.assertTrue(module.initialized) + self.assertTrue(hasattr(module, '__spec__')) + self.assertEqual(module.__spec__.origin, 'frozen') + return module, stdout.getvalue() + + def test_module(self): + name = '__hello__' + module, output = self.exec_module(name) + check = {'__name__': name} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + self.assertEqual(output, 'Hello world!\n') + self.assertTrue(hasattr(module, '__spec__')) + + def test_package(self): + name = '__phello__' + module, output = self.exec_module(name) + check = {'__name__': name} + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + 'for {name}.{attr}, {given!r} != {expected!r}'.format( + name=name, attr=attr, given=attr_value, + expected=value)) + self.assertEqual(output, 'Hello world!\n') + + def test_lacking_parent(self): + name = '__phello__.spam' + with util.uncache('__phello__'): + module, output = self.exec_module(name) + check = {'__name__': name} + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + 'for {name}.{attr}, {given} != {expected!r}'.format( + name=name, attr=attr, given=attr_value, + expected=value)) + self.assertEqual(output, 'Hello world!\n') + + def test_module_repr(self): + name = '__hello__' + module, output = self.exec_module(name) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + repr_str = self.machinery.FrozenImporter.module_repr(module) + self.assertEqual(repr_str, + "") + + def test_module_repr_indirect(self): + name = '__hello__' + module, output = self.exec_module(name) + self.assertEqual(repr(module), + "") + + # No way to trigger an error in a frozen module. + test_state_after_failure = None + + def test_unloadable(self): + assert self.machinery.FrozenImporter.find_module('_not_real') is None + with self.assertRaises(ImportError) as cm: + self.exec_module('_not_real') + self.assertEqual(cm.exception.name, '_not_real') + + +(Frozen_ExecModuleTests, + Source_ExecModuleTests + ) = util.test_both(ExecModuleTests, machinery=machinery) + + +class LoaderTests(abc.LoaderTests): + + def test_module(self): + with util.uncache('__hello__'), captured_stdout() as stdout: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.machinery.FrozenImporter.load_module('__hello__') + check = {'__name__': '__hello__', + '__package__': '', + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) + + def test_package(self): + with util.uncache('__phello__'), captured_stdout() as stdout: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.machinery.FrozenImporter.load_module('__phello__') + check = {'__name__': '__phello__', + '__package__': '__phello__', + '__path__': [], + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) + + def test_lacking_parent(self): + with util.uncache('__phello__', '__phello__.spam'), \ + captured_stdout() as stdout: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.machinery.FrozenImporter.load_module('__phello__.spam') + check = {'__name__': '__phello__.spam', + '__package__': '__phello__', + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.spam.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) + + def test_module_reuse(self): + with util.uncache('__hello__'), captured_stdout() as stdout: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module1 = self.machinery.FrozenImporter.load_module('__hello__') + module2 = self.machinery.FrozenImporter.load_module('__hello__') + self.assertIs(module1, module2) + self.assertEqual(stdout.getvalue(), + 'Hello world!\nHello world!\n') + + def test_module_repr(self): + with util.uncache('__hello__'), captured_stdout(): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.machinery.FrozenImporter.load_module('__hello__') + repr_str = self.machinery.FrozenImporter.module_repr(module) + self.assertEqual(repr_str, + "") + + def test_module_repr_indirect(self): + with util.uncache('__hello__'), captured_stdout(): + module = self.machinery.FrozenImporter.load_module('__hello__') + self.assertEqual(repr(module), + "") + + # No way to trigger an error in a frozen module. + test_state_after_failure = None + + def test_unloadable(self): + assert self.machinery.FrozenImporter.find_module('_not_real') is None + with self.assertRaises(ImportError) as cm: + self.machinery.FrozenImporter.load_module('_not_real') + self.assertEqual(cm.exception.name, '_not_real') + + +(Frozen_LoaderTests, + Source_LoaderTests + ) = util.test_both(LoaderTests, machinery=machinery) + + +class InspectLoaderTests: + + """Tests for the InspectLoader methods for FrozenImporter.""" + + def test_get_code(self): + # Make sure that the code object is good. + name = '__hello__' + with captured_stdout() as stdout: + code = self.machinery.FrozenImporter.get_code(name) + mod = types.ModuleType(name) + exec(code, mod.__dict__) + self.assertTrue(hasattr(mod, 'initialized')) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + + def test_get_source(self): + # Should always return None. + result = self.machinery.FrozenImporter.get_source('__hello__') + self.assertIsNone(result) + + def test_is_package(self): + # Should be able to tell what is a package. + test_for = (('__hello__', False), ('__phello__', True), + ('__phello__.spam', False)) + for name, is_package in test_for: + result = self.machinery.FrozenImporter.is_package(name) + self.assertEqual(bool(result), is_package) + + def test_failure(self): + # Raise ImportError for modules that are not frozen. + for meth_name in ('get_code', 'get_source', 'is_package'): + method = getattr(self.machinery.FrozenImporter, meth_name) + with self.assertRaises(ImportError) as cm: + method('importlib') + self.assertEqual(cm.exception.name, 'importlib') + +(Frozen_ILTests, + Source_ILTests + ) = util.test_both(InspectLoaderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/__init__.py b/Lib/test/test_importlib/import_/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/import_/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/import_/__main__.py b/Lib/test/test_importlib/import_/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/import_/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py new file mode 100644 index 000000000..4b18093cf --- /dev/null +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -0,0 +1,75 @@ +from importlib import machinery +import sys +import types +import unittest + +from .. import util + + +class SpecLoaderMock: + + def find_spec(self, fullname, path=None, target=None): + return machinery.ModuleSpec(fullname, self) + + def create_module(self, spec): + return None + + def exec_module(self, module): + pass + + +class SpecLoaderAttributeTests: + + def test___loader__(self): + loader = SpecLoaderMock() + with util.uncache('blah'), util.import_state(meta_path=[loader]): + module = self.__import__('blah') + self.assertEqual(loader, module.__loader__) + + +(Frozen_SpecTests, + Source_SpecTests + ) = util.test_both(SpecLoaderAttributeTests, __import__=util.__import__) + + +class LoaderMock: + + def find_module(self, fullname, path=None): + return self + + def load_module(self, fullname): + sys.modules[fullname] = self.module + return self.module + + +class LoaderAttributeTests: + + def test___loader___missing(self): + module = types.ModuleType('blah') + try: + del module.__loader__ + except AttributeError: + pass + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + module = self.__import__('blah') + self.assertEqual(loader, module.__loader__) + + def test___loader___is_None(self): + module = types.ModuleType('blah') + module.__loader__ = None + loader = LoaderMock() + loader.module = module + with util.uncache('blah'), util.import_state(meta_path=[loader]): + returned_module = self.__import__('blah') + self.assertEqual(loader, module.__loader__) + + +(Frozen_Tests, + Source_Tests + ) = util.test_both(LoaderAttributeTests, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py new file mode 100644 index 000000000..761b256b3 --- /dev/null +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -0,0 +1,163 @@ +"""PEP 366 ("Main module explicit relative imports") specifies the +semantics for the __package__ attribute on modules. This attribute is +used, when available, to detect which package a module belongs to (instead +of using the typical __path__/__name__ test). + +""" +import unittest +import warnings +from .. import util + + +class Using__package__: + + """Use of __package__ supersedes the use of __name__/__path__ to calculate + what package a module belongs to. The basic algorithm is [__package__]:: + + def resolve_name(name, package, level): + level -= 1 + base = package.rsplit('.', level)[0] + return '{0}.{1}'.format(base, name) + + But since there is no guarantee that __package__ has been set (or not been + set to None [None]), there has to be a way to calculate the attribute's value + [__name__]:: + + def calc_package(caller_name, has___path__): + if has__path__: + return caller_name + else: + return caller_name.rsplit('.', 1)[0] + + Then the normal algorithm for relative name imports can proceed as if + __package__ had been set. + + """ + + def import_module(self, globals_): + with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: + with util.import_state(meta_path=[importer]): + self.__import__('pkg.fake') + module = self.__import__('', + globals=globals_, + fromlist=['attr'], level=2) + return module + + def test_using___package__(self): + # [__package__] + module = self.import_module({'__package__': 'pkg.fake'}) + self.assertEqual(module.__name__, 'pkg') + + def test_using___name__(self): + # [__name__] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({'__name__': 'pkg.fake', + '__path__': []}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_using___name__(self): + with self.assertWarns(ImportWarning): + self.import_module({'__name__': 'pkg.fake', '__path__': []}) + + def test_None_as___package__(self): + # [None] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + self.assertEqual(module.__name__, 'pkg') + + def test_spec_fallback(self): + # If __package__ isn't defined, fall back on __spec__.parent. + module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_package_and_spec_disagree(self): + # Raise an ImportWarning if __package__ != __spec__.parent. + with self.assertWarns(ImportWarning): + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) + + def test_bad__package__(self): + globals = {'__package__': ''} + with self.assertRaises(ModuleNotFoundError): + self.__import__('', globals, {}, ['relimport'], 1) + + def test_bunk__package__(self): + globals = {'__package__': 42} + with self.assertRaises(TypeError): + self.__import__('', globals, {}, ['relimport'], 1) + + +class FakeSpec: + def __init__(self, parent): + self.parent = parent + + +class Using__package__PEP302(Using__package__): + mock_modules = util.mock_modules + + +(Frozen_UsingPackagePEP302, + Source_UsingPackagePEP302 + ) = util.test_both(Using__package__PEP302, __import__=util.__import__) + + +class Using__package__PEP451(Using__package__): + mock_modules = util.mock_spec + + +(Frozen_UsingPackagePEP451, + Source_UsingPackagePEP451 + ) = util.test_both(Using__package__PEP451, __import__=util.__import__) + + +class Setting__package__: + + """Because __package__ is a new feature, it is not always set by a loader. + Import will set it as needed to help with the transition to relying on + __package__. + + For a top-level module, __package__ is set to None [top-level]. For a + package __name__ is used for __package__ [package]. For submodules the + value is __name__.rsplit('.', 1)[0] [submodule]. + + """ + + __import__ = util.__import__['Source'] + + # [top-level] + def test_top_level(self): + with self.mock_modules('top_level') as mock: + with util.import_state(meta_path=[mock]): + del mock['top_level'].__package__ + module = self.__import__('top_level') + self.assertEqual(module.__package__, '') + + # [package] + def test_package(self): + with self.mock_modules('pkg.__init__') as mock: + with util.import_state(meta_path=[mock]): + del mock['pkg'].__package__ + module = self.__import__('pkg') + self.assertEqual(module.__package__, 'pkg') + + # [submodule] + def test_submodule(self): + with self.mock_modules('pkg.__init__', 'pkg.mod') as mock: + with util.import_state(meta_path=[mock]): + del mock['pkg.mod'].__package__ + pkg = self.__import__('pkg.mod') + module = getattr(pkg, 'mod') + self.assertEqual(module.__package__, 'pkg') + +class Setting__package__PEP302(Setting__package__, unittest.TestCase): + mock_modules = util.mock_modules + +class Setting__package__PEP451(Setting__package__, unittest.TestCase): + mock_modules = util.mock_spec + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py new file mode 100644 index 000000000..0cd9de4da --- /dev/null +++ b/Lib/test/test_importlib/import_/test_api.py @@ -0,0 +1,119 @@ +from .. import util + +from importlib import machinery +import sys +import types +import unittest + +PKG_NAME = 'fine' +SUBMOD_NAME = 'fine.bogus' + + +class BadSpecFinderLoader: + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if fullname == SUBMOD_NAME: + spec = machinery.ModuleSpec(fullname, cls) + return spec + + @staticmethod + def create_module(spec): + return None + + @staticmethod + def exec_module(module): + if module.__name__ == SUBMOD_NAME: + raise ImportError('I cannot be loaded!') + + +class BadLoaderFinder: + @classmethod + def find_module(cls, fullname, path): + if fullname == SUBMOD_NAME: + return cls + + @classmethod + def load_module(cls, fullname): + if fullname == SUBMOD_NAME: + raise ImportError('I cannot be loaded!') + + +class APITest: + + """Test API-specific details for __import__ (e.g. raising the right + exception when passing in an int for the module name).""" + + def test_raises_ModuleNotFoundError(self): + with self.assertRaises(ModuleNotFoundError): + util.import_importlib('some module that does not exist') + + def test_name_requires_rparition(self): + # Raise TypeError if a non-string is passed in for the module name. + with self.assertRaises(TypeError): + self.__import__(42) + + def test_negative_level(self): + # Raise ValueError when a negative level is specified. + # PEP 328 did away with sys.module None entries and the ambiguity of + # absolute/relative imports. + with self.assertRaises(ValueError): + self.__import__('os', globals(), level=-1) + + def test_nonexistent_fromlist_entry(self): + # If something in fromlist doesn't exist, that's okay. + # issue15715 + mod = types.ModuleType(PKG_NAME) + mod.__path__ = ['XXX'] + with util.import_state(meta_path=[self.bad_finder_loader]): + with util.uncache(PKG_NAME): + sys.modules[PKG_NAME] = mod + self.__import__(PKG_NAME, fromlist=['not here']) + + def test_fromlist_load_error_propagates(self): + # If something in fromlist triggers an exception not related to not + # existing, let that exception propagate. + # issue15316 + mod = types.ModuleType(PKG_NAME) + mod.__path__ = ['XXX'] + with util.import_state(meta_path=[self.bad_finder_loader]): + with util.uncache(PKG_NAME): + sys.modules[PKG_NAME] = mod + with self.assertRaises(ImportError): + self.__import__(PKG_NAME, + fromlist=[SUBMOD_NAME.rpartition('.')[-1]]) + + def test_blocked_fromlist(self): + # If fromlist entry is None, let a ModuleNotFoundError propagate. + # issue31642 + mod = types.ModuleType(PKG_NAME) + mod.__path__ = [] + with util.import_state(meta_path=[self.bad_finder_loader]): + with util.uncache(PKG_NAME, SUBMOD_NAME): + sys.modules[PKG_NAME] = mod + sys.modules[SUBMOD_NAME] = None + with self.assertRaises(ModuleNotFoundError) as cm: + self.__import__(PKG_NAME, + fromlist=[SUBMOD_NAME.rpartition('.')[-1]]) + self.assertEqual(cm.exception.name, SUBMOD_NAME) + + +class OldAPITests(APITest): + bad_finder_loader = BadLoaderFinder + + +(Frozen_OldAPITests, + Source_OldAPITests + ) = util.test_both(OldAPITests, __import__=util.__import__) + + +class SpecAPITests(APITest): + bad_finder_loader = BadSpecFinderLoader + + +(Frozen_SpecAPITests, + Source_SpecAPITests + ) = util.test_both(SpecAPITests, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py new file mode 100644 index 000000000..8079add5b --- /dev/null +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -0,0 +1,93 @@ +"""Test that sys.modules is used properly by import.""" +from .. import util +import sys +from types import MethodType +import unittest + + +class UseCache: + + """When it comes to sys.modules, import prefers it over anything else. + + Once a name has been resolved, sys.modules is checked to see if it contains + the module desired. If so, then it is returned [use cache]. If it is not + found, then the proper steps are taken to perform the import, but + sys.modules is still used to return the imported module (e.g., not what a + loader returns) [from cache on return]. This also applies to imports of + things contained within a package and thus get assigned as an attribute + [from cache to attribute] or pulled in thanks to a fromlist import + [from cache for fromlist]. But if sys.modules contains None then + ImportError is raised [None in cache]. + + """ + + def test_using_cache(self): + # [use cache] + module_to_use = "some module found!" + with util.uncache('some_module'): + sys.modules['some_module'] = module_to_use + module = self.__import__('some_module') + self.assertEqual(id(module_to_use), id(module)) + + def test_None_in_cache(self): + #[None in cache] + name = 'using_None' + with util.uncache(name): + sys.modules[name] = None + with self.assertRaises(ImportError) as cm: + self.__import__(name) + self.assertEqual(cm.exception.name, name) + + +(Frozen_UseCache, + Source_UseCache + ) = util.test_both(UseCache, __import__=util.__import__) + + +class ImportlibUseCache(UseCache, unittest.TestCase): + + # Pertinent only to PEP 302; exec_module() doesn't return a module. + + __import__ = util.__import__['Source'] + + def create_mock(self, *names, return_=None): + mock = util.mock_modules(*names) + original_load = mock.load_module + def load_module(self, fullname): + original_load(fullname) + return return_ + mock.load_module = MethodType(load_module, mock) + return mock + + # __import__ inconsistent between loaders and built-in import when it comes + # to when to use the module in sys.modules and when not to. + def test_using_cache_after_loader(self): + # [from cache on return] + with self.create_mock('module') as mock: + with util.import_state(meta_path=[mock]): + module = self.__import__('module') + self.assertEqual(id(module), id(sys.modules['module'])) + + # See test_using_cache_after_loader() for reasoning. + def test_using_cache_for_assigning_to_attribute(self): + # [from cache to attribute] + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg.module') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) + + # See test_using_cache_after_loader() for reasoning. + def test_using_cache_for_fromlist(self): + # [from cache for fromlist] + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['module']) + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py new file mode 100644 index 000000000..018c17217 --- /dev/null +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -0,0 +1,175 @@ +"""Test that the semantics relating to the 'fromlist' argument are correct.""" +from .. import util +import warnings +import unittest + + +class ReturnValue: + + """The use of fromlist influences what import returns. + + If direct ``import ...`` statement is used, the root module or package is + returned [import return]. But if fromlist is set, then the specified module + is actually returned (whether it is a relative import or not) + [from return]. + + """ + + def test_return_from_import(self): + # [import return] + with util.mock_spec('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg.module') + self.assertEqual(module.__name__, 'pkg') + + def test_return_from_from_import(self): + # [from return] + with util.mock_modules('pkg.__init__', 'pkg.module')as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg.module', fromlist=['attr']) + self.assertEqual(module.__name__, 'pkg.module') + + +(Frozen_ReturnValue, + Source_ReturnValue + ) = util.test_both(ReturnValue, __import__=util.__import__) + + +class HandlingFromlist: + + """Using fromlist triggers different actions based on what is being asked + of it. + + If fromlist specifies an object on a module, nothing special happens + [object case]. This is even true if the object does not exist [bad object]. + + If a package is being imported, then what is listed in fromlist may be + treated as a module to be imported [module]. And this extends to what is + contained in __all__ when '*' is imported [using *]. And '*' does not need + to be the only name in the fromlist [using * with others]. + + """ + + def test_object(self): + # [object case] + with util.mock_modules('module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('module', fromlist=['attr']) + self.assertEqual(module.__name__, 'module') + + def test_nonexistent_object(self): + # [bad object] + with util.mock_modules('module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('module', fromlist=['non_existent']) + self.assertEqual(module.__name__, 'module') + self.assertFalse(hasattr(module, 'non_existent')) + + def test_module_from_package(self): + # [module] + with util.mock_modules('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['module']) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.__name__, 'pkg.module') + + def test_nonexistent_from_package(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['non_existent']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, 'non_existent')) + + def test_module_from_package_triggers_ModuleNotFoundError(self): + # If a submodule causes an ModuleNotFoundError because it tries + # to import a module which doesn't exist, that should let the + # ModuleNotFoundError propagate. + def module_code(): + import i_do_not_exist + with util.mock_modules('pkg.__init__', 'pkg.mod', + module_code={'pkg.mod': module_code}) as importer: + with util.import_state(meta_path=[importer]): + with self.assertRaises(ModuleNotFoundError) as exc: + self.__import__('pkg', fromlist=['mod']) + self.assertEqual('i_do_not_exist', exc.exception.name) + + def test_empty_string(self): + with util.mock_modules('pkg.__init__', 'pkg.mod') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg.mod', fromlist=['']) + self.assertEqual(module.__name__, 'pkg.mod') + + def basic_star_test(self, fromlist=['*']): + # [using *] + with util.mock_modules('pkg.__init__', 'pkg.module') as mock: + with util.import_state(meta_path=[mock]): + mock['pkg'].__all__ = ['module'] + module = self.__import__('pkg', fromlist=fromlist) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.__name__, 'pkg.module') + + def test_using_star(self): + # [using *] + self.basic_star_test() + + def test_fromlist_as_tuple(self): + self.basic_star_test(('*',)) + + def test_star_with_others(self): + # [using * with others] + context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2') + with context as mock: + with util.import_state(meta_path=[mock]): + mock['pkg'].__all__ = ['module1'] + module = self.__import__('pkg', fromlist=['module2', '*']) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module1')) + self.assertTrue(hasattr(module, 'module2')) + self.assertEqual(module.module1.__name__, 'pkg.module1') + self.assertEqual(module.module2.__name__, 'pkg.module2') + + def test_nonexistent_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + importer['pkg'].__all__ = ['non_existent'] + module = self.__import__('pkg', fromlist=['*']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, 'non_existent')) + + def test_star_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + importer['pkg'].__all__ = ['*'] + module = self.__import__('pkg', fromlist=['*']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, '*')) + + def test_invalid_type(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]), \ + warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + with self.assertRaisesRegex(TypeError, r'\bfrom\b'): + self.__import__('pkg', fromlist=[b'attr']) + with self.assertRaisesRegex(TypeError, r'\bfrom\b'): + self.__import__('pkg', fromlist=iter([b'attr'])) + + def test_invalid_type_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]), \ + warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + importer['pkg'].__all__ = [b'attr'] + with self.assertRaisesRegex(TypeError, r'\bpkg\.__all__\b'): + self.__import__('pkg', fromlist=['*']) + + +(Frozen_FromList, + Source_FromList + ) = util.test_both(HandlingFromlist, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py new file mode 100644 index 000000000..5a41e8968 --- /dev/null +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -0,0 +1,125 @@ +from .. import util +import importlib._bootstrap +import sys +from types import MethodType +import unittest +import warnings + + +class CallingOrder: + + """Calls to the importers on sys.meta_path happen in order that they are + specified in the sequence, starting with the first importer + [first called], and then continuing on down until one is found that doesn't + return None [continuing].""" + + + def test_first_called(self): + # [first called] + mod = 'top_level' + with util.mock_spec(mod) as first, util.mock_spec(mod) as second: + with util.import_state(meta_path=[first, second]): + self.assertIs(self.__import__(mod), first.modules[mod]) + + def test_continuing(self): + # [continuing] + mod_name = 'for_real' + with util.mock_spec('nonexistent') as first, \ + util.mock_spec(mod_name) as second: + first.find_spec = lambda self, fullname, path=None, parent=None: None + with util.import_state(meta_path=[first, second]): + self.assertIs(self.__import__(mod_name), second.modules[mod_name]) + + def test_empty(self): + # Raise an ImportWarning if sys.meta_path is empty. + module_name = 'nothing' + try: + del sys.modules[module_name] + except KeyError: + pass + with util.import_state(meta_path=[]): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIsNone(importlib._bootstrap._find_spec('nothing', + None)) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, ImportWarning)) + + +(Frozen_CallingOrder, + Source_CallingOrder + ) = util.test_both(CallingOrder, __import__=util.__import__) + + +class CallSignature: + + """If there is no __path__ entry on the parent module, then 'path' is None + [no path]. Otherwise, the value for __path__ is passed in for the 'path' + argument [path set].""" + + def log_finder(self, importer): + fxn = getattr(importer, self.finder_name) + log = [] + def wrapper(self, *args, **kwargs): + log.append([args, kwargs]) + return fxn(*args, **kwargs) + return log, wrapper + + def test_no_path(self): + # [no path] + mod_name = 'top_level' + assert '.' not in mod_name + with self.mock_modules(mod_name) as importer: + log, wrapped_call = self.log_finder(importer) + setattr(importer, self.finder_name, MethodType(wrapped_call, importer)) + with util.import_state(meta_path=[importer]): + self.__import__(mod_name) + assert len(log) == 1 + args = log[0][0] + # Assuming all arguments are positional. + self.assertEqual(args[0], mod_name) + self.assertIsNone(args[1]) + + def test_with_path(self): + # [path set] + pkg_name = 'pkg' + mod_name = pkg_name + '.module' + path = [42] + assert '.' in mod_name + with self.mock_modules(pkg_name+'.__init__', mod_name) as importer: + importer.modules[pkg_name].__path__ = path + log, wrapped_call = self.log_finder(importer) + setattr(importer, self.finder_name, MethodType(wrapped_call, importer)) + with util.import_state(meta_path=[importer]): + self.__import__(mod_name) + assert len(log) == 2 + args = log[1][0] + kwargs = log[1][1] + # Assuming all arguments are positional. + self.assertFalse(kwargs) + self.assertEqual(args[0], mod_name) + self.assertIs(args[1], path) + + +class CallSignaturePEP302(CallSignature): + mock_modules = util.mock_modules + finder_name = 'find_module' + + +(Frozen_CallSignaturePEP302, + Source_CallSignaturePEP302 + ) = util.test_both(CallSignaturePEP302, __import__=util.__import__) + + +class CallSignaturePEP451(CallSignature): + mock_modules = util.mock_spec + finder_name = 'find_spec' + + +(Frozen_CallSignaturePEP451, + Source_CallSignaturePEP451 + ) = util.test_both(CallSignaturePEP451, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py new file mode 100644 index 000000000..24396044a --- /dev/null +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -0,0 +1,110 @@ +from .. import util +import sys +import unittest +from test import support + + +class ParentModuleTests: + + """Importing a submodule should import the parent modules.""" + + def test_import_parent(self): + with util.mock_spec('pkg.__init__', 'pkg.module') as mock: + with util.import_state(meta_path=[mock]): + module = self.__import__('pkg.module') + self.assertIn('pkg', sys.modules) + + def test_bad_parent(self): + with util.mock_spec('pkg.module') as mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises(ImportError) as cm: + self.__import__('pkg.module') + self.assertEqual(cm.exception.name, 'pkg') + + def test_raising_parent_after_importing_child(self): + def __init__(): + import pkg.module + 1/0 + mock = util.mock_spec('pkg.__init__', 'pkg.module', + module_code={'pkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises(ZeroDivisionError): + self.__import__('pkg') + self.assertNotIn('pkg', sys.modules) + self.assertIn('pkg.module', sys.modules) + with self.assertRaises(ZeroDivisionError): + self.__import__('pkg.module') + self.assertNotIn('pkg', sys.modules) + self.assertIn('pkg.module', sys.modules) + + def test_raising_parent_after_relative_importing_child(self): + def __init__(): + from . import module + 1/0 + mock = util.mock_spec('pkg.__init__', 'pkg.module', + module_code={'pkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises((ZeroDivisionError, ImportError)): + # This raises ImportError on the "from . import module" + # line, not sure why. + self.__import__('pkg') + self.assertNotIn('pkg', sys.modules) + with self.assertRaises((ZeroDivisionError, ImportError)): + self.__import__('pkg.module') + self.assertNotIn('pkg', sys.modules) + # XXX False + #self.assertIn('pkg.module', sys.modules) + + def test_raising_parent_after_double_relative_importing_child(self): + def __init__(): + from ..subpkg import module + 1/0 + mock = util.mock_spec('pkg.__init__', 'pkg.subpkg.__init__', + 'pkg.subpkg.module', + module_code={'pkg.subpkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises((ZeroDivisionError, ImportError)): + # This raises ImportError on the "from ..subpkg import module" + # line, not sure why. + self.__import__('pkg.subpkg') + self.assertNotIn('pkg.subpkg', sys.modules) + with self.assertRaises((ZeroDivisionError, ImportError)): + self.__import__('pkg.subpkg.module') + self.assertNotIn('pkg.subpkg', sys.modules) + # XXX False + #self.assertIn('pkg.subpkg.module', sys.modules) + + def test_module_not_package(self): + # Try to import a submodule from a non-package should raise ImportError. + assert not hasattr(sys, '__path__') + with self.assertRaises(ImportError) as cm: + self.__import__('sys.no_submodules_here') + self.assertEqual(cm.exception.name, 'sys.no_submodules_here') + + def test_module_not_package_but_side_effects(self): + # If a module injects something into sys.modules as a side-effect, then + # pick up on that fact. + name = 'mod' + subname = name + '.b' + def module_injection(): + sys.modules[subname] = 'total bunk' + mock_spec = util.mock_spec('mod', + module_code={'mod': module_injection}) + with mock_spec as mock: + with util.import_state(meta_path=[mock]): + try: + submodule = self.__import__(subname) + finally: + support.unload(subname) + + +(Frozen_ParentTests, + Source_ParentTests + ) = util.test_both(ParentModuleTests, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py new file mode 100644 index 000000000..18c81dda4 --- /dev/null +++ b/Lib/test/test_importlib/import_/test_path.py @@ -0,0 +1,278 @@ +from .. import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + +import os +import sys +import tempfile +from types import ModuleType +import unittest +import warnings +import zipimport + + +class FinderTests: + + """Tests for PathFinder.""" + + find = None + check_found = None + + def test_failure(self): + # Test None returned upon not finding a suitable loader. + module = '' + with util.import_state(): + self.assertIsNone(self.find(module)) + + def test_sys_path(self): + # Test that sys.path is used when 'path' is None. + # Implicitly tests that sys.path_importer_cache is used. + module = '' + path = '' + importer = util.mock_spec(module) + with util.import_state(path_importer_cache={path: importer}, + path=[path]): + found = self.find(module) + self.check_found(found, importer) + + def test_path(self): + # Test that 'path' is used when set. + # Implicitly tests that sys.path_importer_cache is used. + module = '' + path = '' + importer = util.mock_spec(module) + with util.import_state(path_importer_cache={path: importer}): + found = self.find(module, [path]) + self.check_found(found, importer) + + def test_empty_list(self): + # An empty list should not count as asking for sys.path. + module = 'module' + path = '' + importer = util.mock_spec(module) + with util.import_state(path_importer_cache={path: importer}, + path=[path]): + self.assertIsNone(self.find('module', [])) + + def test_path_hooks(self): + # Test that sys.path_hooks is used. + # Test that sys.path_importer_cache is set. + module = '' + path = '' + importer = util.mock_spec(module) + hook = util.mock_path_hook(path, importer=importer) + with util.import_state(path_hooks=[hook]): + found = self.find(module, [path]) + self.check_found(found, importer) + self.assertIn(path, sys.path_importer_cache) + self.assertIs(sys.path_importer_cache[path], importer) + + def test_empty_path_hooks(self): + # Test that if sys.path_hooks is empty a warning is raised, + # sys.path_importer_cache gets None set, and PathFinder returns None. + path_entry = 'bogus_path' + with util.import_state(path_importer_cache={}, path_hooks=[], + path=[path_entry]): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIsNone(self.find('os')) + self.assertIsNone(sys.path_importer_cache[path_entry]) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, ImportWarning)) + + def test_path_importer_cache_empty_string(self): + # The empty string should create a finder using the cwd. + path = '' + module = '' + importer = util.mock_spec(module) + hook = util.mock_path_hook(os.getcwd(), importer=importer) + with util.import_state(path=[path], path_hooks=[hook]): + found = self.find(module) + self.check_found(found, importer) + self.assertIn(os.getcwd(), sys.path_importer_cache) + + def test_None_on_sys_path(self): + # Putting None in sys.path[0] caused an import regression from Python + # 3.2: http://bugs.python.org/issue16514 + new_path = sys.path[:] + new_path.insert(0, None) + new_path_importer_cache = sys.path_importer_cache.copy() + new_path_importer_cache.pop(None, None) + new_path_hooks = [zipimport.zipimporter, + self.machinery.FileFinder.path_hook( + *self.importlib._bootstrap_external._get_supported_file_loaders())] + missing = object() + email = sys.modules.pop('email', missing) + try: + with util.import_state(meta_path=sys.meta_path[:], + path=new_path, + path_importer_cache=new_path_importer_cache, + path_hooks=new_path_hooks): + module = self.importlib.import_module('email') + self.assertIsInstance(module, ModuleType) + finally: + if email is not missing: + sys.modules['email'] = email + + def test_finder_with_find_module(self): + class TestFinder: + def find_module(self, fullname): + return self.to_return + failing_finder = TestFinder() + failing_finder.to_return = None + path = 'testing path' + with util.import_state(path_importer_cache={path: failing_finder}): + self.assertIsNone( + self.machinery.PathFinder.find_spec('whatever', [path])) + success_finder = TestFinder() + success_finder.to_return = __loader__ + with util.import_state(path_importer_cache={path: success_finder}): + spec = self.machinery.PathFinder.find_spec('whatever', [path]) + self.assertEqual(spec.loader, __loader__) + + def test_finder_with_find_loader(self): + class TestFinder: + loader = None + portions = [] + def find_loader(self, fullname): + return self.loader, self.portions + path = 'testing path' + with util.import_state(path_importer_cache={path: TestFinder()}): + self.assertIsNone( + self.machinery.PathFinder.find_spec('whatever', [path])) + success_finder = TestFinder() + success_finder.loader = __loader__ + with util.import_state(path_importer_cache={path: success_finder}): + spec = self.machinery.PathFinder.find_spec('whatever', [path]) + self.assertEqual(spec.loader, __loader__) + + def test_finder_with_find_spec(self): + class TestFinder: + spec = None + def find_spec(self, fullname, target=None): + return self.spec + path = 'testing path' + with util.import_state(path_importer_cache={path: TestFinder()}): + self.assertIsNone( + self.machinery.PathFinder.find_spec('whatever', [path])) + success_finder = TestFinder() + success_finder.spec = self.machinery.ModuleSpec('whatever', __loader__) + with util.import_state(path_importer_cache={path: success_finder}): + got = self.machinery.PathFinder.find_spec('whatever', [path]) + self.assertEqual(got, success_finder.spec) + + def test_deleted_cwd(self): + # Issue #22834 + old_dir = os.getcwd() + self.addCleanup(os.chdir, old_dir) + new_dir = tempfile.mkdtemp() + try: + os.chdir(new_dir) + try: + os.rmdir(new_dir) + except OSError: + # EINVAL on Solaris, EBUSY on AIX, ENOTEMPTY on Windows + self.skipTest("platform does not allow " + "the deletion of the cwd") + except: + os.chdir(old_dir) + os.rmdir(new_dir) + raise + + with util.import_state(path=['']): + # Do not want FileNotFoundError raised. + self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + + def test_invalidate_caches_finders(self): + # Finders with an invalidate_caches() method have it called. + class FakeFinder: + def __init__(self): + self.called = False + + def invalidate_caches(self): + self.called = True + + cache = {'leave_alone': object(), 'finder_to_invalidate': FakeFinder()} + with util.import_state(path_importer_cache=cache): + self.machinery.PathFinder.invalidate_caches() + self.assertTrue(cache['finder_to_invalidate'].called) + + def test_invalidate_caches_clear_out_None(self): + # Clear out None in sys.path_importer_cache() when invalidating caches. + cache = {'clear_out': None} + with util.import_state(path_importer_cache=cache): + self.machinery.PathFinder.invalidate_caches() + self.assertEqual(len(cache), 0) + + +class FindModuleTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_module(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found, importer) + + +(Frozen_FindModuleTests, + Source_FindModuleTests +) = util.test_both(FindModuleTests, importlib=importlib, machinery=machinery) + + +class FindSpecTests(FinderTests): + def find(self, *args, **kwargs): + return self.machinery.PathFinder.find_spec(*args, **kwargs) + def check_found(self, found, importer): + self.assertIs(found.loader, importer) + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, importlib=importlib, machinery=machinery) + + +class PathEntryFinderTests: + + def test_finder_with_failing_find_spec(self): + # PathEntryFinder with find_module() defined should work. + # Issue #20763. + class Finder: + path_location = 'test_finder_with_find_module' + def __init__(self, path): + if path != self.path_location: + raise ImportError + + @staticmethod + def find_module(fullname): + return None + + + with util.import_state(path=[Finder.path_location]+sys.path[:], + path_hooks=[Finder]): + self.machinery.PathFinder.find_spec('importlib') + + def test_finder_with_failing_find_module(self): + # PathEntryFinder with find_module() defined should work. + # Issue #20763. + class Finder: + path_location = 'test_finder_with_find_module' + def __init__(self, path): + if path != self.path_location: + raise ImportError + + @staticmethod + def find_module(fullname): + return None + + + with util.import_state(path=[Finder.path_location]+sys.path[:], + path_hooks=[Finder]): + self.machinery.PathFinder.find_module('importlib') + + +(Frozen_PEFTests, + Source_PEFTests + ) = util.test_both(PathEntryFinderTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py new file mode 100644 index 000000000..8a95a3210 --- /dev/null +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -0,0 +1,232 @@ +"""Test relative imports (PEP 328).""" +from .. import util +import unittest +import warnings + + +class RelativeImports: + + """PEP 328 introduced relative imports. This allows for imports to occur + from within a package without having to specify the actual package name. + + A simple example is to import another module within the same package + [module from module]:: + + # From pkg.mod1 with pkg.mod2 being a module. + from . import mod2 + + This also works for getting an attribute from a module that is specified + in a relative fashion [attr from module]:: + + # From pkg.mod1. + from .mod2 import attr + + But this is in no way restricted to working between modules; it works + from [package to module],:: + + # From pkg, importing pkg.module which is a module. + from . import module + + [module to package],:: + + # Pull attr from pkg, called from pkg.module which is a module. + from . import attr + + and [package to package]:: + + # From pkg.subpkg1 (both pkg.subpkg[1,2] are packages). + from .. import subpkg2 + + The number of dots used is in no way restricted [deep import]:: + + # Import pkg.attr from pkg.pkg1.pkg2.pkg3.pkg4.pkg5. + from ...... import attr + + To prevent someone from accessing code that is outside of a package, one + cannot reach the location containing the root package itself:: + + # From pkg.__init__ [too high from package] + from .. import top_level + + # From pkg.module [too high from module] + from .. import top_level + + Relative imports are the only type of import that allow for an empty + module name for an import [empty name]. + + """ + + def relative_import_test(self, create, globals_, callback): + """Abstract out boilerplace for setting up for an import test.""" + uncache_names = [] + for name in create: + if not name.endswith('.__init__'): + uncache_names.append(name) + else: + uncache_names.append(name[:-len('.__init__')]) + with util.mock_spec(*create) as importer: + with util.import_state(meta_path=[importer]): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) + + + def test_module_from_module(self): + # [module from module] + create = 'pkg.__init__', 'pkg.mod2' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'} + def callback(global_): + self.__import__('pkg') # For __import__(). + module = self.__import__('', global_, fromlist=['mod2'], level=1) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'mod2')) + self.assertEqual(module.mod2.attr, 'pkg.mod2') + self.relative_import_test(create, globals_, callback) + + def test_attr_from_module(self): + # [attr from module] + create = 'pkg.__init__', 'pkg.mod2' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'} + def callback(global_): + self.__import__('pkg') # For __import__(). + module = self.__import__('mod2', global_, fromlist=['attr'], + level=1) + self.assertEqual(module.__name__, 'pkg.mod2') + self.assertEqual(module.attr, 'pkg.mod2') + self.relative_import_test(create, globals_, callback) + + def test_package_to_module(self): + # [package to module] + create = 'pkg.__init__', 'pkg.module' + globals_ = ({'__package__': 'pkg'}, + {'__name__': 'pkg', '__path__': ['blah']}) + def callback(global_): + self.__import__('pkg') # For __import__(). + module = self.__import__('', global_, fromlist=['module'], + level=1) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.attr, 'pkg.module') + self.relative_import_test(create, globals_, callback) + + def test_module_to_package(self): + # [module to package] + create = 'pkg.__init__', 'pkg.module' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} + def callback(global_): + self.__import__('pkg') # For __import__(). + module = self.__import__('', global_, fromlist=['attr'], level=1) + self.assertEqual(module.__name__, 'pkg') + self.relative_import_test(create, globals_, callback) + + def test_package_to_package(self): + # [package to package] + create = ('pkg.__init__', 'pkg.subpkg1.__init__', + 'pkg.subpkg2.__init__') + globals_ = ({'__package__': 'pkg.subpkg1'}, + {'__name__': 'pkg.subpkg1', '__path__': ['blah']}) + def callback(global_): + module = self.__import__('', global_, fromlist=['subpkg2'], + level=2) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'subpkg2')) + self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') + + def test_deep_import(self): + # [deep import] + create = ['pkg.__init__'] + for count in range(1,6): + create.append('{0}.pkg{1}.__init__'.format( + create[-1][:-len('.__init__')], count)) + globals_ = ({'__package__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5'}, + {'__name__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5', + '__path__': ['blah']}) + def callback(global_): + self.__import__(globals_[0]['__package__']) + module = self.__import__('', global_, fromlist=['attr'], level=6) + self.assertEqual(module.__name__, 'pkg') + self.relative_import_test(create, globals_, callback) + + def test_too_high_from_package(self): + # [too high from package] + create = ['top_level', 'pkg.__init__'] + globals_ = ({'__package__': 'pkg'}, + {'__name__': 'pkg', '__path__': ['blah']}) + def callback(global_): + self.__import__('pkg') + with self.assertRaises(ValueError): + self.__import__('', global_, fromlist=['top_level'], + level=2) + self.relative_import_test(create, globals_, callback) + + def test_too_high_from_module(self): + # [too high from module] + create = ['top_level', 'pkg.__init__', 'pkg.module'] + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} + def callback(global_): + self.__import__('pkg') + with self.assertRaises(ValueError): + self.__import__('', global_, fromlist=['top_level'], + level=2) + self.relative_import_test(create, globals_, callback) + + def test_empty_name_w_level_0(self): + # [empty name] + with self.assertRaises(ValueError): + self.__import__('') + + def test_import_from_different_package(self): + # Test importing from a different package than the caller. + # in pkg.subpkg1.mod + # from ..subpkg2 import mod + create = ['__runpy_pkg__.__init__', + '__runpy_pkg__.__runpy_pkg__.__init__', + '__runpy_pkg__.uncle.__init__', + '__runpy_pkg__.uncle.cousin.__init__', + '__runpy_pkg__.uncle.cousin.nephew'] + globals_ = {'__package__': '__runpy_pkg__.__runpy_pkg__'} + def callback(global_): + self.__import__('__runpy_pkg__.__runpy_pkg__') + module = self.__import__('uncle.cousin', globals_, {}, + fromlist=['nephew'], + level=2) + self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin') + self.relative_import_test(create, globals_, callback) + + def test_import_relative_import_no_fromlist(self): + # Import a relative module w/ no fromlist. + create = ['crash.__init__', 'crash.mod'] + globals_ = [{'__package__': 'crash', '__name__': 'crash'}] + def callback(global_): + self.__import__('crash') + mod = self.__import__('mod', global_, {}, [], 1) + self.assertEqual(mod.__name__, 'crash.mod') + self.relative_import_test(create, globals_, callback) + + def test_relative_import_no_globals(self): + # No globals for a relative import is an error. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with self.assertRaises(KeyError): + self.__import__('sys', level=1) + + def test_relative_import_no_package(self): + with self.assertRaises(ImportError): + self.__import__('a', {'__package__': '', '__spec__': None}, + level=1) + + def test_relative_import_no_package_exists_absolute(self): + with self.assertRaises(ImportError): + self.__import__('sys', {'__package__': '', '__spec__': None}, + level=1) + + +(Frozen_RelativeImports, + Source_RelativeImports + ) = util.test_both(RelativeImports, __import__=util.__import__) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py new file mode 100644 index 000000000..3080f6f8f --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py @@ -0,0 +1 @@ +attr = 'both_portions foo one' diff --git a/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py new file mode 100644 index 000000000..4131d3d4b --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py @@ -0,0 +1 @@ +attr = 'both_portions foo two' diff --git a/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip b/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip new file mode 100644 index 0000000000000000000000000000000000000000..836a9106bcdd5d171dcf54386bb51e0f7b6a520c GIT binary patch literal 515 zcmWIWW@h1H0D)bqOB}!qD8a%Y!;q9%q#qi>$-rzseX)NM5SLbPGcdBeU}j(d69Fi? zUGA0cQwHh=VJ@idlJb1Lf=ajG`=B3A{WEQ0+m*f{!Dx~G- zD**MYb0MtR19arK*^@P-fL4Go$QqzB{rtRCm^CgyE1Ym!Q;=U&l9``p2sRz44P-ha zlN>XyK#&010s;aIe;q+IBuH2xL4pw+s3xHX2*e~tpcz{l?Vu*Xf`_nqm_A1~Zz<3` b`_$-w-Wd$GUjw8j3V z72FJrEH9WD7{EjT+$43?B`79|U@<8zKObV67SJR+LZ*G2Jy}Bv=q?DxbQj39{Jd1X zf=aLpL1wCBnweNqQlwz3pbqnn0#KU*P@6hefHxzP95b$PkbsBCUq=uVGgMe1p@I=2 z2vae`1ld$CET%$22xv0MHv~*Z4IhZfj6gHDG|s?bGAy)kna|1w)X2aLgaSZ=*g%Fc F003}~b~XS2 literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py new file mode 100644 index 000000000..d8f5c831f --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py @@ -0,0 +1 @@ +attr = 'portion1 foo one' diff --git a/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py new file mode 100644 index 000000000..d8f5c831f --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py @@ -0,0 +1 @@ +attr = 'portion1 foo one' diff --git a/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py new file mode 100644 index 000000000..d092e1e99 --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py @@ -0,0 +1 @@ +attr = 'portion2 foo two' diff --git a/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py new file mode 100644 index 000000000..2776fcdfd --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py @@ -0,0 +1 @@ +attr = 'parent child one' diff --git a/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py new file mode 100644 index 000000000..8b037bcb0 --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py @@ -0,0 +1 @@ +attr = 'parent child two' diff --git a/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py new file mode 100644 index 000000000..f8abfe1c1 --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py @@ -0,0 +1 @@ +attr = 'parent child three' diff --git a/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip b/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip new file mode 100644 index 0000000000000000000000000000000000000000..3b866c914ad2f2fe348405799a482235854bac10 GIT binary patch literal 332 zcmWIWW@h1H00B0&B@SQ)lwe_yVMxo**AEThWMDoseX;*TATF)oW?*D_!OXw_CIV1& zf15p7Lkg%Hgt?%)^Yc>m3M%1doWL|Av81F(!B#=NAit<2Ge6Hz0cfcLP@6hefHxzP z95XK0NkH8x!0^`*#Dut&72;M5ccPkv>QabFj6gHCG}=K;g1H-)d8}+8JDGrR7Lc9; H;xGUJj8#CG literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/source/__init__.py b/Lib/test/test_importlib/source/__init__.py new file mode 100644 index 000000000..4b16ecc31 --- /dev/null +++ b/Lib/test/test_importlib/source/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_importlib/source/__main__.py b/Lib/test/test_importlib/source/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_importlib/source/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py new file mode 100644 index 000000000..12ce0cb93 --- /dev/null +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -0,0 +1,85 @@ +"""Test case-sensitivity (PEP 235).""" +from .. import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + +import os +from test import support as test_support +import unittest + + +@util.case_insensitive_tests +class CaseSensitivityTest(util.CASEOKTestBase): + + """PEP 235 dictates that on case-preserving, case-insensitive file systems + that imports are case-sensitive unless the PYTHONCASEOK environment + variable is set.""" + + name = 'MoDuLe' + assert name != name.lower() + + def finder(self, path): + return self.machinery.FileFinder(path, + (self.machinery.SourceFileLoader, + self.machinery.SOURCE_SUFFIXES), + (self.machinery.SourcelessFileLoader, + self.machinery.BYTECODE_SUFFIXES)) + + def sensitivity_test(self): + """Look for a module with matching and non-matching sensitivity.""" + sensitive_pkg = 'sensitive.{0}'.format(self.name) + insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) + context = util.create_modules(insensitive_pkg, sensitive_pkg) + with context as mapping: + sensitive_path = os.path.join(mapping['.root'], 'sensitive') + insensitive_path = os.path.join(mapping['.root'], 'insensitive') + sensitive_finder = self.finder(sensitive_path) + insensitive_finder = self.finder(insensitive_path) + return self.find(sensitive_finder), self.find(insensitive_finder) + + def test_sensitive(self): + with test_support.EnvironmentVarGuard() as env: + env.unset('PYTHONCASEOK') + self.caseok_env_changed(should_exist=False) + sensitive, insensitive = self.sensitivity_test() + self.assertIsNotNone(sensitive) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertIsNone(insensitive) + + def test_insensitive(self): + with test_support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + self.caseok_env_changed(should_exist=True) + sensitive, insensitive = self.sensitivity_test() + self.assertIsNotNone(sensitive) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertIsNotNone(insensitive) + self.assertIn(self.name, insensitive.get_filename(self.name)) + + +class CaseSensitivityTestPEP302(CaseSensitivityTest): + def find(self, finder): + return finder.find_module(self.name) + + +(Frozen_CaseSensitivityTestPEP302, + Source_CaseSensitivityTestPEP302 + ) = util.test_both(CaseSensitivityTestPEP302, importlib=importlib, + machinery=machinery) + + +class CaseSensitivityTestPEP451(CaseSensitivityTest): + def find(self, finder): + found = finder.find_spec(self.name) + return found.loader if found is not None else found + + +(Frozen_CaseSensitivityTestPEP451, + Source_CaseSensitivityTestPEP451 + ) = util.test_both(CaseSensitivityTestPEP451, importlib=importlib, + machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py new file mode 100644 index 000000000..3ffb2aa1e --- /dev/null +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -0,0 +1,797 @@ +from .. import abc +from .. import util + +importlib = util.import_importlib('importlib') +importlib_abc = util.import_importlib('importlib.abc') +machinery = util.import_importlib('importlib.machinery') +importlib_util = util.import_importlib('importlib.util') + +import errno +import marshal +import os +import py_compile +import shutil +import stat +import sys +import types +import unittest +import warnings + +from test.support import make_legacy_pyc, unload + +from test.test_py_compile import without_source_date_epoch +from test.test_py_compile import SourceDateEpochTestMeta + + +class SimpleTest(abc.LoaderTests): + + """Should have no issue importing a source module [basic]. And if there is + a syntax error, it should raise a SyntaxError [syntax error]. + + """ + + def setUp(self): + self.name = 'spam' + self.filepath = os.path.join('ham', self.name + '.py') + self.loader = self.machinery.SourceFileLoader(self.name, self.filepath) + + def test_load_module_API(self): + class Tester(self.abc.FileLoader): + def get_source(self, _): return 'attr = 42' + def is_package(self, _): return False + + loader = Tester('blah', 'blah.py') + self.addCleanup(unload, 'blah') + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module() # Should not raise an exception. + + def test_get_filename_API(self): + # If fullname is not set then assume self.path is desired. + class Tester(self.abc.FileLoader): + def get_code(self, _): pass + def get_source(self, _): pass + def is_package(self, _): pass + def module_repr(self, _): pass + + path = 'some_path' + name = 'some_name' + loader = Tester(name, path) + self.assertEqual(path, loader.get_filename(name)) + self.assertEqual(path, loader.get_filename()) + self.assertEqual(path, loader.get_filename(None)) + with self.assertRaises(ImportError): + loader.get_filename(name + 'XXX') + + def test_equality(self): + other = self.machinery.SourceFileLoader(self.name, self.filepath) + self.assertEqual(self.loader, other) + + def test_inequality(self): + other = self.machinery.SourceFileLoader('_' + self.name, self.filepath) + self.assertNotEqual(self.loader, other) + + # [basic] + def test_module(self): + with util.create_modules('_temp') as mapping: + loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module('_temp') + self.assertIn('_temp', sys.modules) + check = {'__name__': '_temp', '__file__': mapping['_temp'], + '__package__': ''} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + def test_package(self): + with util.create_modules('_pkg.__init__') as mapping: + loader = self.machinery.SourceFileLoader('_pkg', + mapping['_pkg.__init__']) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module('_pkg') + self.assertIn('_pkg', sys.modules) + check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], + '__path__': [os.path.dirname(mapping['_pkg.__init__'])], + '__package__': '_pkg'} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + + def test_lacking_parent(self): + with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: + loader = self.machinery.SourceFileLoader('_pkg.mod', + mapping['_pkg.mod']) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module('_pkg.mod') + self.assertIn('_pkg.mod', sys.modules) + check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], + '__package__': '_pkg'} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + def fake_mtime(self, fxn): + """Fake mtime to always be higher than expected.""" + return lambda name: fxn(name) + 1 + + def test_module_reuse(self): + with util.create_modules('_temp') as mapping: + loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module('_temp') + module_id = id(module) + module_dict_id = id(module.__dict__) + with open(mapping['_temp'], 'w') as file: + file.write("testing_var = 42\n") + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module('_temp') + self.assertIn('testing_var', module.__dict__, + "'testing_var' not in " + "{0}".format(list(module.__dict__.keys()))) + self.assertEqual(module, sys.modules['_temp']) + self.assertEqual(id(module), module_id) + self.assertEqual(id(module.__dict__), module_dict_id) + + def test_state_after_failure(self): + # A failed reload should leave the original module intact. + attributes = ('__file__', '__path__', '__package__') + value = '' + name = '_temp' + with util.create_modules(name) as mapping: + orig_module = types.ModuleType(name) + for attr in attributes: + setattr(orig_module, attr, value) + with open(mapping[name], 'w') as file: + file.write('+++ bad syntax +++') + loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.exec_module(orig_module) + for attr in attributes: + self.assertEqual(getattr(orig_module, attr), value) + with self.assertRaises(SyntaxError): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + loader.load_module(name) + for attr in attributes: + self.assertEqual(getattr(orig_module, attr), value) + + # [syntax error] + def test_bad_syntax(self): + with util.create_modules('_temp') as mapping: + with open(mapping['_temp'], 'w') as file: + file.write('=') + loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + loader.load_module('_temp') + self.assertNotIn('_temp', sys.modules) + + def test_file_from_empty_string_dir(self): + # Loading a module found from an empty string entry on sys.path should + # not only work, but keep all attributes relative. + file_path = '_temp.py' + with open(file_path, 'w') as file: + file.write("# test file for importlib") + try: + with util.uncache('_temp'): + loader = self.machinery.SourceFileLoader('_temp', file_path) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + mod = loader.load_module('_temp') + self.assertEqual(file_path, mod.__file__) + self.assertEqual(self.util.cache_from_source(file_path), + mod.__cached__) + finally: + os.unlink(file_path) + pycache = os.path.dirname(self.util.cache_from_source(file_path)) + if os.path.exists(pycache): + shutil.rmtree(pycache) + + @util.writes_bytecode_files + def test_timestamp_overflow(self): + # When a modification timestamp is larger than 2**32, it should be + # truncated rather than raise an OverflowError. + with util.create_modules('_temp') as mapping: + source = mapping['_temp'] + compiled = self.util.cache_from_source(source) + with open(source, 'w') as f: + f.write("x = 5") + try: + os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) + except OverflowError: + self.skipTest("cannot set modification time to large integer") + except OSError as e: + if e.errno != getattr(errno, 'EOVERFLOW', None): + raise + self.skipTest("cannot set modification time to large integer ({})".format(e)) + loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) + # PEP 451 + module = types.ModuleType('_temp') + module.__spec__ = self.util.spec_from_loader('_temp', loader) + loader.exec_module(module) + self.assertEqual(module.x, 5) + self.assertTrue(os.path.exists(compiled)) + os.unlink(compiled) + # PEP 302 + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + mod = loader.load_module('_temp') + # Sanity checks. + self.assertEqual(mod.__cached__, compiled) + self.assertEqual(mod.x, 5) + # The pyc file was created. + self.assertTrue(os.path.exists(compiled)) + + def test_unloadable(self): + loader = self.machinery.SourceFileLoader('good name', {}) + module = types.ModuleType('bad name') + module.__spec__ = self.machinery.ModuleSpec('bad name', loader) + with self.assertRaises(ImportError): + loader.exec_module(module) + with self.assertRaises(ImportError): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + loader.load_module('bad name') + + @util.writes_bytecode_files + def test_checked_hash_based_pyc(self): + with util.create_modules('_temp') as mapping: + source = mapping['_temp'] + pyc = self.util.cache_from_source(source) + with open(source, 'wb') as fp: + fp.write(b'state = "old"') + os.utime(source, (50, 50)) + py_compile.compile( + source, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + loader = self.machinery.SourceFileLoader('_temp', source) + mod = types.ModuleType('_temp') + mod.__spec__ = self.util.spec_from_loader('_temp', loader) + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + # Write a new source with the same mtime and size as before. + with open(source, 'wb') as fp: + fp.write(b'state = "new"') + os.utime(source, (50, 50)) + loader.exec_module(mod) + self.assertEqual(mod.state, 'new') + with open(pyc, 'rb') as fp: + data = fp.read() + self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11) + self.assertEqual( + self.util.source_hash(b'state = "new"'), + data[8:16], + ) + + @util.writes_bytecode_files + def test_overridden_checked_hash_based_pyc(self): + with util.create_modules('_temp') as mapping, \ + unittest.mock.patch('_imp.check_hash_based_pycs', 'never'): + source = mapping['_temp'] + pyc = self.util.cache_from_source(source) + with open(source, 'wb') as fp: + fp.write(b'state = "old"') + os.utime(source, (50, 50)) + py_compile.compile( + source, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + loader = self.machinery.SourceFileLoader('_temp', source) + mod = types.ModuleType('_temp') + mod.__spec__ = self.util.spec_from_loader('_temp', loader) + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + # Write a new source with the same mtime and size as before. + with open(source, 'wb') as fp: + fp.write(b'state = "new"') + os.utime(source, (50, 50)) + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + + @util.writes_bytecode_files + def test_unchecked_hash_based_pyc(self): + with util.create_modules('_temp') as mapping: + source = mapping['_temp'] + pyc = self.util.cache_from_source(source) + with open(source, 'wb') as fp: + fp.write(b'state = "old"') + os.utime(source, (50, 50)) + py_compile.compile( + source, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + loader = self.machinery.SourceFileLoader('_temp', source) + mod = types.ModuleType('_temp') + mod.__spec__ = self.util.spec_from_loader('_temp', loader) + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + # Update the source file, which should be ignored. + with open(source, 'wb') as fp: + fp.write(b'state = "new"') + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + with open(pyc, 'rb') as fp: + data = fp.read() + self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1) + self.assertEqual( + self.util.source_hash(b'state = "old"'), + data[8:16], + ) + + @util.writes_bytecode_files + def test_overiden_unchecked_hash_based_pyc(self): + with util.create_modules('_temp') as mapping, \ + unittest.mock.patch('_imp.check_hash_based_pycs', 'always'): + source = mapping['_temp'] + pyc = self.util.cache_from_source(source) + with open(source, 'wb') as fp: + fp.write(b'state = "old"') + os.utime(source, (50, 50)) + py_compile.compile( + source, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + loader = self.machinery.SourceFileLoader('_temp', source) + mod = types.ModuleType('_temp') + mod.__spec__ = self.util.spec_from_loader('_temp', loader) + loader.exec_module(mod) + self.assertEqual(mod.state, 'old') + # Update the source file, which should be ignored. + with open(source, 'wb') as fp: + fp.write(b'state = "new"') + loader.exec_module(mod) + self.assertEqual(mod.state, 'new') + with open(pyc, 'rb') as fp: + data = fp.read() + self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1) + self.assertEqual( + self.util.source_hash(b'state = "new"'), + data[8:16], + ) + + +(Frozen_SimpleTest, + Source_SimpleTest + ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, + abc=importlib_abc, util=importlib_util) + + +class SourceDateEpochTestMeta(SourceDateEpochTestMeta, + type(Source_SimpleTest)): + pass + + +class SourceDateEpoch_SimpleTest(Source_SimpleTest, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=True): + pass + + +class BadBytecodeTest: + + def import_(self, file, module_name): + raise NotImplementedError + + def manipulate_bytecode(self, + name, mapping, manipulator, *, + del_source=False, + invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP): + """Manipulate the bytecode of a module by passing it into a callable + that returns what to use as the new bytecode.""" + try: + del sys.modules['_temp'] + except KeyError: + pass + py_compile.compile(mapping[name], invalidation_mode=invalidation_mode) + if not del_source: + bytecode_path = self.util.cache_from_source(mapping[name]) + else: + os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) + if manipulator: + with open(bytecode_path, 'rb') as file: + bc = file.read() + new_bc = manipulator(bc) + with open(bytecode_path, 'wb') as file: + if new_bc is not None: + file.write(new_bc) + return bytecode_path + + def _test_empty_file(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'', + del_source=del_source) + test('_temp', mapping, bc_path) + + @util.writes_bytecode_files + def _test_partial_magic(self, test, *, del_source=False): + # When their are less than 4 bytes to a .pyc, regenerate it if + # possible, else raise ImportError. + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:3], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_magic_only(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:4], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_flags(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:7], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_hash(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode( + '_temp', + mapping, + lambda bc: bc[:13], + del_source=del_source, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + test('_temp', mapping, bc_path) + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode( + '_temp', + mapping, + lambda bc: bc[:13], + del_source=del_source, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + test('_temp', mapping, bc_path) + + def _test_partial_timestamp(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:11], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_size(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:15], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_no_marshal(self, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:16], + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bc_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_non_code_marshal(self, *, del_source=False): + with util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:16] + marshal.dumps(b'abcd'), + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(ImportError) as cm: + self.import_(file_path, '_temp') + self.assertEqual(cm.exception.name, '_temp') + self.assertEqual(cm.exception.path, bytecode_path) + + def _test_bad_marshal(self, *, del_source=False): + with util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:16] + b'', + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_bad_magic(self, test, *, del_source=False): + with util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'\x00\x00\x00\x00' + bc[4:]) + test('_temp', mapping, bc_path) + + +class BadBytecodeTestPEP451(BadBytecodeTest): + + def import_(self, file, module_name): + loader = self.loader(module_name, file) + module = types.ModuleType(module_name) + module.__spec__ = self.util.spec_from_loader(module_name, loader) + loader.exec_module(module) + + +class BadBytecodeTestPEP302(BadBytecodeTest): + + def import_(self, file, module_name): + loader = self.loader(module_name, file) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = loader.load_module(module_name) + self.assertIn(module_name, sys.modules) + + +class SourceLoaderBadBytecodeTest: + + @classmethod + def setUpClass(cls): + cls.loader = cls.machinery.SourceFileLoader + + @util.writes_bytecode_files + def test_empty_file(self): + # When a .pyc is empty, regenerate it if possible, else raise + # ImportError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_empty_file(test) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_partial_magic(test) + + @util.writes_bytecode_files + def test_magic_only(self): + # When there is only the magic number, regenerate the .pyc if possible, + # else raise EOFError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_magic_only(test) + + @util.writes_bytecode_files + def test_bad_magic(self): + # When the magic number is different, the bytecode should be + # regenerated. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as bytecode_file: + self.assertEqual(bytecode_file.read(4), + self.util.MAGIC_NUMBER) + + self._test_bad_magic(test) + + @util.writes_bytecode_files + def test_partial_timestamp(self): + # When the timestamp is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_partial_timestamp(test) + + @util.writes_bytecode_files + def test_partial_flags(self): + # When the flags is partial, regenerate the .pyc, else raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_partial_flags(test) + + @util.writes_bytecode_files + def test_partial_hash(self): + # When the hash is partial, regenerate the .pyc, else raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_partial_hash(test) + + @util.writes_bytecode_files + def test_partial_size(self): + # When the size is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 16) + + self._test_partial_size(test) + + @util.writes_bytecode_files + def test_no_marshal(self): + # When there is only the magic number and timestamp, raise EOFError. + self._test_no_marshal() + + @util.writes_bytecode_files + def test_non_code_marshal(self): + self._test_non_code_marshal() + # XXX ImportError when sourceless + + # [bad marshal] + @util.writes_bytecode_files + def test_bad_marshal(self): + # Bad marshal data should raise a ValueError. + self._test_bad_marshal() + + # [bad timestamp] + @util.writes_bytecode_files + @without_source_date_epoch + def test_old_timestamp(self): + # When the timestamp is older than the source, bytecode should be + # regenerated. + zeros = b'\x00\x00\x00\x00' + with util.create_modules('_temp') as mapping: + py_compile.compile(mapping['_temp']) + bytecode_path = self.util.cache_from_source(mapping['_temp']) + with open(bytecode_path, 'r+b') as bytecode_file: + bytecode_file.seek(8) + bytecode_file.write(zeros) + self.import_(mapping['_temp'], '_temp') + source_mtime = os.path.getmtime(mapping['_temp']) + source_timestamp = self.importlib._pack_uint32(source_mtime) + with open(bytecode_path, 'rb') as bytecode_file: + bytecode_file.seek(8) + self.assertEqual(bytecode_file.read(4), source_timestamp) + + # [bytecode read-only] + @util.writes_bytecode_files + def test_read_only_bytecode(self): + # When bytecode is read-only but should be rewritten, fail silently. + with util.create_modules('_temp') as mapping: + # Create bytecode that will need to be re-created. + py_compile.compile(mapping['_temp']) + bytecode_path = self.util.cache_from_source(mapping['_temp']) + with open(bytecode_path, 'r+b') as bytecode_file: + bytecode_file.seek(0) + bytecode_file.write(b'\x00\x00\x00\x00') + # Make the bytecode read-only. + os.chmod(bytecode_path, + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + try: + # Should not raise OSError! + self.import_(mapping['_temp'], '_temp') + finally: + # Make writable for eventual clean-up. + os.chmod(bytecode_path, stat.S_IWUSR) + + +class SourceLoaderBadBytecodeTestPEP451( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): + pass + + +(Frozen_SourceBadBytecodePEP451, + Source_SourceBadBytecodePEP451 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + + +class SourceLoaderBadBytecodeTestPEP302( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): + pass + + +(Frozen_SourceBadBytecodePEP302, + Source_SourceBadBytecodePEP302 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + + +class SourcelessLoaderBadBytecodeTest: + + @classmethod + def setUpClass(cls): + cls.loader = cls.machinery.SourcelessFileLoader + + def test_empty_file(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError) as cm: + self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) + + self._test_empty_file(test, del_source=True) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError) as cm: + self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) + self._test_partial_magic(test, del_source=True) + + def test_magic_only(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_magic_only(test, del_source=True) + + def test_bad_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError) as cm: + self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) + + self._test_bad_magic(test, del_source=True) + + def test_partial_timestamp(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_timestamp(test, del_source=True) + + def test_partial_flags(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_flags(test, del_source=True) + + def test_partial_hash(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_hash(test, del_source=True) + + def test_partial_size(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_size(test, del_source=True) + + def test_no_marshal(self): + self._test_no_marshal(del_source=True) + + def test_non_code_marshal(self): + self._test_non_code_marshal(del_source=True) + + +class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP451): + pass + + +(Frozen_SourcelessBadBytecodePEP451, + Source_SourcelessBadBytecodePEP451 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + + +class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP302): + pass + + +(Frozen_SourcelessBadBytecodePEP302, + Source_SourcelessBadBytecodePEP302 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py new file mode 100644 index 000000000..f372b850d --- /dev/null +++ b/Lib/test/test_importlib/source/test_finder.py @@ -0,0 +1,236 @@ +from .. import abc +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import errno +import os +import py_compile +import stat +import sys +import tempfile +from test.support import make_legacy_pyc +import unittest +import warnings + + +class FinderTests(abc.FinderTests): + + """For a top-level module, it should just be found directly in the + directory being searched. This is true for a directory with source + [top-level source], bytecode [top-level bc], or both [top-level both]. + There is also the possibility that it is a package [top-level package], in + which case there will be a directory with the module name and an + __init__.py file. If there is a directory without an __init__.py an + ImportWarning is returned [empty dir]. + + For sub-modules and sub-packages, the same happens as above but only use + the tail end of the name [sub module] [sub package] [sub empty]. + + When there is a conflict between a package and module having the same name + in the same directory, the package wins out [package over module]. This is + so that imports of modules within the package can occur rather than trigger + an import error. + + When there is a package and module with the same name, always pick the + package over the module [package over module]. This is so that imports from + the package have the possibility of succeeding. + + """ + + def get_finder(self, root): + loader_details = [(self.machinery.SourceFileLoader, + self.machinery.SOURCE_SUFFIXES), + (self.machinery.SourcelessFileLoader, + self.machinery.BYTECODE_SUFFIXES)] + return self.machinery.FileFinder(root, *loader_details) + + def import_(self, root, module): + finder = self.get_finder(root) + return self._find(finder, module, loader_only=True) + + def run_test(self, test, create=None, *, compile_=None, unlink=None): + """Test the finding of 'test' with the creation of modules listed in + 'create'. + + Any names listed in 'compile_' are byte-compiled. Modules + listed in 'unlink' have their source files deleted. + + """ + if create is None: + create = {test} + with util.create_modules(*create) as mapping: + if compile_: + for name in compile_: + py_compile.compile(mapping[name]) + if unlink: + for name in unlink: + os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise + loader = self.import_(mapping['.root'], test) + self.assertTrue(hasattr(loader, 'load_module')) + return loader + + def test_module(self): + # [top-level source] + self.run_test('top_level') + # [top-level bc] + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) + # [top-level both] + self.run_test('top_level', compile_={'top_level'}) + + # [top-level package] + def test_package(self): + # Source. + self.run_test('pkg', {'pkg.__init__'}) + # Bytecode. + self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'}, + unlink={'pkg.__init__'}) + # Both. + self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'}) + + # [sub module] + def test_module_in_package(self): + with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: + pkg_dir = os.path.dirname(mapping['pkg.__init__']) + loader = self.import_(pkg_dir, 'pkg.sub') + self.assertTrue(hasattr(loader, 'load_module')) + + # [sub package] + def test_package_in_package(self): + context = util.create_modules('pkg.__init__', 'pkg.sub.__init__') + with context as mapping: + pkg_dir = os.path.dirname(mapping['pkg.__init__']) + loader = self.import_(pkg_dir, 'pkg.sub') + self.assertTrue(hasattr(loader, 'load_module')) + + # [package over modules] + def test_package_over_module(self): + name = '_temp' + loader = self.run_test(name, {'{0}.__init__'.format(name), name}) + self.assertIn('__init__', loader.get_filename(name)) + + def test_failure(self): + with util.create_modules('blah') as mapping: + nothing = self.import_(mapping['.root'], 'sdfsadsadf') + self.assertIsNone(nothing) + + def test_empty_string_for_dir(self): + # The empty string from sys.path means to search in the cwd. + finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader, + self.machinery.SOURCE_SUFFIXES)) + with open('mod.py', 'w') as file: + file.write("# test file for importlib") + try: + loader = self._find(finder, 'mod', loader_only=True) + self.assertTrue(hasattr(loader, 'load_module')) + finally: + os.unlink('mod.py') + + def test_invalidate_caches(self): + # invalidate_caches() should reset the mtime. + finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader, + self.machinery.SOURCE_SUFFIXES)) + finder._path_mtime = 42 + finder.invalidate_caches() + self.assertEqual(finder._path_mtime, -1) + + # Regression test for http://bugs.python.org/issue14846 + def test_dir_removal_handling(self): + mod = 'mod' + with util.create_modules(mod) as mapping: + finder = self.get_finder(mapping['.root']) + found = self._find(finder, 'mod', loader_only=True) + self.assertIsNotNone(found) + found = self._find(finder, 'mod', loader_only=True) + self.assertIsNone(found) + + @unittest.skipUnless(sys.platform != 'win32', + 'os.chmod() does not support the needed arguments under Windows') + def test_no_read_directory(self): + # Issue #16730 + tempdir = tempfile.TemporaryDirectory() + original_mode = os.stat(tempdir.name).st_mode + def cleanup(tempdir): + """Cleanup function for the temporary directory. + + Since we muck with the permissions, we want to set them back to + their original values to make sure the directory can be properly + cleaned up. + + """ + os.chmod(tempdir.name, original_mode) + # If this is not explicitly called then the __del__ method is used, + # but since already mucking around might as well explicitly clean + # up. + tempdir.__exit__(None, None, None) + self.addCleanup(cleanup, tempdir) + os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR) + finder = self.get_finder(tempdir.name) + found = self._find(finder, 'doesnotexist') + self.assertEqual(found, self.NOT_FOUND) + + def test_ignore_file(self): + # If a directory got changed to a file from underneath us, then don't + # worry about looking for submodules. + with tempfile.NamedTemporaryFile() as file_obj: + finder = self.get_finder(file_obj.name) + found = self._find(finder, 'doesnotexist') + self.assertEqual(found, self.NOT_FOUND) + + +class FinderTestsPEP451(FinderTests): + + NOT_FOUND = None + + def _find(self, finder, name, loader_only=False): + spec = finder.find_spec(name) + return spec.loader if spec is not None else spec + + +(Frozen_FinderTestsPEP451, + Source_FinderTestsPEP451 + ) = util.test_both(FinderTestsPEP451, machinery=machinery) + + +class FinderTestsPEP420(FinderTests): + + NOT_FOUND = (None, []) + + def _find(self, finder, name, loader_only=False): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + loader_portions = finder.find_loader(name) + return loader_portions[0] if loader_only else loader_portions + + +(Frozen_FinderTestsPEP420, + Source_FinderTestsPEP420 + ) = util.test_both(FinderTestsPEP420, machinery=machinery) + + +class FinderTestsPEP302(FinderTests): + + NOT_FOUND = None + + def _find(self, finder, name, loader_only=False): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return finder.find_module(name) + + +(Frozen_FinderTestsPEP302, + Source_FinderTestsPEP302 + ) = util.test_both(FinderTestsPEP302, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py new file mode 100644 index 000000000..795d436c3 --- /dev/null +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -0,0 +1,41 @@ +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import unittest + + +class PathHookTest: + + """Test the path hook for source.""" + + def path_hook(self): + return self.machinery.FileFinder.path_hook((self.machinery.SourceFileLoader, + self.machinery.SOURCE_SUFFIXES)) + + def test_success(self): + with util.create_modules('dummy') as mapping: + self.assertTrue(hasattr(self.path_hook()(mapping['.root']), + 'find_spec')) + + def test_success_legacy(self): + with util.create_modules('dummy') as mapping: + self.assertTrue(hasattr(self.path_hook()(mapping['.root']), + 'find_module')) + + def test_empty_string(self): + # The empty string represents the cwd. + self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + + def test_empty_string_legacy(self): + # The empty string represents the cwd. + self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) + + +(Frozen_PathHookTest, + Source_PathHooktest + ) = util.test_both(PathHookTest, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py new file mode 100644 index 000000000..980855fe1 --- /dev/null +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -0,0 +1,175 @@ +from .. import util + +machinery = util.import_importlib('importlib.machinery') + +import codecs +import importlib.util +import re +import types +# Because sys.path gets essentially blanked, need to have unicodedata already +# imported for the parser to use. +import unicodedata +import unittest +import warnings + + +CODING_RE = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) + + +class EncodingTest: + + """PEP 3120 makes UTF-8 the default encoding for source code + [default encoding]. + + PEP 263 specifies how that can change on a per-file basis. Either the first + or second line can contain the encoding line [encoding first line] + encoding second line]. If the file has the BOM marker it is considered UTF-8 + implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is + an error [BOM and utf-8][BOM conflict]. + + """ + + variable = '\u00fc' + character = '\u00c9' + source_line = "{0} = '{1}'\n".format(variable, character) + module_name = '_temp' + + def run_test(self, source): + with util.create_modules(self.module_name) as mapping: + with open(mapping[self.module_name], 'wb') as file: + file.write(source) + loader = self.machinery.SourceFileLoader(self.module_name, + mapping[self.module_name]) + return self.load(loader) + + def create_source(self, encoding): + encoding_line = "# coding={0}".format(encoding) + assert CODING_RE.match(encoding_line) + source_lines = [encoding_line.encode('utf-8')] + source_lines.append(self.source_line.encode(encoding)) + return b'\n'.join(source_lines) + + def test_non_obvious_encoding(self): + # Make sure that an encoding that has never been a standard one for + # Python works. + encoding_line = "# coding=koi8-r" + assert CODING_RE.match(encoding_line) + source = "{0}\na=42\n".format(encoding_line).encode("koi8-r") + self.run_test(source) + + # [default encoding] + def test_default_encoding(self): + self.run_test(self.source_line.encode('utf-8')) + + # [encoding first line] + def test_encoding_on_first_line(self): + encoding = 'Latin-1' + source = self.create_source(encoding) + self.run_test(source) + + # [encoding second line] + def test_encoding_on_second_line(self): + source = b"#/usr/bin/python\n" + self.create_source('Latin-1') + self.run_test(source) + + # [BOM] + def test_bom(self): + self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8')) + + # [BOM and utf-8] + def test_bom_and_utf_8(self): + source = codecs.BOM_UTF8 + self.create_source('utf-8') + self.run_test(source) + + # [BOM conflict] + def test_bom_conflict(self): + source = codecs.BOM_UTF8 + self.create_source('latin-1') + with self.assertRaises(SyntaxError): + self.run_test(source) + + +class EncodingTestPEP451(EncodingTest): + + def load(self, loader): + module = types.ModuleType(self.module_name) + module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader) + loader.exec_module(module) + return module + + +(Frozen_EncodingTestPEP451, + Source_EncodingTestPEP451 + ) = util.test_both(EncodingTestPEP451, machinery=machinery) + + +class EncodingTestPEP302(EncodingTest): + + def load(self, loader): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return loader.load_module(self.module_name) + + +(Frozen_EncodingTestPEP302, + Source_EncodingTestPEP302 + ) = util.test_both(EncodingTestPEP302, machinery=machinery) + + +class LineEndingTest: + + r"""Source written with the three types of line endings (\n, \r\n, \r) + need to be readable [cr][crlf][lf].""" + + def run_test(self, line_ending): + module_name = '_temp' + source_lines = [b"a = 42", b"b = -13", b''] + source = line_ending.join(source_lines) + with util.create_modules(module_name) as mapping: + with open(mapping[module_name], 'wb') as file: + file.write(source) + loader = self.machinery.SourceFileLoader(module_name, + mapping[module_name]) + return self.load(loader, module_name) + + # [cr] + def test_cr(self): + self.run_test(b'\r') + + # [crlf] + def test_crlf(self): + self.run_test(b'\r\n') + + # [lf] + def test_lf(self): + self.run_test(b'\n') + + +class LineEndingTestPEP451(LineEndingTest): + + def load(self, loader, module_name): + module = types.ModuleType(module_name) + module.__spec__ = importlib.util.spec_from_loader(module_name, loader) + loader.exec_module(module) + return module + + +(Frozen_LineEndingTestPEP451, + Source_LineEndingTestPEP451 + ) = util.test_both(LineEndingTestPEP451, machinery=machinery) + + +class LineEndingTestPEP302(LineEndingTest): + + def load(self, loader, module_name): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return loader.load_module(module_name) + + +(Frozen_LineEndingTestPEP302, + Source_LineEndingTestPEP302 + ) = util.test_both(LineEndingTestPEP302, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py new file mode 100644 index 000000000..05608bbb8 --- /dev/null +++ b/Lib/test/test_importlib/test_abc.py @@ -0,0 +1,1001 @@ +import io +import marshal +import os +import sys +from test import support +import types +import unittest +from unittest import mock +import warnings + +from . import util as test_util + +init = test_util.import_importlib('importlib') +abc = test_util.import_importlib('importlib.abc') +machinery = test_util.import_importlib('importlib.machinery') +util = test_util.import_importlib('importlib.util') + + +##### Inheritance ############################################################## +class InheritanceTests: + + """Test that the specified class is a subclass/superclass of the expected + classes.""" + + subclasses = [] + superclasses = [] + + def setUp(self): + self.superclasses = [getattr(self.abc, class_name) + for class_name in self.superclass_names] + if hasattr(self, 'subclass_names'): + # Because test.support.import_fresh_module() creates a new + # importlib._bootstrap per module, inheritance checks fail when + # checking across module boundaries (i.e. the _bootstrap in abc is + # not the same as the one in machinery). That means stealing one of + # the modules from the other to make sure the same instance is used. + machinery = self.abc.machinery + self.subclasses = [getattr(machinery, class_name) + for class_name in self.subclass_names] + assert self.subclasses or self.superclasses, self.__class__ + self.__test = getattr(self.abc, self._NAME) + + def test_subclasses(self): + # Test that the expected subclasses inherit. + for subclass in self.subclasses: + self.assertTrue(issubclass(subclass, self.__test), + "{0} is not a subclass of {1}".format(subclass, self.__test)) + + def test_superclasses(self): + # Test that the class inherits from the expected superclasses. + for superclass in self.superclasses: + self.assertTrue(issubclass(self.__test, superclass), + "{0} is not a superclass of {1}".format(superclass, self.__test)) + + +class MetaPathFinder(InheritanceTests): + superclass_names = ['Finder'] + subclass_names = ['BuiltinImporter', 'FrozenImporter', 'PathFinder', + 'WindowsRegistryFinder'] + + +(Frozen_MetaPathFinderInheritanceTests, + Source_MetaPathFinderInheritanceTests + ) = test_util.test_both(MetaPathFinder, abc=abc) + + +class PathEntryFinder(InheritanceTests): + superclass_names = ['Finder'] + subclass_names = ['FileFinder'] + + +(Frozen_PathEntryFinderInheritanceTests, + Source_PathEntryFinderInheritanceTests + ) = test_util.test_both(PathEntryFinder, abc=abc) + + +class ResourceLoader(InheritanceTests): + superclass_names = ['Loader'] + + +(Frozen_ResourceLoaderInheritanceTests, + Source_ResourceLoaderInheritanceTests + ) = test_util.test_both(ResourceLoader, abc=abc) + + +class InspectLoader(InheritanceTests): + superclass_names = ['Loader'] + subclass_names = ['BuiltinImporter', 'FrozenImporter', 'ExtensionFileLoader'] + + +(Frozen_InspectLoaderInheritanceTests, + Source_InspectLoaderInheritanceTests + ) = test_util.test_both(InspectLoader, abc=abc) + + +class ExecutionLoader(InheritanceTests): + superclass_names = ['InspectLoader'] + subclass_names = ['ExtensionFileLoader'] + + +(Frozen_ExecutionLoaderInheritanceTests, + Source_ExecutionLoaderInheritanceTests + ) = test_util.test_both(ExecutionLoader, abc=abc) + + +class FileLoader(InheritanceTests): + superclass_names = ['ResourceLoader', 'ExecutionLoader'] + subclass_names = ['SourceFileLoader', 'SourcelessFileLoader'] + + +(Frozen_FileLoaderInheritanceTests, + Source_FileLoaderInheritanceTests + ) = test_util.test_both(FileLoader, abc=abc) + + +class SourceLoader(InheritanceTests): + superclass_names = ['ResourceLoader', 'ExecutionLoader'] + subclass_names = ['SourceFileLoader'] + + +(Frozen_SourceLoaderInheritanceTests, + Source_SourceLoaderInheritanceTests + ) = test_util.test_both(SourceLoader, abc=abc) + + +##### Default return values #################################################### + +def make_abc_subclasses(base_class, name=None, inst=False, **kwargs): + if name is None: + name = base_class.__name__ + base = {kind: getattr(splitabc, name) + for kind, splitabc in abc.items()} + return {cls._KIND: cls() if inst else cls + for cls in test_util.split_frozen(base_class, base, **kwargs)} + + +class ABCTestHarness: + + @property + def ins(self): + # Lazily set ins on the class. + cls = self.SPLIT[self._KIND] + ins = cls() + self.__class__.ins = ins + return ins + + +class MetaPathFinder: + + def find_module(self, fullname, path): + return super().find_module(fullname, path) + + +class MetaPathFinderDefaultsTests(ABCTestHarness): + + SPLIT = make_abc_subclasses(MetaPathFinder) + + def test_find_module(self): + # Default should return None. + with self.assertWarns(DeprecationWarning): + found = self.ins.find_module('something', None) + self.assertIsNone(found) + + def test_invalidate_caches(self): + # Calling the method is a no-op. + self.ins.invalidate_caches() + + +(Frozen_MPFDefaultTests, + Source_MPFDefaultTests + ) = test_util.test_both(MetaPathFinderDefaultsTests) + + +class PathEntryFinder: + + def find_loader(self, fullname): + return super().find_loader(fullname) + + +class PathEntryFinderDefaultsTests(ABCTestHarness): + + SPLIT = make_abc_subclasses(PathEntryFinder) + + def test_find_loader(self): + with self.assertWarns(DeprecationWarning): + found = self.ins.find_loader('something') + self.assertEqual(found, (None, [])) + + def find_module(self): + self.assertEqual(None, self.ins.find_module('something')) + + def test_invalidate_caches(self): + # Should be a no-op. + self.ins.invalidate_caches() + + +(Frozen_PEFDefaultTests, + Source_PEFDefaultTests + ) = test_util.test_both(PathEntryFinderDefaultsTests) + + +class Loader: + + def load_module(self, fullname): + return super().load_module(fullname) + + +class LoaderDefaultsTests(ABCTestHarness): + + SPLIT = make_abc_subclasses(Loader) + + def test_create_module(self): + spec = 'a spec' + self.assertIsNone(self.ins.create_module(spec)) + + def test_load_module(self): + with self.assertRaises(ImportError): + self.ins.load_module('something') + + def test_module_repr(self): + mod = types.ModuleType('blah') + with self.assertRaises(NotImplementedError): + self.ins.module_repr(mod) + original_repr = repr(mod) + mod.__loader__ = self.ins + # Should still return a proper repr. + self.assertTrue(repr(mod)) + + +(Frozen_LDefaultTests, + SourceLDefaultTests + ) = test_util.test_both(LoaderDefaultsTests) + + +class ResourceLoader(Loader): + + def get_data(self, path): + return super().get_data(path) + + +class ResourceLoaderDefaultsTests(ABCTestHarness): + + SPLIT = make_abc_subclasses(ResourceLoader) + + def test_get_data(self): + with self.assertRaises(IOError): + self.ins.get_data('/some/path') + + +(Frozen_RLDefaultTests, + Source_RLDefaultTests + ) = test_util.test_both(ResourceLoaderDefaultsTests) + + +class InspectLoader(Loader): + + def is_package(self, fullname): + return super().is_package(fullname) + + def get_source(self, fullname): + return super().get_source(fullname) + + +SPLIT_IL = make_abc_subclasses(InspectLoader) + + +class InspectLoaderDefaultsTests(ABCTestHarness): + + SPLIT = SPLIT_IL + + def test_is_package(self): + with self.assertRaises(ImportError): + self.ins.is_package('blah') + + def test_get_source(self): + with self.assertRaises(ImportError): + self.ins.get_source('blah') + + +(Frozen_ILDefaultTests, + Source_ILDefaultTests + ) = test_util.test_both(InspectLoaderDefaultsTests) + + +class ExecutionLoader(InspectLoader): + + def get_filename(self, fullname): + return super().get_filename(fullname) + + +SPLIT_EL = make_abc_subclasses(ExecutionLoader) + + +class ExecutionLoaderDefaultsTests(ABCTestHarness): + + SPLIT = SPLIT_EL + + def test_get_filename(self): + with self.assertRaises(ImportError): + self.ins.get_filename('blah') + + +(Frozen_ELDefaultTests, + Source_ELDefaultsTests + ) = test_util.test_both(InspectLoaderDefaultsTests) + + +class ResourceReader: + + def open_resource(self, *args, **kwargs): + return super().open_resource(*args, **kwargs) + + def resource_path(self, *args, **kwargs): + return super().resource_path(*args, **kwargs) + + def is_resource(self, *args, **kwargs): + return super().is_resource(*args, **kwargs) + + def contents(self, *args, **kwargs): + return super().contents(*args, **kwargs) + + +class ResourceReaderDefaultsTests(ABCTestHarness): + + SPLIT = make_abc_subclasses(ResourceReader) + + def test_open_resource(self): + with self.assertRaises(FileNotFoundError): + self.ins.open_resource('dummy_file') + + def test_resource_path(self): + with self.assertRaises(FileNotFoundError): + self.ins.resource_path('dummy_file') + + def test_is_resource(self): + with self.assertRaises(FileNotFoundError): + self.ins.is_resource('dummy_file') + + def test_contents(self): + self.assertEqual([], list(self.ins.contents())) + +(Frozen_RRDefaultTests, + Source_RRDefaultsTests + ) = test_util.test_both(ResourceReaderDefaultsTests) + + +##### MetaPathFinder concrete methods ########################################## +class MetaPathFinderFindModuleTests: + + @classmethod + def finder(cls, spec): + class MetaPathSpecFinder(cls.abc.MetaPathFinder): + + def find_spec(self, fullname, path, target=None): + self.called_for = fullname, path + return spec + + return MetaPathSpecFinder() + + def test_no_spec(self): + finder = self.finder(None) + path = ['a', 'b', 'c'] + name = 'blah' + with self.assertWarns(DeprecationWarning): + found = finder.find_module(name, path) + self.assertIsNone(found) + self.assertEqual(name, finder.called_for[0]) + self.assertEqual(path, finder.called_for[1]) + + def test_spec(self): + loader = object() + spec = self.util.spec_from_loader('blah', loader) + finder = self.finder(spec) + with self.assertWarns(DeprecationWarning): + found = finder.find_module('blah', None) + self.assertIs(found, spec.loader) + + +(Frozen_MPFFindModuleTests, + Source_MPFFindModuleTests + ) = test_util.test_both(MetaPathFinderFindModuleTests, abc=abc, util=util) + + +##### PathEntryFinder concrete methods ######################################### +class PathEntryFinderFindLoaderTests: + + @classmethod + def finder(cls, spec): + class PathEntrySpecFinder(cls.abc.PathEntryFinder): + + def find_spec(self, fullname, target=None): + self.called_for = fullname + return spec + + return PathEntrySpecFinder() + + def test_no_spec(self): + finder = self.finder(None) + name = 'blah' + with self.assertWarns(DeprecationWarning): + found = finder.find_loader(name) + self.assertIsNone(found[0]) + self.assertEqual([], found[1]) + self.assertEqual(name, finder.called_for) + + def test_spec_with_loader(self): + loader = object() + spec = self.util.spec_from_loader('blah', loader) + finder = self.finder(spec) + with self.assertWarns(DeprecationWarning): + found = finder.find_loader('blah') + self.assertIs(found[0], spec.loader) + + def test_spec_with_portions(self): + spec = self.machinery.ModuleSpec('blah', None) + paths = ['a', 'b', 'c'] + spec.submodule_search_locations = paths + finder = self.finder(spec) + with self.assertWarns(DeprecationWarning): + found = finder.find_loader('blah') + self.assertIsNone(found[0]) + self.assertEqual(paths, found[1]) + + +(Frozen_PEFFindLoaderTests, + Source_PEFFindLoaderTests + ) = test_util.test_both(PathEntryFinderFindLoaderTests, abc=abc, util=util, + machinery=machinery) + + +##### Loader concrete methods ################################################## +class LoaderLoadModuleTests: + + def loader(self): + class SpecLoader(self.abc.Loader): + found = None + def exec_module(self, module): + self.found = module + + def is_package(self, fullname): + """Force some non-default module state to be set.""" + return True + + return SpecLoader() + + def test_fresh(self): + loader = self.loader() + name = 'blah' + with test_util.uncache(name): + loader.load_module(name) + module = loader.found + self.assertIs(sys.modules[name], module) + self.assertEqual(loader, module.__loader__) + self.assertEqual(loader, module.__spec__.loader) + self.assertEqual(name, module.__name__) + self.assertEqual(name, module.__spec__.name) + self.assertIsNotNone(module.__path__) + self.assertIsNotNone(module.__path__, + module.__spec__.submodule_search_locations) + + def test_reload(self): + name = 'blah' + loader = self.loader() + module = types.ModuleType(name) + module.__spec__ = self.util.spec_from_loader(name, loader) + module.__loader__ = loader + with test_util.uncache(name): + sys.modules[name] = module + loader.load_module(name) + found = loader.found + self.assertIs(found, sys.modules[name]) + self.assertIs(module, sys.modules[name]) + + +(Frozen_LoaderLoadModuleTests, + Source_LoaderLoadModuleTests + ) = test_util.test_both(LoaderLoadModuleTests, abc=abc, util=util) + + +##### InspectLoader concrete methods ########################################### +class InspectLoaderSourceToCodeTests: + + def source_to_module(self, data, path=None): + """Help with source_to_code() tests.""" + module = types.ModuleType('blah') + loader = self.InspectLoaderSubclass() + if path is None: + code = loader.source_to_code(data) + else: + code = loader.source_to_code(data, path) + exec(code, module.__dict__) + return module + + def test_source_to_code_source(self): + # Since compile() can handle strings, so should source_to_code(). + source = 'attr = 42' + module = self.source_to_module(source) + self.assertTrue(hasattr(module, 'attr')) + self.assertEqual(module.attr, 42) + + def test_source_to_code_bytes(self): + # Since compile() can handle bytes, so should source_to_code(). + source = b'attr = 42' + module = self.source_to_module(source) + self.assertTrue(hasattr(module, 'attr')) + self.assertEqual(module.attr, 42) + + def test_source_to_code_path(self): + # Specifying a path should set it for the code object. + path = 'path/to/somewhere' + loader = self.InspectLoaderSubclass() + code = loader.source_to_code('', path) + self.assertEqual(code.co_filename, path) + + def test_source_to_code_no_path(self): + # Not setting a path should still work and be set to since that + # is a pre-existing practice as a default to compile(). + loader = self.InspectLoaderSubclass() + code = loader.source_to_code('') + self.assertEqual(code.co_filename, '') + + +(Frozen_ILSourceToCodeTests, + Source_ILSourceToCodeTests + ) = test_util.test_both(InspectLoaderSourceToCodeTests, + InspectLoaderSubclass=SPLIT_IL) + + +class InspectLoaderGetCodeTests: + + def test_get_code(self): + # Test success. + module = types.ModuleType('blah') + with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = 'attr = 42' + loader = self.InspectLoaderSubclass() + code = loader.get_code('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked: + mocked.return_value = None + loader = self.InspectLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = self.InspectLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + +(Frozen_ILGetCodeTests, + Source_ILGetCodeTests + ) = test_util.test_both(InspectLoaderGetCodeTests, + InspectLoaderSubclass=SPLIT_IL) + + +class InspectLoaderLoadModuleTests: + + """Test InspectLoader.load_module().""" + + module_name = 'blah' + + def setUp(self): + support.unload(self.module_name) + self.addCleanup(support.unload, self.module_name) + + def load(self, loader): + spec = self.util.spec_from_loader(self.module_name, loader) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return self.init._bootstrap._load_unlocked(spec) + + def mock_get_code(self): + return mock.patch.object(self.InspectLoaderSubclass, 'get_code') + + def test_get_code_ImportError(self): + # If get_code() raises ImportError, it should propagate. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.side_effect = ImportError + with self.assertRaises(ImportError): + loader = self.InspectLoaderSubclass() + self.load(loader) + + def test_get_code_None(self): + # If get_code() returns None, raise ImportError. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = None + with self.assertRaises(ImportError): + loader = self.InspectLoaderSubclass() + self.load(loader) + + def test_module_returned(self): + # The loaded module should be returned. + code = compile('attr = 42', '', 'exec') + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = code + loader = self.InspectLoaderSubclass() + module = self.load(loader) + self.assertEqual(module, sys.modules[self.module_name]) + + +(Frozen_ILLoadModuleTests, + Source_ILLoadModuleTests + ) = test_util.test_both(InspectLoaderLoadModuleTests, + InspectLoaderSubclass=SPLIT_IL, + init=init, + util=util) + + +##### ExecutionLoader concrete methods ######################################### +class ExecutionLoaderGetCodeTests: + + def mock_methods(self, *, get_source=False, get_filename=False): + source_mock_context, filename_mock_context = None, None + if get_source: + source_mock_context = mock.patch.object(self.ExecutionLoaderSubclass, + 'get_source') + if get_filename: + filename_mock_context = mock.patch.object(self.ExecutionLoaderSubclass, + 'get_filename') + return source_mock_context, filename_mock_context + + def test_get_code(self): + path = 'blah.py' + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.return_value = path + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, path) + module = types.ModuleType('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + def test_get_code_source_is_None(self): + # If get_source() is None then this should be None. + source_mock_context, _ = self.mock_methods(get_source=True) + with source_mock_context as mocked: + mocked.return_value = None + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertIsNone(code) + + def test_get_code_source_not_found(self): + # If there is no source then there is no code object. + loader = self.ExecutionLoaderSubclass() + with self.assertRaises(ImportError): + loader.get_code('blah') + + def test_get_code_no_path(self): + # If get_filename() raises ImportError then simply skip setting the path + # on the code object. + source_mock_context, filename_mock_context = self.mock_methods( + get_source=True, get_filename=True) + with source_mock_context as source_mock, filename_mock_context as name_mock: + source_mock.return_value = 'attr = 42' + name_mock.side_effect = ImportError + loader = self.ExecutionLoaderSubclass() + code = loader.get_code('blah') + self.assertEqual(code.co_filename, '') + module = types.ModuleType('blah') + exec(code, module.__dict__) + self.assertEqual(module.attr, 42) + + +(Frozen_ELGetCodeTests, + Source_ELGetCodeTests + ) = test_util.test_both(ExecutionLoaderGetCodeTests, + ExecutionLoaderSubclass=SPLIT_EL) + + +##### SourceLoader concrete methods ############################################ +class SourceOnlyLoader: + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " + b"repr(__loader__)])") + + def __init__(self, path): + self.path = path + + def get_data(self, path): + if path != self.path: + raise IOError + return self.source + + def get_filename(self, fullname): + return self.path + + def module_repr(self, module): + return '' + + +SPLIT_SOL = make_abc_subclasses(SourceOnlyLoader, 'SourceLoader') + + +class SourceLoader(SourceOnlyLoader): + + source_mtime = 1 + + def __init__(self, path, magic=None): + super().__init__(path) + self.bytecode_path = self.util.cache_from_source(self.path) + self.source_size = len(self.source) + if magic is None: + magic = self.util.MAGIC_NUMBER + data = bytearray(magic) + data.extend(self.init._pack_uint32(0)) + data.extend(self.init._pack_uint32(self.source_mtime)) + data.extend(self.init._pack_uint32(self.source_size)) + code_object = compile(self.source, self.path, 'exec', + dont_inherit=True) + data.extend(marshal.dumps(code_object)) + self.bytecode = bytes(data) + self.written = {} + + def get_data(self, path): + if path == self.path: + return super().get_data(path) + elif path == self.bytecode_path: + return self.bytecode + else: + raise OSError + + def path_stats(self, path): + if path != self.path: + raise IOError + return {'mtime': self.source_mtime, 'size': self.source_size} + + def set_data(self, path, data): + self.written[path] = bytes(data) + return path == self.bytecode_path + + +SPLIT_SL = make_abc_subclasses(SourceLoader, util=util, init=init) + + +class SourceLoaderTestHarness: + + def setUp(self, *, is_package=True, **kwargs): + self.package = 'pkg' + if is_package: + self.path = os.path.join(self.package, '__init__.py') + self.name = self.package + else: + module_name = 'mod' + self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) + self.name = '.'.join([self.package, module_name]) + self.cached = self.util.cache_from_source(self.path) + self.loader = self.loader_mock(self.path, **kwargs) + + def verify_module(self, module): + self.assertEqual(module.__name__, self.name) + self.assertEqual(module.__file__, self.path) + self.assertEqual(module.__cached__, self.cached) + self.assertEqual(module.__package__, self.package) + self.assertEqual(module.__loader__, self.loader) + values = module._.split('::') + self.assertEqual(values[0], self.name) + self.assertEqual(values[1], self.path) + self.assertEqual(values[2], self.cached) + self.assertEqual(values[3], self.package) + self.assertEqual(values[4], repr(self.loader)) + + def verify_code(self, code_object): + module = types.ModuleType(self.name) + module.__file__ = self.path + module.__cached__ = self.cached + module.__package__ = self.package + module.__loader__ = self.loader + module.__path__ = [] + exec(code_object, module.__dict__) + self.verify_module(module) + + +class SourceOnlyLoaderTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader for source-only loading. + + Reload testing is subsumed by the tests for + importlib.util.module_for_loader. + + """ + + def test_get_source(self): + # Verify the source code is returned as a string. + # If an OSError is raised by get_data then raise ImportError. + expected_source = self.loader.source.decode('utf-8') + self.assertEqual(self.loader.get_source(self.name), expected_source) + def raise_OSError(path): + raise OSError + self.loader.get_data = raise_OSError + with self.assertRaises(ImportError) as cm: + self.loader.get_source(self.name) + self.assertEqual(cm.exception.name, self.name) + + def test_is_package(self): + # Properly detect when loading a package. + self.setUp(is_package=False) + self.assertFalse(self.loader.is_package(self.name)) + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.assertFalse(self.loader.is_package(self.name + '.__init__')) + + def test_get_code(self): + # Verify the code object is created. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_source_to_code(self): + # Verify the compiled code object. + code = self.loader.source_to_code(self.loader.source, self.path) + self.verify_code(code) + + def test_load_module(self): + # Loading a module should set __name__, __loader__, __package__, + # __path__ (for packages), __file__, and __cached__. + # The module should also be put into sys.modules. + with test_util.uncache(self.name): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertIn(self.name, sys.modules) + + def test_package_settings(self): + # __package__ needs to be set, while __path__ is set on if the module + # is a package. + # Testing the values for a package are covered by test_load_module. + self.setUp(is_package=False) + with test_util.uncache(self.name): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertFalse(hasattr(module, '__path__')) + + def test_get_source_encoding(self): + # Source is considered encoded in UTF-8 by default unless otherwise + # specified by an encoding line. + source = "_ = 'ü'" + self.loader.source = source.encode('utf-8') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + source = "# coding: latin-1\n_ = ü" + self.loader.source = source.encode('latin-1') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + + +(Frozen_SourceOnlyLoaderTests, + Source_SourceOnlyLoaderTests + ) = test_util.test_both(SourceOnlyLoaderTests, util=util, + loader_mock=SPLIT_SOL) + + +@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +class SourceLoaderBytecodeTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader's use of bytecode. + + Source-only testing handled by SourceOnlyLoaderTests. + + """ + + def verify_code(self, code_object, *, bytecode_written=False): + super().verify_code(code_object) + if bytecode_written: + self.assertIn(self.cached, self.loader.written) + data = bytearray(self.util.MAGIC_NUMBER) + data.extend(self.init._pack_uint32(0)) + data.extend(self.init._pack_uint32(self.loader.source_mtime)) + data.extend(self.init._pack_uint32(self.loader.source_size)) + data.extend(marshal.dumps(code_object)) + self.assertEqual(self.loader.written[self.cached], bytes(data)) + + def test_code_with_everything(self): + # When everything should work. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_no_bytecode(self): + # If no bytecode exists then move on to the source. + self.loader.bytecode_path = "" + # Sanity check + with self.assertRaises(OSError): + bytecode_path = self.util.cache_from_source(self.path) + self.loader.get_data(bytecode_path) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_code_bad_timestamp(self): + # Bytecode is only used when the timestamp matches the source EXACTLY. + for source_mtime in (0, 2): + assert source_mtime != self.loader.source_mtime + original = self.loader.source_mtime + self.loader.source_mtime = source_mtime + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + self.loader.source_mtime = original + + def test_code_bad_magic(self): + # Skip over bytecode with a bad magic number. + self.setUp(magic=b'0000') + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_dont_write_bytecode(self): + # Bytecode is not written if sys.dont_write_bytecode is true. + # Can assume it is false already thanks to the skipIf class decorator. + try: + sys.dont_write_bytecode = True + self.loader.bytecode_path = "" + code_object = self.loader.get_code(self.name) + self.assertNotIn(self.cached, self.loader.written) + finally: + sys.dont_write_bytecode = False + + def test_no_set_data(self): + # If set_data is not defined, one can still read bytecode. + self.setUp(magic=b'0000') + original_set_data = self.loader.__class__.mro()[1].set_data + try: + del self.loader.__class__.mro()[1].set_data + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + finally: + self.loader.__class__.mro()[1].set_data = original_set_data + + def test_set_data_raises_exceptions(self): + # Raising NotImplementedError or OSError is okay for set_data. + def raise_exception(exc): + def closure(*args, **kwargs): + raise exc + return closure + + self.setUp(magic=b'0000') + self.loader.set_data = raise_exception(NotImplementedError) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + +(Frozen_SLBytecodeTests, + SourceSLBytecodeTests + ) = test_util.test_both(SourceLoaderBytecodeTests, init=init, util=util, + loader_mock=SPLIT_SL) + + +class SourceLoaderGetSourceTests: + + """Tests for importlib.abc.SourceLoader.get_source().""" + + def test_default_encoding(self): + # Should have no problems with UTF-8 text. + name = 'mod' + mock = self.SourceOnlyLoaderMock('mod.file') + source = 'x = "ü"' + mock.source = source.encode('utf-8') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_decoded_source(self): + # Decoding should work. + name = 'mod' + mock = self.SourceOnlyLoaderMock("mod.file") + source = "# coding: Latin-1\nx='ü'" + assert source.encode('latin-1') != source.encode('utf-8') + mock.source = source.encode('latin-1') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_universal_newlines(self): + # PEP 302 says universal newlines should be used. + name = 'mod' + mock = self.SourceOnlyLoaderMock('mod.file') + source = "x = 42\r\ny = -13\r\n" + mock.source = source.encode('utf-8') + expect = io.IncrementalNewlineDecoder(None, True).decode(source) + self.assertEqual(mock.get_source(name), expect) + + +(Frozen_SourceOnlyLoaderGetSourceTests, + Source_SourceOnlyLoaderGetSourceTests + ) = test_util.test_both(SourceLoaderGetSourceTests, + SourceOnlyLoaderMock=SPLIT_SOL) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py new file mode 100644 index 000000000..edb745c2c --- /dev/null +++ b/Lib/test/test_importlib/test_api.py @@ -0,0 +1,461 @@ +from . import util as test_util + +init = test_util.import_importlib('importlib') +util = test_util.import_importlib('importlib.util') +machinery = test_util.import_importlib('importlib.machinery') + +import os.path +import sys +from test import support +import types +import unittest +import warnings + + +class ImportModuleTests: + + """Test importlib.import_module.""" + + def test_module_import(self): + # Test importing a top-level module. + with test_util.mock_modules('top_level') as mock: + with test_util.import_state(meta_path=[mock]): + module = self.init.import_module('top_level') + self.assertEqual(module.__name__, 'top_level') + + def test_absolute_package_import(self): + # Test importing a module from a package with an absolute name. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + name = '{0}.mod'.format(pkg_name) + with test_util.mock_modules(pkg_long_name, name) as mock: + with test_util.import_state(meta_path=[mock]): + module = self.init.import_module(name) + self.assertEqual(module.__name__, name) + + def test_shallow_relative_package_import(self): + # Test importing a module from a package through a relative import. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + module_name = 'mod' + absolute_name = '{0}.{1}'.format(pkg_name, module_name) + relative_name = '.{0}'.format(module_name) + with test_util.mock_modules(pkg_long_name, absolute_name) as mock: + with test_util.import_state(meta_path=[mock]): + self.init.import_module(pkg_name) + module = self.init.import_module(relative_name, pkg_name) + self.assertEqual(module.__name__, absolute_name) + + def test_deep_relative_package_import(self): + modules = ['a.__init__', 'a.b.__init__', 'a.c'] + with test_util.mock_modules(*modules) as mock: + with test_util.import_state(meta_path=[mock]): + self.init.import_module('a') + self.init.import_module('a.b') + module = self.init.import_module('..c', 'a.b') + self.assertEqual(module.__name__, 'a.c') + + def test_absolute_import_with_package(self): + # Test importing a module from a package with an absolute name with + # the 'package' argument given. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + name = '{0}.mod'.format(pkg_name) + with test_util.mock_modules(pkg_long_name, name) as mock: + with test_util.import_state(meta_path=[mock]): + self.init.import_module(pkg_name) + module = self.init.import_module(name, pkg_name) + self.assertEqual(module.__name__, name) + + def test_relative_import_wo_package(self): + # Relative imports cannot happen without the 'package' argument being + # set. + with self.assertRaises(TypeError): + self.init.import_module('.support') + + + def test_loaded_once(self): + # Issue #13591: Modules should only be loaded once when + # initializing the parent package attempts to import the + # module currently being imported. + b_load_count = 0 + def load_a(): + self.init.import_module('a.b') + def load_b(): + nonlocal b_load_count + b_load_count += 1 + code = {'a': load_a, 'a.b': load_b} + modules = ['a.__init__', 'a.b'] + with test_util.mock_modules(*modules, module_code=code) as mock: + with test_util.import_state(meta_path=[mock]): + self.init.import_module('a.b') + self.assertEqual(b_load_count, 1) + + +(Frozen_ImportModuleTests, + Source_ImportModuleTests + ) = test_util.test_both(ImportModuleTests, init=init) + + +class FindLoaderTests: + + FakeMetaFinder = None + + def test_sys_modules(self): + # If a module with __loader__ is in sys.modules, then return it. + name = 'some_mod' + with test_util.uncache(name): + module = types.ModuleType(name) + loader = 'a loader!' + module.__loader__ = loader + sys.modules[name] = module + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + found = self.init.find_loader(name) + self.assertEqual(loader, found) + + def test_sys_modules_loader_is_None(self): + # If sys.modules[name].__loader__ is None, raise ValueError. + name = 'some_mod' + with test_util.uncache(name): + module = types.ModuleType(name) + module.__loader__ = None + sys.modules[name] = module + with self.assertRaises(ValueError): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.init.find_loader(name) + + def test_sys_modules_loader_is_not_set(self): + # Should raise ValueError + # Issue #17099 + name = 'some_mod' + with test_util.uncache(name): + module = types.ModuleType(name) + try: + del module.__loader__ + except AttributeError: + pass + sys.modules[name] = module + with self.assertRaises(ValueError): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.init.find_loader(name) + + def test_success(self): + # Return the loader found on sys.meta_path. + name = 'some_mod' + with test_util.uncache(name): + with test_util.import_state(meta_path=[self.FakeMetaFinder]): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual((name, None), self.init.find_loader(name)) + + def test_success_path(self): + # Searching on a path should work. + name = 'some_mod' + path = 'path to some place' + with test_util.uncache(name): + with test_util.import_state(meta_path=[self.FakeMetaFinder]): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual((name, path), + self.init.find_loader(name, path)) + + def test_nothing(self): + # None is returned upon failure to find a loader. + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule')) + + +class FindLoaderPEP451Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_spec(name, path=None, target=None): + return machinery['Source'].ModuleSpec(name, (name, path)) + + +(Frozen_FindLoaderPEP451Tests, + Source_FindLoaderPEP451Tests + ) = test_util.test_both(FindLoaderPEP451Tests, init=init) + + +class FindLoaderPEP302Tests(FindLoaderTests): + + class FakeMetaFinder: + @staticmethod + def find_module(name, path=None): + return name, path + + +(Frozen_FindLoaderPEP302Tests, + Source_FindLoaderPEP302Tests + ) = test_util.test_both(FindLoaderPEP302Tests, init=init) + + +class ReloadTests: + + def test_reload_modules(self): + for mod in ('tokenize', 'time', 'marshal'): + with self.subTest(module=mod): + with support.CleanImport(mod): + module = self.init.import_module(mod) + self.init.reload(module) + + def test_module_replaced(self): + def code(): + import sys + module = type(sys)('top_level') + module.spam = 3 + sys.modules['top_level'] = module + mock = test_util.mock_modules('top_level', + module_code={'top_level': code}) + with mock: + with test_util.import_state(meta_path=[mock]): + module = self.init.import_module('top_level') + reloaded = self.init.reload(module) + actual = sys.modules['top_level'] + self.assertEqual(actual.spam, 3) + self.assertEqual(reloaded.spam, 3) + + def test_reload_missing_loader(self): + with support.CleanImport('types'): + import types + loader = types.__loader__ + del types.__loader__ + reloaded = self.init.reload(types) + + self.assertIs(reloaded, types) + self.assertIs(sys.modules['types'], types) + self.assertEqual(reloaded.__loader__.path, loader.path) + + def test_reload_loader_replaced(self): + with support.CleanImport('types'): + import types + types.__loader__ = None + self.init.invalidate_caches() + reloaded = self.init.reload(types) + + self.assertIsNot(reloaded.__loader__, None) + self.assertIs(reloaded, types) + self.assertIs(sys.modules['types'], types) + + def test_reload_location_changed(self): + name = 'spam' + with support.temp_cwd(None) as cwd: + with test_util.uncache('spam'): + with support.DirsOnSysPath(cwd): + # Start as a plain module. + self.init.invalidate_caches() + path = os.path.join(cwd, name + '.py') + cached = self.util.cache_from_source(path) + expected = {'__name__': name, + '__package__': '', + '__file__': path, + '__cached__': cached, + '__doc__': None, + } + support.create_empty_file(path) + module = self.init.import_module(name) + ns = vars(module).copy() + loader = ns.pop('__loader__') + spec = ns.pop('__spec__') + ns.pop('__builtins__', None) # An implementation detail. + self.assertEqual(spec.name, name) + self.assertEqual(spec.loader, loader) + self.assertEqual(loader.path, path) + self.assertEqual(ns, expected) + + # Change to a package. + self.init.invalidate_caches() + init_path = os.path.join(cwd, name, '__init__.py') + cached = self.util.cache_from_source(init_path) + expected = {'__name__': name, + '__package__': name, + '__file__': init_path, + '__cached__': cached, + '__path__': [os.path.dirname(init_path)], + '__doc__': None, + } + os.mkdir(name) + os.rename(path, init_path) + reloaded = self.init.reload(module) + ns = vars(reloaded).copy() + loader = ns.pop('__loader__') + spec = ns.pop('__spec__') + ns.pop('__builtins__', None) # An implementation detail. + self.assertEqual(spec.name, name) + self.assertEqual(spec.loader, loader) + self.assertIs(reloaded, module) + self.assertEqual(loader.path, init_path) + self.maxDiff = None + self.assertEqual(ns, expected) + + def test_reload_namespace_changed(self): + name = 'spam' + with support.temp_cwd(None) as cwd: + with test_util.uncache('spam'): + with support.DirsOnSysPath(cwd): + # Start as a namespace package. + self.init.invalidate_caches() + bad_path = os.path.join(cwd, name, '__init.py') + cached = self.util.cache_from_source(bad_path) + expected = {'__name__': name, + '__package__': name, + '__doc__': None, + '__file__': None, + } + os.mkdir(name) + with open(bad_path, 'w') as init_file: + init_file.write('eggs = None') + module = self.init.import_module(name) + ns = vars(module).copy() + loader = ns.pop('__loader__') + path = ns.pop('__path__') + spec = ns.pop('__spec__') + ns.pop('__builtins__', None) # An implementation detail. + self.assertEqual(spec.name, name) + self.assertIsNotNone(spec.loader) + self.assertIsNotNone(loader) + self.assertEqual(spec.loader, loader) + self.assertEqual(set(path), + set([os.path.dirname(bad_path)])) + with self.assertRaises(AttributeError): + # a NamespaceLoader + loader.path + self.assertEqual(ns, expected) + + # Change to a regular package. + self.init.invalidate_caches() + init_path = os.path.join(cwd, name, '__init__.py') + cached = self.util.cache_from_source(init_path) + expected = {'__name__': name, + '__package__': name, + '__file__': init_path, + '__cached__': cached, + '__path__': [os.path.dirname(init_path)], + '__doc__': None, + 'eggs': None, + } + os.rename(bad_path, init_path) + reloaded = self.init.reload(module) + ns = vars(reloaded).copy() + loader = ns.pop('__loader__') + spec = ns.pop('__spec__') + ns.pop('__builtins__', None) # An implementation detail. + self.assertEqual(spec.name, name) + self.assertEqual(spec.loader, loader) + self.assertIs(reloaded, module) + self.assertEqual(loader.path, init_path) + self.assertEqual(ns, expected) + + def test_reload_submodule(self): + # See #19851. + name = 'spam' + subname = 'ham' + with test_util.temp_module(name, pkg=True) as pkg_dir: + fullname, _ = test_util.submodule(name, subname, pkg_dir) + ham = self.init.import_module(fullname) + reloaded = self.init.reload(ham) + self.assertIs(reloaded, ham) + + def test_module_missing_spec(self): + #Test that reload() throws ModuleNotFounderror when reloading + # a module who's missing a spec. (bpo-29851) + name = 'spam' + with test_util.uncache(name): + module = sys.modules[name] = types.ModuleType(name) + # Sanity check by attempting an import. + module = self.init.import_module(name) + self.assertIsNone(module.__spec__) + with self.assertRaises(ModuleNotFoundError): + self.init.reload(module) + + +(Frozen_ReloadTests, + Source_ReloadTests + ) = test_util.test_both(ReloadTests, init=init, util=util) + + +class InvalidateCacheTests: + + def test_method_called(self): + # If defined the method should be called. + class InvalidatingNullFinder: + def __init__(self, *ignored): + self.called = False + def find_module(self, *args): + return None + def invalidate_caches(self): + self.called = True + + key = 'gobledeegook' + meta_ins = InvalidatingNullFinder() + path_ins = InvalidatingNullFinder() + sys.meta_path.insert(0, meta_ins) + self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) + sys.path_importer_cache[key] = path_ins + self.addCleanup(lambda: sys.meta_path.remove(meta_ins)) + self.init.invalidate_caches() + self.assertTrue(meta_ins.called) + self.assertTrue(path_ins.called) + + def test_method_lacking(self): + # There should be no issues if the method is not defined. + key = 'gobbledeegook' + sys.path_importer_cache[key] = None + self.addCleanup(lambda: sys.path_importer_cache.pop(key, None)) + self.init.invalidate_caches() # Shouldn't trigger an exception. + + +(Frozen_InvalidateCacheTests, + Source_InvalidateCacheTests + ) = test_util.test_both(InvalidateCacheTests, init=init) + + +class FrozenImportlibTests(unittest.TestCase): + + def test_no_frozen_importlib(self): + # Should be able to import w/o _frozen_importlib being defined. + # Can't do an isinstance() check since separate copies of importlib + # may have been used for import, so just check the name is not for the + # frozen loader. + source_init = init['Source'] + self.assertNotEqual(source_init.__loader__.__class__.__name__, + 'FrozenImporter') + + +class StartupTests: + + def test_everyone_has___loader__(self): + # Issue #17098: all modules should have __loader__ defined. + for name, module in sys.modules.items(): + if isinstance(module, types.ModuleType): + with self.subTest(name=name): + self.assertTrue(hasattr(module, '__loader__'), + '{!r} lacks a __loader__ attribute'.format(name)) + if self.machinery.BuiltinImporter.find_module(name): + self.assertIsNot(module.__loader__, None) + elif self.machinery.FrozenImporter.find_module(name): + self.assertIsNot(module.__loader__, None) + + def test_everyone_has___spec__(self): + for name, module in sys.modules.items(): + if isinstance(module, types.ModuleType): + with self.subTest(name=name): + self.assertTrue(hasattr(module, '__spec__')) + if self.machinery.BuiltinImporter.find_module(name): + self.assertIsNot(module.__spec__, None) + elif self.machinery.FrozenImporter.find_module(name): + self.assertIsNot(module.__spec__, None) + + +(Frozen_StartupTests, + Source_StartupTests + ) = test_util.test_both(StartupTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py new file mode 100644 index 000000000..28608e95d --- /dev/null +++ b/Lib/test/test_importlib/test_lazy.py @@ -0,0 +1,145 @@ +import importlib +from importlib import abc +from importlib import util +import sys +import types +import unittest + +from . import util as test_util + + +class CollectInit: + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def exec_module(self, module): + return self + + +class LazyLoaderFactoryTests(unittest.TestCase): + + def test_init(self): + factory = util.LazyLoader.factory(CollectInit) + # E.g. what importlib.machinery.FileFinder instantiates loaders with + # plus keyword arguments. + lazy_loader = factory('module name', 'module path', kw='kw') + loader = lazy_loader.loader + self.assertEqual(('module name', 'module path'), loader.args) + self.assertEqual({'kw': 'kw'}, loader.kwargs) + + def test_validation(self): + # No exec_module(), no lazy loading. + with self.assertRaises(TypeError): + util.LazyLoader.factory(object) + + +class TestingImporter(abc.MetaPathFinder, abc.Loader): + + module_name = 'lazy_loader_test' + mutated_name = 'changed' + loaded = None + source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) + + def find_spec(self, name, path, target=None): + if name != self.module_name: + return None + return util.spec_from_loader(name, util.LazyLoader(self)) + + def exec_module(self, module): + exec(self.source_code, module.__dict__) + self.loaded = module + + +class LazyLoaderTests(unittest.TestCase): + + def test_init(self): + with self.assertRaises(TypeError): + # Classes that don't define exec_module() trigger TypeError. + util.LazyLoader(object) + + def new_module(self, source_code=None): + loader = TestingImporter() + if source_code is not None: + loader.source_code = source_code + spec = util.spec_from_loader(TestingImporter.module_name, + util.LazyLoader(loader)) + module = spec.loader.create_module(spec) + if module is None: + module = types.ModuleType(TestingImporter.module_name) + module.__spec__ = spec + module.__loader__ = spec.loader + spec.loader.exec_module(module) + # Module is now lazy. + self.assertIsNone(loader.loaded) + return module + + def test_e2e(self): + # End-to-end test to verify the load is in fact lazy. + importer = TestingImporter() + assert importer.loaded is None + with test_util.uncache(importer.module_name): + with test_util.import_state(meta_path=[importer]): + module = importlib.import_module(importer.module_name) + self.assertIsNone(importer.loaded) + # Trigger load. + self.assertEqual(module.__loader__, importer) + self.assertIsNotNone(importer.loaded) + self.assertEqual(module, importer.loaded) + + def test_attr_unchanged(self): + # An attribute only mutated as a side-effect of import should not be + # changed needlessly. + module = self.new_module() + self.assertEqual(TestingImporter.mutated_name, module.__name__) + + def test_new_attr(self): + # A new attribute should persist. + module = self.new_module() + module.new_attr = 42 + self.assertEqual(42, module.new_attr) + + def test_mutated_preexisting_attr(self): + # Changing an attribute that already existed on the module -- + # e.g. __name__ -- should persist. + module = self.new_module() + module.__name__ = 'bogus' + self.assertEqual('bogus', module.__name__) + + def test_mutated_attr(self): + # Changing an attribute that comes into existence after an import + # should persist. + module = self.new_module() + module.attr = 6 + self.assertEqual(6, module.attr) + + def test_delete_eventual_attr(self): + # Deleting an attribute should stay deleted. + module = self.new_module() + del module.attr + self.assertFalse(hasattr(module, 'attr')) + + def test_delete_preexisting_attr(self): + module = self.new_module() + del module.__name__ + self.assertFalse(hasattr(module, '__name__')) + + def test_module_substitution_error(self): + with test_util.uncache(TestingImporter.module_name): + fresh_module = types.ModuleType(TestingImporter.module_name) + sys.modules[TestingImporter.module_name] = fresh_module + module = self.new_module() + with self.assertRaisesRegex(ValueError, "substituted"): + module.__name__ + + def test_module_already_in_sys(self): + with test_util.uncache(TestingImporter.module_name): + module = self.new_module() + sys.modules[TestingImporter.module_name] = module + # Force the load; just care that no exception is raised. + module.__name__ + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py new file mode 100644 index 000000000..21794d911 --- /dev/null +++ b/Lib/test/test_importlib/test_locks.py @@ -0,0 +1,152 @@ +from . import util as test_util + +init = test_util.import_importlib('importlib') + +import sys +import threading +import weakref + +from test import support +from test import lock_tests + + +class ModuleLockAsRLockTests: + locktype = classmethod(lambda cls: cls.LockType("some_lock")) + + # _is_owned() unsupported + test__is_owned = None + # acquire(blocking=False) unsupported + test_try_acquire = None + test_try_acquire_contended = None + # `with` unsupported + test_with = None + # acquire(timeout=...) unsupported + test_timeout = None + # _release_save() unsupported + test_release_save_unacquired = None + # lock status in repr unsupported + test_repr = None + test_locked_repr = None + +LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock + for kind, splitinit in init.items()} + +(Frozen_ModuleLockAsRLockTests, + Source_ModuleLockAsRLockTests + ) = test_util.test_both(ModuleLockAsRLockTests, lock_tests.RLockTests, + LockType=LOCK_TYPES) + + +class DeadlockAvoidanceTests: + + def setUp(self): + try: + self.old_switchinterval = sys.getswitchinterval() + support.setswitchinterval(0.000001) + except AttributeError: + self.old_switchinterval = None + + def tearDown(self): + if self.old_switchinterval is not None: + sys.setswitchinterval(self.old_switchinterval) + + def run_deadlock_avoidance_test(self, create_deadlock): + NLOCKS = 10 + locks = [self.LockType(str(i)) for i in range(NLOCKS)] + pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)] + if create_deadlock: + NTHREADS = NLOCKS + else: + NTHREADS = NLOCKS - 1 + barrier = threading.Barrier(NTHREADS) + results = [] + + def _acquire(lock): + """Try to acquire the lock. Return True on success, + False on deadlock.""" + try: + lock.acquire() + except self.DeadlockError: + return False + else: + return True + + def f(): + a, b = pairs.pop() + ra = _acquire(a) + barrier.wait() + rb = _acquire(b) + results.append((ra, rb)) + if rb: + b.release() + if ra: + a.release() + lock_tests.Bunch(f, NTHREADS).wait_for_finished() + self.assertEqual(len(results), NTHREADS) + return results + + def test_deadlock(self): + results = self.run_deadlock_avoidance_test(True) + # At least one of the threads detected a potential deadlock on its + # second acquire() call. It may be several of them, because the + # deadlock avoidance mechanism is conservative. + nb_deadlocks = results.count((True, False)) + self.assertGreaterEqual(nb_deadlocks, 1) + self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks) + + def test_no_deadlock(self): + results = self.run_deadlock_avoidance_test(False) + self.assertEqual(results.count((True, False)), 0) + self.assertEqual(results.count((True, True)), len(results)) + + +DEADLOCK_ERRORS = {kind: splitinit._bootstrap._DeadlockError + for kind, splitinit in init.items()} + +(Frozen_DeadlockAvoidanceTests, + Source_DeadlockAvoidanceTests + ) = test_util.test_both(DeadlockAvoidanceTests, + LockType=LOCK_TYPES, + DeadlockError=DEADLOCK_ERRORS) + + +class LifetimeTests: + + @property + def bootstrap(self): + return self.init._bootstrap + + def test_lock_lifetime(self): + name = "xyzzy" + self.assertNotIn(name, self.bootstrap._module_locks) + lock = self.bootstrap._get_module_lock(name) + self.assertIn(name, self.bootstrap._module_locks) + wr = weakref.ref(lock) + del lock + support.gc_collect() + self.assertNotIn(name, self.bootstrap._module_locks) + self.assertIsNone(wr()) + + def test_all_locks(self): + support.gc_collect() + self.assertEqual(0, len(self.bootstrap._module_locks), + self.bootstrap._module_locks) + + +(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) + + +@support.reap_threads +def test_main(): + support.run_unittest(Frozen_ModuleLockAsRLockTests, + Source_ModuleLockAsRLockTests, + Frozen_DeadlockAvoidanceTests, + Source_DeadlockAvoidanceTests, + Frozen_LifetimeTests, + Source_LifetimeTests) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py new file mode 100644 index 000000000..a8f95a035 --- /dev/null +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -0,0 +1,343 @@ +import contextlib +import importlib +import os +import sys +import unittest + +from test.test_importlib import util + +# needed tests: +# +# need to test when nested, so that the top-level path isn't sys.path +# need to test dynamic path detection, both at top-level and nested +# with dynamic path, check when a loader is returned on path reload (that is, +# trying to switch from a namespace package to a regular package) + + +@contextlib.contextmanager +def sys_modules_context(): + """ + Make sure sys.modules is the same object and has the same content + when exiting the context as when entering. + + Similar to importlib.test.util.uncache, but doesn't require explicit + names. + """ + sys_modules_saved = sys.modules + sys_modules_copy = sys.modules.copy() + try: + yield + finally: + sys.modules = sys_modules_saved + sys.modules.clear() + sys.modules.update(sys_modules_copy) + + +@contextlib.contextmanager +def namespace_tree_context(**kwargs): + """ + Save import state and sys.modules cache and restore it on exit. + Typical usage: + + >>> with namespace_tree_context(path=['/tmp/xxyy/portion1', + ... '/tmp/xxyy/portion2']): + ... pass + """ + # use default meta_path and path_hooks unless specified otherwise + kwargs.setdefault('meta_path', sys.meta_path) + kwargs.setdefault('path_hooks', sys.path_hooks) + import_context = util.import_state(**kwargs) + with import_context, sys_modules_context(): + yield + +class NamespacePackageTest(unittest.TestCase): + """ + Subclasses should define self.root and self.paths (under that root) + to be added to sys.path. + """ + root = os.path.join(os.path.dirname(__file__), 'namespace_pkgs') + + def setUp(self): + self.resolved_paths = [ + os.path.join(self.root, path) for path in self.paths + ] + self.ctx = namespace_tree_context(path=self.resolved_paths) + self.ctx.__enter__() + + def tearDown(self): + # TODO: will we ever want to pass exc_info to __exit__? + self.ctx.__exit__(None, None, None) + + +class SingleNamespacePackage(NamespacePackageTest): + paths = ['portion1'] + + def test_simple_package(self): + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + with self.assertRaises(ImportError): + import foo.two + + def test_module_repr(self): + import foo.one + self.assertEqual(repr(foo), "") + + +class DynamicPathNamespacePackage(NamespacePackageTest): + paths = ['portion1'] + + def test_dynamic_path(self): + # Make sure only 'foo.one' can be imported + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + with self.assertRaises(ImportError): + import foo.two + + # Now modify sys.path + sys.path.append(os.path.join(self.root, 'portion2')) + + # And make sure foo.two is now importable + import foo.two + self.assertEqual(foo.two.attr, 'portion2 foo two') + + +class CombinedNamespacePackages(NamespacePackageTest): + paths = ['both_portions'] + + def test_imports(self): + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'both_portions foo one') + self.assertEqual(foo.two.attr, 'both_portions foo two') + + +class SeparatedNamespacePackages(NamespacePackageTest): + paths = ['portion1', 'portion2'] + + def test_imports(self): + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'portion1 foo one') + self.assertEqual(foo.two.attr, 'portion2 foo two') + + +class SeparatedOverlappingNamespacePackages(NamespacePackageTest): + paths = ['portion1', 'both_portions'] + + def test_first_path_wins(self): + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'portion1 foo one') + self.assertEqual(foo.two.attr, 'both_portions foo two') + + def test_first_path_wins_again(self): + sys.path.reverse() + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'both_portions foo one') + self.assertEqual(foo.two.attr, 'both_portions foo two') + + def test_first_path_wins_importing_second_first(self): + import foo.two + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + self.assertEqual(foo.two.attr, 'both_portions foo two') + + +class SingleZipNamespacePackage(NamespacePackageTest): + paths = ['top_level_portion1.zip'] + + def test_simple_package(self): + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + with self.assertRaises(ImportError): + import foo.two + + +class SeparatedZipNamespacePackages(NamespacePackageTest): + paths = ['top_level_portion1.zip', 'portion2'] + + def test_imports(self): + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'portion1 foo one') + self.assertEqual(foo.two.attr, 'portion2 foo two') + self.assertIn('top_level_portion1.zip', foo.one.__file__) + self.assertNotIn('.zip', foo.two.__file__) + + +class SingleNestedZipNamespacePackage(NamespacePackageTest): + paths = ['nested_portion1.zip/nested_portion1'] + + def test_simple_package(self): + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + with self.assertRaises(ImportError): + import foo.two + + +class SeparatedNestedZipNamespacePackages(NamespacePackageTest): + paths = ['nested_portion1.zip/nested_portion1', 'portion2'] + + def test_imports(self): + import foo.one + import foo.two + self.assertEqual(foo.one.attr, 'portion1 foo one') + self.assertEqual(foo.two.attr, 'portion2 foo two') + fn = os.path.join('nested_portion1.zip', 'nested_portion1') + self.assertIn(fn, foo.one.__file__) + self.assertNotIn('.zip', foo.two.__file__) + + +class LegacySupport(NamespacePackageTest): + paths = ['not_a_namespace_pkg', 'portion1', 'portion2', 'both_portions'] + + def test_non_namespace_package_takes_precedence(self): + import foo.one + with self.assertRaises(ImportError): + import foo.two + self.assertIn('__init__', foo.__file__) + self.assertNotIn('namespace', str(foo.__loader__).lower()) + + +class DynamicPathCalculation(NamespacePackageTest): + paths = ['project1', 'project2'] + + def test_project3_fails(self): + import parent.child.one + self.assertEqual(len(parent.__path__), 2) + self.assertEqual(len(parent.child.__path__), 2) + import parent.child.two + self.assertEqual(len(parent.__path__), 2) + self.assertEqual(len(parent.child.__path__), 2) + + self.assertEqual(parent.child.one.attr, 'parent child one') + self.assertEqual(parent.child.two.attr, 'parent child two') + + with self.assertRaises(ImportError): + import parent.child.three + + self.assertEqual(len(parent.__path__), 2) + self.assertEqual(len(parent.child.__path__), 2) + + def test_project3_succeeds(self): + import parent.child.one + self.assertEqual(len(parent.__path__), 2) + self.assertEqual(len(parent.child.__path__), 2) + import parent.child.two + self.assertEqual(len(parent.__path__), 2) + self.assertEqual(len(parent.child.__path__), 2) + + self.assertEqual(parent.child.one.attr, 'parent child one') + self.assertEqual(parent.child.two.attr, 'parent child two') + + with self.assertRaises(ImportError): + import parent.child.three + + # now add project3 + sys.path.append(os.path.join(self.root, 'project3')) + import parent.child.three + + # the paths dynamically get longer, to include the new directories + self.assertEqual(len(parent.__path__), 3) + self.assertEqual(len(parent.child.__path__), 3) + + self.assertEqual(parent.child.three.attr, 'parent child three') + + +class ZipWithMissingDirectory(NamespacePackageTest): + paths = ['missing_directory.zip'] + + @unittest.expectedFailure + def test_missing_directory(self): + # This will fail because missing_directory.zip contains: + # Length Date Time Name + # --------- ---------- ----- ---- + # 29 2012-05-03 18:13 foo/one.py + # 0 2012-05-03 20:57 bar/ + # 38 2012-05-03 20:57 bar/two.py + # --------- ------- + # 67 3 files + + # Because there is no 'foo/', the zipimporter currently doesn't + # know that foo is a namespace package + + import foo.one + + def test_present_directory(self): + # This succeeds because there is a "bar/" in the zip file + import bar.two + self.assertEqual(bar.two.attr, 'missing_directory foo two') + + +class ModuleAndNamespacePackageInSameDir(NamespacePackageTest): + paths = ['module_and_namespace_package'] + + def test_module_before_namespace_package(self): + # Make sure we find the module in preference to the + # namespace package. + import a_test + self.assertEqual(a_test.attr, 'in module') + + +class ReloadTests(NamespacePackageTest): + paths = ['portion1'] + + def test_simple_package(self): + import foo.one + foo = importlib.reload(foo) + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_cant_import_other(self): + import foo + with self.assertRaises(ImportError): + import foo.two + foo = importlib.reload(foo) + with self.assertRaises(ImportError): + import foo.two + + def test_dynamic_path(self): + import foo.one + with self.assertRaises(ImportError): + import foo.two + + # Now modify sys.path and reload. + sys.path.append(os.path.join(self.root, 'portion2')) + foo = importlib.reload(foo) + + # And make sure foo.two is now importable + import foo.two + self.assertEqual(foo.two.attr, 'portion2 foo two') + + +class LoaderTests(NamespacePackageTest): + paths = ['portion1'] + + def test_namespace_loader_consistency(self): + # bpo-32303 + import foo + self.assertEqual(foo.__loader__, foo.__spec__.loader) + self.assertIsNotNone(foo.__loader__) + + def test_namespace_origin_consistency(self): + # bpo-32305 + import foo + self.assertIsNone(foo.__spec__.origin) + self.assertIsNone(foo.__file__) + + def test_path_indexable(self): + # bpo-35843 + import foo + expected_path = os.path.join(self.root, 'portion1', 'foo') + self.assertEqual(foo.__path__[0], expected_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py new file mode 100644 index 000000000..fd6e84b70 --- /dev/null +++ b/Lib/test/test_importlib/test_open.py @@ -0,0 +1,72 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_binary(package, path): + pass + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_text(package, path): + pass + + +class OpenTests: + def test_open_binary(self): + with resources.open_binary(self.data, 'binary.file') as fp: + result = fp.read() + self.assertEqual(result, b'\x00\x01\x02\x03') + + def test_open_text_default_encoding(self): + with resources.open_text(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_open_text_given_encoding(self): + with resources.open_text( + self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_open_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + self.assertRaises(UnicodeError, fp.read) + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + result = fp.read() + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + def test_open_binary_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_binary, self.data, 'does-not-exist') + + def test_open_text_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_text, self.data, 'does-not-exist') + + +class OpenDiskTests(OpenTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py new file mode 100644 index 000000000..2d3dcda7e --- /dev/null +++ b/Lib/test/test_importlib/test_path.py @@ -0,0 +1,39 @@ +import unittest + +from importlib import resources +from . import data01 +from . import util + + +class CommonTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + with resources.path(package, path): + pass + + +class PathTests: + def test_reading(self): + # Path should be readable. + # Test also implicitly verifies the returned object is a pathlib.Path + # instance. + with resources.path(self.data, 'utf-8.file') as path: + # pathlib.Path.read_text() was introduced in Python 3.5. + with path.open('r', encoding='utf-8') as file: + text = file.read() + self.assertEqual('Hello, UTF-8 world!\n', text) + + +class PathDiskTests(PathTests, unittest.TestCase): + data = data01 + + +class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): + def test_remove_in_context_manager(self): + # It is not an error if the file that was temporarily stashed on the + # file system is removed inside the `with` stanza. + with resources.path(self.data, 'utf-8.file') as path: + path.unlink() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py new file mode 100644 index 000000000..ff78d0b29 --- /dev/null +++ b/Lib/test/test_importlib/test_read.py @@ -0,0 +1,62 @@ +import unittest + +from importlib import import_module, resources +from . import data01 +from . import util + + +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_binary(package, path) + + +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): + def execute(self, package, path): + resources.read_text(package, path) + + +class ReadTests: + def test_read_binary(self): + result = resources.read_binary(self.data, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_text_default_encoding(self): + result = resources.read_text(self.data, 'utf-8.file') + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_read_text_given_encoding(self): + result = resources.read_text( + self.data, 'utf-16.file', encoding='utf-16') + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_read_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + self.assertRaises( + UnicodeError, resources.read_text, self.data, 'utf-16.file') + result = resources.read_text(self.data, 'utf-16.file', errors='ignore') + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + +class ReadDiskTests(ReadTests, unittest.TestCase): + data = data01 + + +class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + def test_read_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + result = resources.read_binary( + submodule, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_submodule_resource_by_name(self): + result = resources.read_binary( + 'ziptestdata.subdirectory', 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py new file mode 100644 index 000000000..f88d92d15 --- /dev/null +++ b/Lib/test/test_importlib/test_resource.py @@ -0,0 +1,166 @@ +import sys +import unittest + +from . import data01 +from . import zipdata01, zipdata02 +from . import util +from importlib import resources, import_module + + +class ResourceTests: + # Subclasses are expected to set the `data` attribute. + + def test_is_resource_good_path(self): + self.assertTrue(resources.is_resource(self.data, 'binary.file')) + + def test_is_resource_missing(self): + self.assertFalse(resources.is_resource(self.data, 'not-a-file')) + + def test_is_resource_subresource_directory(self): + # Directories are not resources. + self.assertFalse(resources.is_resource(self.data, 'subdirectory')) + + def test_contents(self): + contents = set(resources.contents(self.data)) + # There may be cruft in the directory listing of the data directory. + # Under Python 3 we could have a __pycache__ directory, and under + # Python 2 we could have .pyc files. These are both artifacts of the + # test suite importing these modules and writing these caches. They + # aren't germane to this test, so just filter them out. + contents.discard('__pycache__') + contents.discard('__init__.pyc') + contents.discard('__init__.pyo') + self.assertEqual(contents, { + '__init__.py', + 'subdirectory', + 'utf-8.file', + 'binary.file', + 'utf-16.file', + }) + + +class ResourceDiskTests(ResourceTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): + pass + + +class ResourceLoaderTests(unittest.TestCase): + def test_resource_contents(self): + package = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + self.assertEqual( + set(resources.contents(package)), + {'A', 'B', 'C'}) + + def test_resource_is_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertTrue(resources.is_resource(package, 'B')) + + def test_resource_directory_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'D')) + + def test_resource_missing_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'Z')) + + +class ResourceCornerCaseTests(unittest.TestCase): + def test_package_has_no_reader_fallback(self): + # Test odd ball packages which: + # 1. Do not have a ResourceReader as a loader + # 2. Are not on the file system + # 3. Are not in a zip file + module = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + # Give the module a dummy loader. + module.__loader__ = object() + # Give the module a dummy origin. + module.__file__ = '/path/which/shall/not/be/named' + if sys.version_info >= (3,): + module.__spec__.loader = module.__loader__ + module.__spec__.origin = module.__file__ + self.assertFalse(resources.is_resource(module, 'A')) + + +class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore + + def test_unrelated_contents(self): + # https://gitlab.com/python-devs/importlib_resources/issues/44 + # + # Here we have a zip file with two unrelated subpackages. The bug + # reports that getting the contents of a resource returns unrelated + # files. + self.assertEqual( + set(resources.contents('ziptestdata.one')), + {'__init__.py', 'resource1.txt'}) + self.assertEqual( + set(resources.contents('ziptestdata.two')), + {'__init__.py', 'resource2.txt'}) + + +class SubdirectoryResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata01 # type: ignore + + def test_is_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertTrue( + resources.is_resource(submodule, 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('ziptestdata.subdirectory', 'binary.file')) + + def test_submodule_contents(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertEqual( + set(resources.contents(submodule)), + {'__init__.py', 'binary.file'}) + + def test_submodule_contents_by_name(self): + self.assertEqual( + set(resources.contents('ziptestdata.subdirectory')), + {'__init__.py', 'binary.file'}) + + +class NamespaceTest(unittest.TestCase): + def test_namespaces_cannot_have_resources(self): + contents = resources.contents('test.test_importlib.data03.namespace') + self.assertFalse(list(contents)) + # Even though there is a file in the namespace directory, it is not + # considered a resource, since namespace packages can't have them. + self.assertFalse(resources.is_resource( + 'test.test_importlib.data03.namespace', + 'resource1.txt')) + # We should get an exception if we try to read it or open it. + self.assertRaises( + FileNotFoundError, + resources.open_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.open_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_text, + 'test.test_importlib.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_binary, + 'test.test_importlib.data03.namespace', 'resource1.txt') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py new file mode 100644 index 000000000..5a16a03de --- /dev/null +++ b/Lib/test/test_importlib/test_spec.py @@ -0,0 +1,819 @@ +from . import util as test_util + +init = test_util.import_importlib('importlib') +machinery = test_util.import_importlib('importlib.machinery') +util = test_util.import_importlib('importlib.util') + +import os.path +import pathlib +from test.support import CleanImport +import unittest +import sys +import warnings + + + +class TestLoader: + + def __init__(self, path=None, is_package=None): + self.path = path + self.package = is_package + + def __repr__(self): + return '' + + def __getattr__(self, name): + if name == 'get_filename' and self.path is not None: + return self._get_filename + if name == 'is_package': + return self._is_package + raise AttributeError(name) + + def _get_filename(self, name): + return self.path + + def _is_package(self, name): + return self.package + + def create_module(self, spec): + return None + + +class NewLoader(TestLoader): + + EGGS = 1 + + def exec_module(self, module): + module.eggs = self.EGGS + + +class LegacyLoader(TestLoader): + + HAM = -1 + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + + frozen_util = util['Frozen'] + + @frozen_util.module_for_loader + def load_module(self, module): + module.ham = self.HAM + return module + + +class ModuleSpecTests: + + def setUp(self): + self.name = 'spam' + self.path = 'spam.py' + self.cached = self.util.cache_from_source(self.path) + self.loader = TestLoader() + self.spec = self.machinery.ModuleSpec(self.name, self.loader) + self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, + origin=self.path) + self.loc_spec._set_fileattr = True + + def test_default(self): + spec = self.machinery.ModuleSpec(self.name, self.loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_default_no_loader(self): + spec = self.machinery.ModuleSpec(self.name, None) + + self.assertEqual(spec.name, self.name) + self.assertIs(spec.loader, None) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_default_is_package_false(self): + spec = self.machinery.ModuleSpec(self.name, self.loader, + is_package=False) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_default_is_package_true(self): + spec = self.machinery.ModuleSpec(self.name, self.loader, + is_package=True) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, []) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_has_location_setter(self): + spec = self.machinery.ModuleSpec(self.name, self.loader, + origin='somewhere') + self.assertFalse(spec.has_location) + spec.has_location = True + self.assertTrue(spec.has_location) + + def test_equality(self): + other = type(sys.implementation)(name=self.name, + loader=self.loader, + origin=None, + submodule_search_locations=None, + has_location=False, + cached=None, + ) + + self.assertTrue(self.spec == other) + + def test_equality_location(self): + other = type(sys.implementation)(name=self.name, + loader=self.loader, + origin=self.path, + submodule_search_locations=None, + has_location=True, + cached=self.cached, + ) + + self.assertEqual(self.loc_spec, other) + + def test_inequality(self): + other = type(sys.implementation)(name='ham', + loader=self.loader, + origin=None, + submodule_search_locations=None, + has_location=False, + cached=None, + ) + + self.assertNotEqual(self.spec, other) + + def test_inequality_incomplete(self): + other = type(sys.implementation)(name=self.name, + loader=self.loader, + ) + + self.assertNotEqual(self.spec, other) + + def test_package(self): + spec = self.machinery.ModuleSpec('spam.eggs', self.loader) + + self.assertEqual(spec.parent, 'spam') + + def test_package_is_package(self): + spec = self.machinery.ModuleSpec('spam.eggs', self.loader, + is_package=True) + + self.assertEqual(spec.parent, 'spam.eggs') + + # cached + + def test_cached_set(self): + before = self.spec.cached + self.spec.cached = 'there' + after = self.spec.cached + + self.assertIs(before, None) + self.assertEqual(after, 'there') + + def test_cached_no_origin(self): + spec = self.machinery.ModuleSpec(self.name, self.loader) + + self.assertIs(spec.cached, None) + + def test_cached_with_origin_not_location(self): + spec = self.machinery.ModuleSpec(self.name, self.loader, + origin=self.path) + + self.assertIs(spec.cached, None) + + def test_cached_source(self): + expected = self.util.cache_from_source(self.path) + + self.assertEqual(self.loc_spec.cached, expected) + + def test_cached_source_unknown_suffix(self): + self.loc_spec.origin = 'spam.spamspamspam' + + self.assertIs(self.loc_spec.cached, None) + + def test_cached_source_missing_cache_tag(self): + original = sys.implementation.cache_tag + sys.implementation.cache_tag = None + try: + cached = self.loc_spec.cached + finally: + sys.implementation.cache_tag = original + + self.assertIs(cached, None) + + def test_cached_sourceless(self): + self.loc_spec.origin = 'spam.pyc' + + self.assertEqual(self.loc_spec.cached, 'spam.pyc') + + +(Frozen_ModuleSpecTests, + Source_ModuleSpecTests + ) = test_util.test_both(ModuleSpecTests, util=util, machinery=machinery) + + +class ModuleSpecMethodsTests: + + @property + def bootstrap(self): + return self.init._bootstrap + + def setUp(self): + self.name = 'spam' + self.path = 'spam.py' + self.cached = self.util.cache_from_source(self.path) + self.loader = TestLoader() + self.spec = self.machinery.ModuleSpec(self.name, self.loader) + self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, + origin=self.path) + self.loc_spec._set_fileattr = True + + # exec() + + def test_exec(self): + self.spec.loader = NewLoader() + module = self.util.module_from_spec(self.spec) + sys.modules[self.name] = module + self.assertFalse(hasattr(module, 'eggs')) + self.bootstrap._exec(self.spec, module) + + self.assertEqual(module.eggs, 1) + + # load() + + def test_load(self): + self.spec.loader = NewLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + installed = sys.modules[self.spec.name] + + self.assertEqual(loaded.eggs, 1) + self.assertIs(loaded, installed) + + def test_load_replaced(self): + replacement = object() + class ReplacingLoader(TestLoader): + def exec_module(self, module): + sys.modules[module.__name__] = replacement + self.spec.loader = ReplacingLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + installed = sys.modules[self.spec.name] + + self.assertIs(loaded, replacement) + self.assertIs(installed, replacement) + + def test_load_failed(self): + class FailedLoader(TestLoader): + def exec_module(self, module): + raise RuntimeError + self.spec.loader = FailedLoader() + with CleanImport(self.spec.name): + with self.assertRaises(RuntimeError): + loaded = self.bootstrap._load(self.spec) + self.assertNotIn(self.spec.name, sys.modules) + + def test_load_failed_removed(self): + class FailedLoader(TestLoader): + def exec_module(self, module): + del sys.modules[module.__name__] + raise RuntimeError + self.spec.loader = FailedLoader() + with CleanImport(self.spec.name): + with self.assertRaises(RuntimeError): + loaded = self.bootstrap._load(self.spec) + self.assertNotIn(self.spec.name, sys.modules) + + def test_load_legacy(self): + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + + self.assertEqual(loaded.ham, -1) + + def test_load_legacy_attributes(self): + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + + self.assertIs(loaded.__loader__, self.spec.loader) + self.assertEqual(loaded.__package__, self.spec.parent) + self.assertIs(loaded.__spec__, self.spec) + + def test_load_legacy_attributes_immutable(self): + module = object() + class ImmutableLoader(TestLoader): + def load_module(self, name): + sys.modules[name] = module + return module + self.spec.loader = ImmutableLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + + self.assertIs(sys.modules[self.spec.name], module) + + # reload() + + def test_reload(self): + self.spec.loader = NewLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + reloaded = self.bootstrap._exec(self.spec, loaded) + installed = sys.modules[self.spec.name] + + self.assertEqual(loaded.eggs, 1) + self.assertIs(reloaded, loaded) + self.assertIs(installed, loaded) + + def test_reload_modified(self): + self.spec.loader = NewLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + loaded.eggs = 2 + reloaded = self.bootstrap._exec(self.spec, loaded) + + self.assertEqual(loaded.eggs, 1) + self.assertIs(reloaded, loaded) + + def test_reload_extra_attributes(self): + self.spec.loader = NewLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + loaded.available = False + reloaded = self.bootstrap._exec(self.spec, loaded) + + self.assertFalse(loaded.available) + self.assertIs(reloaded, loaded) + + def test_reload_init_module_attrs(self): + self.spec.loader = NewLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + loaded.__name__ = 'ham' + del loaded.__loader__ + del loaded.__package__ + del loaded.__spec__ + self.bootstrap._exec(self.spec, loaded) + + self.assertEqual(loaded.__name__, self.spec.name) + self.assertIs(loaded.__loader__, self.spec.loader) + self.assertEqual(loaded.__package__, self.spec.parent) + self.assertIs(loaded.__spec__, self.spec) + self.assertFalse(hasattr(loaded, '__path__')) + self.assertFalse(hasattr(loaded, '__file__')) + self.assertFalse(hasattr(loaded, '__cached__')) + + def test_reload_legacy(self): + self.spec.loader = LegacyLoader() + with CleanImport(self.spec.name): + loaded = self.bootstrap._load(self.spec) + reloaded = self.bootstrap._exec(self.spec, loaded) + installed = sys.modules[self.spec.name] + + self.assertEqual(loaded.ham, -1) + self.assertIs(reloaded, loaded) + self.assertIs(installed, loaded) + + +(Frozen_ModuleSpecMethodsTests, + Source_ModuleSpecMethodsTests + ) = test_util.test_both(ModuleSpecMethodsTests, init=init, util=util, + machinery=machinery) + + +class ModuleReprTests: + + @property + def bootstrap(self): + return self.init._bootstrap + + def setUp(self): + self.module = type(os)('spam') + self.spec = self.machinery.ModuleSpec('spam', TestLoader()) + + def test_module___loader___module_repr(self): + class Loader: + def module_repr(self, module): + return ''.format(module.__name__) + self.module.__loader__ = Loader() + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, '') + + def test_module___loader___module_repr_bad(self): + class Loader(TestLoader): + def module_repr(self, module): + raise Exception + self.module.__loader__ = Loader() + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, + ')>'.format('spam')) + + def test_module___spec__(self): + origin = 'in a hole, in the ground' + self.spec.origin = origin + self.module.__spec__ = self.spec + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, ''.format('spam', origin)) + + def test_module___spec___location(self): + location = 'in_a_galaxy_far_far_away.py' + self.spec.origin = location + self.spec._set_fileattr = True + self.module.__spec__ = self.spec + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, + ''.format('spam', location)) + + def test_module___spec___no_origin(self): + self.spec.loader = TestLoader() + self.module.__spec__ = self.spec + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, + ')>'.format('spam')) + + def test_module___spec___no_origin_no_loader(self): + self.spec.loader = None + self.module.__spec__ = self.spec + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, ''.format('spam')) + + def test_module_no_name(self): + del self.module.__name__ + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, ''.format('?')) + + def test_module_with_file(self): + filename = 'e/i/e/i/o/spam.py' + self.module.__file__ = filename + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, + ''.format('spam', filename)) + + def test_module_no_file(self): + self.module.__loader__ = TestLoader() + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, + ')>'.format('spam')) + + def test_module_no_file_no_loader(self): + modrepr = self.bootstrap._module_repr(self.module) + + self.assertEqual(modrepr, ''.format('spam')) + + +(Frozen_ModuleReprTests, + Source_ModuleReprTests + ) = test_util.test_both(ModuleReprTests, init=init, util=util, + machinery=machinery) + + +class FactoryTests: + + def setUp(self): + self.name = 'spam' + self.path = 'spam.py' + self.cached = self.util.cache_from_source(self.path) + self.loader = TestLoader() + self.fileloader = TestLoader(self.path) + self.pkgloader = TestLoader(self.path, True) + + # spec_from_loader() + + def test_spec_from_loader_default(self): + spec = self.util.spec_from_loader(self.name, self.loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_default_with_bad_is_package(self): + class Loader: + def is_package(self, name): + raise ImportError + loader = Loader() + spec = self.util.spec_from_loader(self.name, loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_origin(self): + origin = 'somewhere over the rainbow' + spec = self.util.spec_from_loader(self.name, self.loader, + origin=origin) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, origin) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_is_package_false(self): + spec = self.util.spec_from_loader(self.name, self.loader, + is_package=False) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_is_package_true(self): + spec = self.util.spec_from_loader(self.name, self.loader, + is_package=True) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, []) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_origin_and_is_package(self): + origin = 'where the streets have no name' + spec = self.util.spec_from_loader(self.name, self.loader, + origin=origin, is_package=True) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertIs(spec.origin, origin) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, []) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_is_package_with_loader_false(self): + loader = TestLoader(is_package=False) + spec = self.util.spec_from_loader(self.name, loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_is_package_with_loader_true(self): + loader = TestLoader(is_package=True) + spec = self.util.spec_from_loader(self.name, loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertIs(spec.origin, None) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, []) + self.assertIs(spec.cached, None) + self.assertFalse(spec.has_location) + + def test_spec_from_loader_default_with_file_loader(self): + spec = self.util.spec_from_loader(self.name, self.fileloader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_loader_is_package_false_with_fileloader(self): + spec = self.util.spec_from_loader(self.name, self.fileloader, + is_package=False) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_loader_is_package_true_with_fileloader(self): + spec = self.util.spec_from_loader(self.name, self.fileloader, + is_package=True) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + # spec_from_file_location() + + def test_spec_from_file_location_default(self): + spec = self.util.spec_from_file_location(self.name, self.path) + + self.assertEqual(spec.name, self.name) + # Need to use a circuitous route to get at importlib.machinery to make + # sure the same class object is used in the isinstance() check as + # would have been used to create the loader. + self.assertIsInstance(spec.loader, + self.util.abc.machinery.SourceFileLoader) + self.assertEqual(spec.loader.name, self.name) + self.assertEqual(spec.loader.path, self.path) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_path_like_arg(self): + spec = self.util.spec_from_file_location(self.name, + pathlib.PurePath(self.path)) + self.assertEqual(spec.origin, self.path) + + def test_spec_from_file_location_default_without_location(self): + spec = self.util.spec_from_file_location(self.name) + + self.assertIs(spec, None) + + def test_spec_from_file_location_default_bad_suffix(self): + spec = self.util.spec_from_file_location(self.name, 'spam.eggs') + + self.assertIs(spec, None) + + def test_spec_from_file_location_loader_no_location(self): + spec = self.util.spec_from_file_location(self.name, + loader=self.fileloader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_loader_no_location_no_get_filename(self): + spec = self.util.spec_from_file_location(self.name, + loader=self.loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.loader) + self.assertEqual(spec.origin, '') + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_loader_no_location_bad_get_filename(self): + class Loader: + def get_filename(self, name): + raise ImportError + loader = Loader() + spec = self.util.spec_from_file_location(self.name, loader=loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertEqual(spec.origin, '') + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertIs(spec.cached, None) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_none(self): + spec = self.util.spec_from_file_location(self.name, self.path, + loader=self.fileloader, + submodule_search_locations=None) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_empty(self): + spec = self.util.spec_from_file_location(self.name, self.path, + loader=self.fileloader, + submodule_search_locations=[]) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_not_empty(self): + spec = self.util.spec_from_file_location(self.name, self.path, + loader=self.fileloader, + submodule_search_locations=['eggs']) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, ['eggs']) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_default(self): + spec = self.util.spec_from_file_location(self.name, self.path, + loader=self.pkgloader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.pkgloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertEqual(spec.submodule_search_locations, ['']) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_default_not_package(self): + class Loader: + def is_package(self, name): + return False + loader = Loader() + spec = self.util.spec_from_file_location(self.name, self.path, + loader=loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_default_no_is_package(self): + spec = self.util.spec_from_file_location(self.name, self.path, + loader=self.fileloader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, self.fileloader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + def test_spec_from_file_location_smsl_default_bad_is_package(self): + class Loader: + def is_package(self, name): + raise ImportError + loader = Loader() + spec = self.util.spec_from_file_location(self.name, self.path, + loader=loader) + + self.assertEqual(spec.name, self.name) + self.assertEqual(spec.loader, loader) + self.assertEqual(spec.origin, self.path) + self.assertIs(spec.loader_state, None) + self.assertIs(spec.submodule_search_locations, None) + self.assertEqual(spec.cached, self.cached) + self.assertTrue(spec.has_location) + + +(Frozen_FactoryTests, + Source_FactoryTests + ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py new file mode 100644 index 000000000..db0899aff --- /dev/null +++ b/Lib/test/test_importlib/test_util.py @@ -0,0 +1,884 @@ +from . import util +abc = util.import_importlib('importlib.abc') +init = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') +importlib_util = util.import_importlib('importlib.util') + +import contextlib +import importlib.util +import os +import pathlib +import string +import sys +from test import support +import types +import unittest +import unittest.mock +import warnings + + +class DecodeSourceBytesTests: + + source = "string ='ü'" + + def test_ut8_default(self): + source_bytes = self.source.encode('utf-8') + self.assertEqual(self.util.decode_source(source_bytes), self.source) + + def test_specified_encoding(self): + source = '# coding=latin-1\n' + self.source + source_bytes = source.encode('latin-1') + assert source_bytes != source.encode('utf-8') + self.assertEqual(self.util.decode_source(source_bytes), source) + + def test_universal_newlines(self): + source = '\r\n'.join([self.source, self.source]) + source_bytes = source.encode('utf-8') + self.assertEqual(self.util.decode_source(source_bytes), + '\n'.join([self.source, self.source])) + + +(Frozen_DecodeSourceBytesTests, + Source_DecodeSourceBytesTests + ) = util.test_both(DecodeSourceBytesTests, util=importlib_util) + + +class ModuleFromSpecTests: + + def test_no_create_module(self): + class Loader: + def exec_module(self, module): + pass + spec = self.machinery.ModuleSpec('test', Loader()) + with self.assertRaises(ImportError): + module = self.util.module_from_spec(spec) + + def test_create_module_returns_None(self): + class Loader(self.abc.Loader): + def create_module(self, spec): + return None + spec = self.machinery.ModuleSpec('test', Loader()) + module = self.util.module_from_spec(spec) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, spec.name) + + def test_create_module(self): + name = 'already set' + class CustomModule(types.ModuleType): + pass + class Loader(self.abc.Loader): + def create_module(self, spec): + module = CustomModule(spec.name) + module.__name__ = name + return module + spec = self.machinery.ModuleSpec('test', Loader()) + module = self.util.module_from_spec(spec) + self.assertIsInstance(module, CustomModule) + self.assertEqual(module.__name__, name) + + def test___name__(self): + spec = self.machinery.ModuleSpec('test', object()) + module = self.util.module_from_spec(spec) + self.assertEqual(module.__name__, spec.name) + + def test___spec__(self): + spec = self.machinery.ModuleSpec('test', object()) + module = self.util.module_from_spec(spec) + self.assertEqual(module.__spec__, spec) + + def test___loader__(self): + loader = object() + spec = self.machinery.ModuleSpec('test', loader) + module = self.util.module_from_spec(spec) + self.assertIs(module.__loader__, loader) + + def test___package__(self): + spec = self.machinery.ModuleSpec('test.pkg', object()) + module = self.util.module_from_spec(spec) + self.assertEqual(module.__package__, spec.parent) + + def test___path__(self): + spec = self.machinery.ModuleSpec('test', object(), is_package=True) + module = self.util.module_from_spec(spec) + self.assertEqual(module.__path__, spec.submodule_search_locations) + + def test___file__(self): + spec = self.machinery.ModuleSpec('test', object(), origin='some/path') + spec.has_location = True + module = self.util.module_from_spec(spec) + self.assertEqual(module.__file__, spec.origin) + + def test___cached__(self): + spec = self.machinery.ModuleSpec('test', object()) + spec.cached = 'some/path' + spec.has_location = True + module = self.util.module_from_spec(spec) + self.assertEqual(module.__cached__, spec.cached) + +(Frozen_ModuleFromSpecTests, + Source_ModuleFromSpecTests +) = util.test_both(ModuleFromSpecTests, abc=abc, machinery=machinery, + util=importlib_util) + + +class ModuleForLoaderTests: + + """Tests for importlib.util.module_for_loader.""" + + @classmethod + def module_for_loader(cls, func): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + return cls.util.module_for_loader(func) + + def test_warning(self): + # Should raise a PendingDeprecationWarning when used. + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with self.assertRaises(DeprecationWarning): + func = self.util.module_for_loader(lambda x: x) + + def return_module(self, name): + fxn = self.module_for_loader(lambda self, module: module) + return fxn(self, name) + + def raise_exception(self, name): + def to_wrap(self, module): + raise ImportError + fxn = self.module_for_loader(to_wrap) + try: + fxn(self, name) + except ImportError: + pass + + def test_new_module(self): + # Test that when no module exists in sys.modules a new module is + # created. + module_name = 'a.b.c' + with util.uncache(module_name): + module = self.return_module(module_name) + self.assertIn(module_name, sys.modules) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, module_name) + + def test_reload(self): + # Test that a module is reused if already in sys.modules. + class FakeLoader: + def is_package(self, name): + return True + @self.module_for_loader + def load_module(self, module): + return module + name = 'a.b.c' + module = types.ModuleType('a.b.c') + module.__loader__ = 42 + module.__package__ = 42 + with util.uncache(name): + sys.modules[name] = module + loader = FakeLoader() + returned_module = loader.load_module(name) + self.assertIs(returned_module, sys.modules[name]) + self.assertEqual(module.__loader__, loader) + self.assertEqual(module.__package__, name) + + def test_new_module_failure(self): + # Test that a module is removed from sys.modules if added but an + # exception is raised. + name = 'a.b.c' + with util.uncache(name): + self.raise_exception(name) + self.assertNotIn(name, sys.modules) + + def test_reload_failure(self): + # Test that a failure on reload leaves the module in-place. + name = 'a.b.c' + module = types.ModuleType(name) + with util.uncache(name): + sys.modules[name] = module + self.raise_exception(name) + self.assertIs(module, sys.modules[name]) + + def test_decorator_attrs(self): + def fxn(self, module): pass + wrapped = self.module_for_loader(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + + def test_false_module(self): + # If for some odd reason a module is considered false, still return it + # from sys.modules. + class FalseModule(types.ModuleType): + def __bool__(self): return False + + name = 'mod' + module = FalseModule(name) + with util.uncache(name): + self.assertFalse(module) + sys.modules[name] = module + given = self.return_module(name) + self.assertIs(given, module) + + def test_attributes_set(self): + # __name__, __loader__, and __package__ should be set (when + # is_package() is defined; undefined implicitly tested elsewhere). + class FakeLoader: + def __init__(self, is_package): + self._pkg = is_package + def is_package(self, name): + return self._pkg + @self.module_for_loader + def load_module(self, module): + return module + + name = 'pkg.mod' + with util.uncache(name): + loader = FakeLoader(False) + module = loader.load_module(name) + self.assertEqual(module.__name__, name) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__package__, 'pkg') + + name = 'pkg.sub' + with util.uncache(name): + loader = FakeLoader(True) + module = loader.load_module(name) + self.assertEqual(module.__name__, name) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__package__, name) + + +(Frozen_ModuleForLoaderTests, + Source_ModuleForLoaderTests + ) = util.test_both(ModuleForLoaderTests, util=importlib_util) + + +class SetPackageTests: + + """Tests for importlib.util.set_package.""" + + def verify(self, module, expect): + """Verify the module has the expected value for __package__ after + passing through set_package.""" + fxn = lambda: module + wrapped = self.util.set_package(fxn) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + wrapped() + self.assertTrue(hasattr(module, '__package__')) + self.assertEqual(expect, module.__package__) + + def test_top_level(self): + # __package__ should be set to the empty string if a top-level module. + # Implicitly tests when package is set to None. + module = types.ModuleType('module') + module.__package__ = None + self.verify(module, '') + + def test_package(self): + # Test setting __package__ for a package. + module = types.ModuleType('pkg') + module.__path__ = [''] + module.__package__ = None + self.verify(module, 'pkg') + + def test_submodule(self): + # Test __package__ for a module in a package. + module = types.ModuleType('pkg.mod') + module.__package__ = None + self.verify(module, 'pkg') + + def test_setting_if_missing(self): + # __package__ should be set if it is missing. + module = types.ModuleType('mod') + if hasattr(module, '__package__'): + delattr(module, '__package__') + self.verify(module, '') + + def test_leaving_alone(self): + # If __package__ is set and not None then leave it alone. + for value in (True, False): + module = types.ModuleType('mod') + module.__package__ = value + self.verify(module, value) + + def test_decorator_attrs(self): + def fxn(module): pass + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + wrapped = self.util.set_package(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + + +(Frozen_SetPackageTests, + Source_SetPackageTests + ) = util.test_both(SetPackageTests, util=importlib_util) + + +class SetLoaderTests: + + """Tests importlib.util.set_loader().""" + + @property + def DummyLoader(self): + # Set DummyLoader on the class lazily. + class DummyLoader: + @self.util.set_loader + def load_module(self, module): + return self.module + self.__class__.DummyLoader = DummyLoader + return DummyLoader + + def test_no_attribute(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + try: + del loader.module.__loader__ + except AttributeError: + pass + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual(loader, loader.load_module('blah').__loader__) + + def test_attribute_is_None(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + loader.module.__loader__ = None + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual(loader, loader.load_module('blah').__loader__) + + def test_not_reset(self): + loader = self.DummyLoader() + loader.module = types.ModuleType('blah') + loader.module.__loader__ = 42 + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertEqual(42, loader.load_module('blah').__loader__) + + +(Frozen_SetLoaderTests, + Source_SetLoaderTests + ) = util.test_both(SetLoaderTests, util=importlib_util) + + +class ResolveNameTests: + + """Tests importlib.util.resolve_name().""" + + def test_absolute(self): + # bacon + self.assertEqual('bacon', self.util.resolve_name('bacon', None)) + + def test_absolute_within_package(self): + # bacon in spam + self.assertEqual('bacon', self.util.resolve_name('bacon', 'spam')) + + def test_no_package(self): + # .bacon in '' + with self.assertRaises(ValueError): + self.util.resolve_name('.bacon', '') + + def test_in_package(self): + # .bacon in spam + self.assertEqual('spam.eggs.bacon', + self.util.resolve_name('.bacon', 'spam.eggs')) + + def test_other_package(self): + # ..bacon in spam.bacon + self.assertEqual('spam.bacon', + self.util.resolve_name('..bacon', 'spam.eggs')) + + def test_escape(self): + # ..bacon in spam + with self.assertRaises(ValueError): + self.util.resolve_name('..bacon', 'spam') + + +(Frozen_ResolveNameTests, + Source_ResolveNameTests + ) = util.test_both(ResolveNameTests, util=importlib_util) + + +class FindSpecTests: + + class FakeMetaFinder: + @staticmethod + def find_spec(name, path=None, target=None): return name, path, target + + def test_sys_modules(self): + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + loader = 'a loader!' + spec = self.machinery.ModuleSpec(name, loader) + module.__loader__ = loader + module.__spec__ = spec + sys.modules[name] = module + found = self.util.find_spec(name) + self.assertEqual(found, spec) + + def test_sys_modules_without___loader__(self): + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + del module.__loader__ + loader = 'a loader!' + spec = self.machinery.ModuleSpec(name, loader) + module.__spec__ = spec + sys.modules[name] = module + found = self.util.find_spec(name) + self.assertEqual(found, spec) + + def test_sys_modules_spec_is_None(self): + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + module.__spec__ = None + sys.modules[name] = module + with self.assertRaises(ValueError): + self.util.find_spec(name) + + def test_sys_modules_loader_is_None(self): + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + spec = self.machinery.ModuleSpec(name, None) + module.__spec__ = spec + sys.modules[name] = module + found = self.util.find_spec(name) + self.assertEqual(found, spec) + + def test_sys_modules_spec_is_not_set(self): + name = 'some_mod' + with util.uncache(name): + module = types.ModuleType(name) + try: + del module.__spec__ + except AttributeError: + pass + sys.modules[name] = module + with self.assertRaises(ValueError): + self.util.find_spec(name) + + def test_success(self): + name = 'some_mod' + with util.uncache(name): + with util.import_state(meta_path=[self.FakeMetaFinder]): + self.assertEqual((name, None, None), + self.util.find_spec(name)) + + def test_nothing(self): + # None is returned upon failure to find a loader. + self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule')) + + def test_find_submodule(self): + name = 'spam' + subname = 'ham' + with util.temp_module(name, pkg=True) as pkg_dir: + fullname, _ = util.submodule(name, subname, pkg_dir) + spec = self.util.find_spec(fullname) + self.assertIsNot(spec, None) + self.assertIn(name, sorted(sys.modules)) + self.assertNotIn(fullname, sorted(sys.modules)) + # Ensure successive calls behave the same. + spec_again = self.util.find_spec(fullname) + self.assertEqual(spec_again, spec) + + def test_find_submodule_parent_already_imported(self): + name = 'spam' + subname = 'ham' + with util.temp_module(name, pkg=True) as pkg_dir: + self.init.import_module(name) + fullname, _ = util.submodule(name, subname, pkg_dir) + spec = self.util.find_spec(fullname) + self.assertIsNot(spec, None) + self.assertIn(name, sorted(sys.modules)) + self.assertNotIn(fullname, sorted(sys.modules)) + # Ensure successive calls behave the same. + spec_again = self.util.find_spec(fullname) + self.assertEqual(spec_again, spec) + + def test_find_relative_module(self): + name = 'spam' + subname = 'ham' + with util.temp_module(name, pkg=True) as pkg_dir: + fullname, _ = util.submodule(name, subname, pkg_dir) + relname = '.' + subname + spec = self.util.find_spec(relname, name) + self.assertIsNot(spec, None) + self.assertIn(name, sorted(sys.modules)) + self.assertNotIn(fullname, sorted(sys.modules)) + # Ensure successive calls behave the same. + spec_again = self.util.find_spec(fullname) + self.assertEqual(spec_again, spec) + + def test_find_relative_module_missing_package(self): + name = 'spam' + subname = 'ham' + with util.temp_module(name, pkg=True) as pkg_dir: + fullname, _ = util.submodule(name, subname, pkg_dir) + relname = '.' + subname + with self.assertRaises(ValueError): + self.util.find_spec(relname) + self.assertNotIn(name, sorted(sys.modules)) + self.assertNotIn(fullname, sorted(sys.modules)) + + def test_find_submodule_in_module(self): + # ModuleNotFoundError raised when a module is specified as + # a parent instead of a package. + with self.assertRaises(ModuleNotFoundError): + self.util.find_spec('module.name') + + +(Frozen_FindSpecTests, + Source_FindSpecTests + ) = util.test_both(FindSpecTests, init=init, util=importlib_util, + machinery=machinery) + + +class MagicNumberTests: + + def test_length(self): + # Should be 4 bytes. + self.assertEqual(len(self.util.MAGIC_NUMBER), 4) + + def test_incorporates_rn(self): + # The magic number uses \r\n to come out wrong when splitting on lines. + self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) + + +(Frozen_MagicNumberTests, + Source_MagicNumberTests + ) = util.test_both(MagicNumberTests, util=importlib_util) + + +class PEP3147Tests: + + """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" + + tag = sys.implementation.cache_tag + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag not be None') + def test_cache_from_source(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyc file (i.e. under __pycache__). + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + def test_cache_from_source_no_cache_tag(self): + # No cache tag means NotImplementedError. + with support.swap_attr(sys.implementation, 'cache_tag', None): + with self.assertRaises(NotImplementedError): + self.util.cache_from_source('whatever.py') + + def test_cache_from_source_no_dot(self): + # Directory with a dot, filename without dot. + path = os.path.join('foo.bar', 'file') + expect = os.path.join('foo.bar', '__pycache__', + 'file{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + def test_cache_from_source_debug_override(self): + # Given the path to a .py file, return the path to its PEP 3147/PEP 488 + # defined .pyc file (i.e. under __pycache__). + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.assertEqual(self.util.cache_from_source(path, False), + self.util.cache_from_source(path, optimization=1)) + self.assertEqual(self.util.cache_from_source(path, True), + self.util.cache_from_source(path, optimization='')) + with warnings.catch_warnings(): + warnings.simplefilter('error') + with self.assertRaises(DeprecationWarning): + self.util.cache_from_source(path, False) + with self.assertRaises(DeprecationWarning): + self.util.cache_from_source(path, True) + + def test_cache_from_source_cwd(self): + path = 'foo.py' + expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + def test_cache_from_source_override(self): + # When debug_override is not None, it can be any true-ish or false-ish + # value. + path = os.path.join('foo', 'bar', 'baz.py') + # However if the bool-ishness can't be determined, the exception + # propagates. + class Bearish: + def __bool__(self): raise RuntimeError + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.assertEqual(self.util.cache_from_source(path, []), + self.util.cache_from_source(path, optimization=1)) + self.assertEqual(self.util.cache_from_source(path, [17]), + self.util.cache_from_source(path, optimization='')) + with self.assertRaises(RuntimeError): + self.util.cache_from_source('/foo/bar/baz.py', Bearish()) + + + def test_cache_from_source_optimization_empty_string(self): + # Setting 'optimization' to '' leads to no optimization tag (PEP 488). + path = 'foo.py' + expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + def test_cache_from_source_optimization_None(self): + # Setting 'optimization' to None uses the interpreter's optimization. + # (PEP 488) + path = 'foo.py' + optimization_level = sys.flags.optimize + almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag)) + if optimization_level == 0: + expect = almost_expect + '.pyc' + elif optimization_level <= 2: + expect = almost_expect + '.opt-{}.pyc'.format(optimization_level) + else: + msg = '{!r} is a non-standard optimization level'.format(optimization_level) + self.skipTest(msg) + self.assertEqual(self.util.cache_from_source(path, optimization=None), + expect) + + def test_cache_from_source_optimization_set(self): + # The 'optimization' parameter accepts anything that has a string repr + # that passes str.alnum(). + path = 'foo.py' + valid_characters = string.ascii_letters + string.digits + almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag)) + got = self.util.cache_from_source(path, optimization=valid_characters) + # Test all valid characters are accepted. + self.assertEqual(got, + almost_expect + '.opt-{}.pyc'.format(valid_characters)) + # str() should be called on argument. + self.assertEqual(self.util.cache_from_source(path, optimization=42), + almost_expect + '.opt-42.pyc') + # Invalid characters raise ValueError. + with self.assertRaises(ValueError): + self.util.cache_from_source(path, optimization='path/is/bad') + + def test_cache_from_source_debug_override_optimization_both_set(self): + # Can only set one of the optimization-related parameters. + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + with self.assertRaises(TypeError): + self.util.cache_from_source('foo.py', False, optimization='') + + @unittest.skipUnless(os.sep == '\\' and os.altsep == '/', + 'test meaningful only where os.altsep is defined') + def test_sep_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where sep is right of altsep. + self.assertEqual( + self.util.cache_from_source('\\foo\\bar\\baz/qux.py', optimization=''), + '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag not be None') + def test_cache_from_source_path_like_arg(self): + path = pathlib.PurePath('foo', 'bar', 'baz', 'qux.py') + expect = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + self.assertEqual(self.util.cache_from_source(path, optimization=''), + expect) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_source_from_cache(self): + # Given the path to a PEP 3147 defined .pyc file, return the path to + # its source. This tests the good path. + path = os.path.join('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + self.assertEqual(self.util.source_from_cache(path), expect) + + def test_source_from_cache_no_cache_tag(self): + # If sys.implementation.cache_tag is None, raise NotImplementedError. + path = os.path.join('blah', '__pycache__', 'whatever.pyc') + with support.swap_attr(sys.implementation, 'cache_tag', None): + with self.assertRaises(NotImplementedError): + self.util.source_from_cache(path) + + def test_source_from_cache_bad_path(self): + # When the path to a pyc file is not in PEP 3147 format, a ValueError + # is raised. + self.assertRaises( + ValueError, self.util.source_from_cache, '/foo/bar/bazqux.pyc') + + def test_source_from_cache_no_slash(self): + # No slashes at all in path -> ValueError + self.assertRaises( + ValueError, self.util.source_from_cache, 'foo.cpython-32.pyc') + + def test_source_from_cache_too_few_dots(self): + # Too few dots in final path component -> ValueError + self.assertRaises( + ValueError, self.util.source_from_cache, '__pycache__/foo.pyc') + + def test_source_from_cache_too_many_dots(self): + with self.assertRaises(ValueError): + self.util.source_from_cache( + '__pycache__/foo.cpython-32.opt-1.foo.pyc') + + def test_source_from_cache_not_opt(self): + # Non-`opt-` path component -> ValueError + self.assertRaises( + ValueError, self.util.source_from_cache, + '__pycache__/foo.cpython-32.foo.pyc') + + def test_source_from_cache_no__pycache__(self): + # Another problem with the path -> ValueError + self.assertRaises( + ValueError, self.util.source_from_cache, + '/foo/bar/foo.cpython-32.foo.pyc') + + def test_source_from_cache_optimized_bytecode(self): + # Optimized bytecode is not an issue. + path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag)) + self.assertEqual(self.util.source_from_cache(path), 'foo.py') + + def test_source_from_cache_missing_optimization(self): + # An empty optimization level is a no-no. + path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag)) + with self.assertRaises(ValueError): + self.util.source_from_cache(path) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_source_from_cache_path_like_arg(self): + path = pathlib.PurePath('foo', 'bar', 'baz', '__pycache__', + 'qux.{}.pyc'.format(self.tag)) + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + self.assertEqual(self.util.source_from_cache(path), expect) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_respects_pycache_prefix(self): + # If pycache_prefix is set, cache_from_source will return a bytecode + # path inside that directory (in a subdirectory mirroring the .py file's + # path) rather than in a __pycache__ dir next to the py file. + pycache_prefixes = [ + os.path.join(os.path.sep, 'tmp', 'bytecode'), + os.path.join(os.path.sep, 'tmp', '\u2603'), # non-ASCII in path! + os.path.join(os.path.sep, 'tmp', 'trailing-slash') + os.path.sep, + ] + drive = '' + if os.name == 'nt': + drive = 'C:' + pycache_prefixes = [ + f'{drive}{prefix}' for prefix in pycache_prefixes] + pycache_prefixes += [r'\\?\C:\foo', r'\\localhost\c$\bar'] + for pycache_prefix in pycache_prefixes: + with self.subTest(path=pycache_prefix): + path = drive + os.path.join( + os.path.sep, 'foo', 'bar', 'baz', 'qux.py') + expect = os.path.join( + pycache_prefix, 'foo', 'bar', 'baz', + 'qux.{}.pyc'.format(self.tag)) + with util.temporary_pycache_prefix(pycache_prefix): + self.assertEqual( + self.util.cache_from_source(path, optimization=''), + expect) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_respects_pycache_prefix_relative(self): + # If the .py path we are given is relative, we will resolve to an + # absolute path before prefixing with pycache_prefix, to avoid any + # possible ambiguity. + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = os.path.join('foo', 'bar', 'baz', 'qux.py') + root = os.path.splitdrive(os.getcwd())[0] + os.path.sep + expect = os.path.join( + pycache_prefix, + os.path.relpath(os.getcwd(), root), + 'foo', 'bar', 'baz', f'qux.{self.tag}.pyc') + with util.temporary_pycache_prefix(pycache_prefix): + self.assertEqual( + self.util.cache_from_source(path, optimization=''), + expect) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_source_from_cache_inside_pycache_prefix(self): + # If pycache_prefix is set and the cache path we get is inside it, + # we return an absolute path to the py file based on the remainder of + # the path within pycache_prefix. + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = os.path.join(pycache_prefix, 'foo', 'bar', 'baz', + f'qux.{self.tag}.pyc') + expect = os.path.join(os.path.sep, 'foo', 'bar', 'baz', 'qux.py') + with util.temporary_pycache_prefix(pycache_prefix): + self.assertEqual(self.util.source_from_cache(path), expect) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_source_from_cache_outside_pycache_prefix(self): + # If pycache_prefix is set but the cache path we get is not inside + # it, just ignore it and handle the cache path according to the default + # behavior. + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = os.path.join('foo', 'bar', 'baz', '__pycache__', + f'qux.{self.tag}.pyc') + expect = os.path.join('foo', 'bar', 'baz', 'qux.py') + with util.temporary_pycache_prefix(pycache_prefix): + self.assertEqual(self.util.source_from_cache(path), expect) + + +(Frozen_PEP3147Tests, + Source_PEP3147Tests + ) = util.test_both(PEP3147Tests, util=importlib_util) + + +class MagicNumberTests(unittest.TestCase): + """ + Test release compatibility issues relating to importlib + """ + @unittest.skipUnless( + sys.version_info.releaselevel in ('candidate', 'final'), + 'only applies to candidate or final python release levels' + ) + def test_magic_number(self): + """ + Each python minor release should generally have a MAGIC_NUMBER + that does not change once the release reaches candidate status. + + Once a release reaches candidate status, the value of the constant + EXPECTED_MAGIC_NUMBER in this test should be changed. + This test will then check that the actual MAGIC_NUMBER matches + the expected value for the release. + + In exceptional cases, it may be required to change the MAGIC_NUMBER + for a maintenance release. In this case the change should be + discussed in python-dev. If a change is required, community + stakeholders such as OS package maintainers must be notified + in advance. Such exceptional releases will then require an + adjustment to this test case. + """ + EXPECTED_MAGIC_NUMBER = 3410 + actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') + + msg = ( + "To avoid breaking backwards compatibility with cached bytecode " + "files that can't be automatically regenerated by the current " + "user, candidate and final releases require the current " + "importlib.util.MAGIC_NUMBER to match the expected " + "magic number in this test. Set the expected " + "magic number in this test to the current MAGIC_NUMBER to " + "continue with the release.\n\n" + "Changing the MAGIC_NUMBER for a maintenance release " + "requires discussion in python-dev and notification of " + "community stakeholders." + ) + self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py new file mode 100644 index 000000000..005b685cc --- /dev/null +++ b/Lib/test/test_importlib/test_windows.py @@ -0,0 +1,109 @@ +from . import util as test_util +machinery = test_util.import_importlib('importlib.machinery') + +import os +import re +import sys +import unittest +from test import support +from distutils.util import get_platform +from contextlib import contextmanager +from .util import temp_module + +support.import_module('winreg', required_on=['win']) +from winreg import ( + CreateKey, HKEY_CURRENT_USER, + SetValue, REG_SZ, KEY_ALL_ACCESS, + EnumKey, CloseKey, DeleteKey, OpenKey +) + +def delete_registry_tree(root, subkey): + try: + hkey = OpenKey(root, subkey, access=KEY_ALL_ACCESS) + except OSError: + # subkey does not exist + return + while True: + try: + subsubkey = EnumKey(hkey, 0) + except OSError: + # no more subkeys + break + delete_registry_tree(hkey, subsubkey) + CloseKey(hkey) + DeleteKey(root, subkey) + +@contextmanager +def setup_module(machinery, name, path=None): + if machinery.WindowsRegistryFinder.DEBUG_BUILD: + root = machinery.WindowsRegistryFinder.REGISTRY_KEY_DEBUG + else: + root = machinery.WindowsRegistryFinder.REGISTRY_KEY + key = root.format(fullname=name, + sys_version='%d.%d' % sys.version_info[:2]) + try: + with temp_module(name, "a = 1") as location: + subkey = CreateKey(HKEY_CURRENT_USER, key) + if path is None: + path = location + ".py" + SetValue(subkey, "", REG_SZ, path) + yield + finally: + if machinery.WindowsRegistryFinder.DEBUG_BUILD: + key = os.path.dirname(key) + delete_registry_tree(HKEY_CURRENT_USER, key) + + +@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') +class WindowsRegistryFinderTests: + # The module name is process-specific, allowing for + # simultaneous runs of the same test on a single machine. + test_module = "spamham{}".format(os.getpid()) + + def test_find_spec_missing(self): + spec = self.machinery.WindowsRegistryFinder.find_spec('spam') + self.assertIs(spec, None) + + def test_find_module_missing(self): + loader = self.machinery.WindowsRegistryFinder.find_module('spam') + self.assertIs(loader, None) + + def test_module_found(self): + with setup_module(self.machinery, self.test_module): + loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + self.assertIsNot(loader, None) + self.assertIsNot(spec, None) + + def test_module_not_found(self): + with setup_module(self.machinery, self.test_module, path="."): + loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + self.assertIsNone(loader) + self.assertIsNone(spec) + +(Frozen_WindowsRegistryFinderTests, + Source_WindowsRegistryFinderTests + ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery) + +@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') +class WindowsExtensionSuffixTests: + def test_tagged_suffix(self): + suffixes = self.machinery.EXTENSION_SUFFIXES + expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info, + re.sub('[^a-zA-Z0-9]', '_', get_platform())) + try: + untagged_i = suffixes.index(".pyd") + except ValueError: + untagged_i = suffixes.index("_d.pyd") + expected_tag = "_d" + expected_tag + + self.assertIn(expected_tag, suffixes) + + # Ensure the tags are in the correct order + tagged_i = suffixes.index(expected_tag) + self.assertLess(tagged_i, untagged_i) + +(Frozen_WindowsExtensionSuffixTests, + Source_WindowsExtensionSuffixTests + ) = test_util.test_both(WindowsExtensionSuffixTests, machinery=machinery) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py new file mode 100644 index 000000000..196ea1c9d --- /dev/null +++ b/Lib/test/test_importlib/util.py @@ -0,0 +1,569 @@ +import abc +import builtins +import contextlib +import errno +import functools +import importlib +from importlib import machinery, util, invalidate_caches +from importlib.abc import ResourceReader +import io +import os +import os.path +from pathlib import Path, PurePath +from test import support +import unittest +import sys +import tempfile +import types + +from . import data01 +from . import zipdata01 + + +BUILTINS = types.SimpleNamespace() +BUILTINS.good_name = None +BUILTINS.bad_name = None +if 'errno' in sys.builtin_module_names: + BUILTINS.good_name = 'errno' +if 'importlib' not in sys.builtin_module_names: + BUILTINS.bad_name = 'importlib' + +EXTENSIONS = types.SimpleNamespace() +EXTENSIONS.path = None +EXTENSIONS.ext = None +EXTENSIONS.filename = None +EXTENSIONS.file_path = None +EXTENSIONS.name = '_testcapi' + +def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return + +_extension_details() + + +def import_importlib(module_name): + """Import a module from importlib both w/ and w/o _frozen_importlib.""" + fresh = ('importlib',) if '.' in module_name else () + frozen = support.import_fresh_module(module_name) + source = support.import_fresh_module(module_name, fresh=fresh, + blocked=('_frozen_importlib', '_frozen_importlib_external')) + return {'Frozen': frozen, 'Source': source} + + +def specialize_class(cls, kind, base=None, **kwargs): + # XXX Support passing in submodule names--load (and cache) them? + # That would clean up the test modules a bit more. + if base is None: + base = unittest.TestCase + elif not isinstance(base, type): + base = base[kind] + name = '{}_{}'.format(kind, cls.__name__) + bases = (cls, base) + specialized = types.new_class(name, bases) + specialized.__module__ = cls.__module__ + specialized._NAME = cls.__name__ + specialized._KIND = kind + for attr, values in kwargs.items(): + value = values[kind] + setattr(specialized, attr, value) + return specialized + + +def split_frozen(cls, base=None, **kwargs): + frozen = specialize_class(cls, 'Frozen', base, **kwargs) + source = specialize_class(cls, 'Source', base, **kwargs) + return frozen, source + + +def test_both(test_class, base=None, **kwargs): + return split_frozen(test_class, base, **kwargs) + + +CASE_INSENSITIVE_FS = True +# Windows is the only OS that is *always* case-insensitive +# (OS X *can* be case-sensitive). +if sys.platform not in ('win32', 'cygwin'): + changed_name = __file__.upper() + if changed_name == __file__: + changed_name = __file__.lower() + if not os.path.exists(changed_name): + CASE_INSENSITIVE_FS = False + +source_importlib = import_importlib('importlib')['Source'] +__import__ = {'Frozen': staticmethod(builtins.__import__), + 'Source': staticmethod(source_importlib.__import__)} + + +def case_insensitive_tests(test): + """Class decorator that nullifies tests requiring a case-insensitive + file system.""" + return unittest.skipIf(not CASE_INSENSITIVE_FS, + "requires a case-insensitive filesystem")(test) + + +def submodule(parent, name, pkg_dir, content=''): + path = os.path.join(pkg_dir, name + '.py') + with open(path, 'w') as subfile: + subfile.write(content) + return '{}.{}'.format(parent, name), path + + +@contextlib.contextmanager +def uncache(*names): + """Uncache a module from sys.modules. + + A basic sanity check is performed to prevent uncaching modules that either + cannot/shouldn't be uncached. + + """ + for name in names: + if name in ('sys', 'marshal', 'imp'): + raise ValueError( + "cannot uncache {0}".format(name)) + try: + del sys.modules[name] + except KeyError: + pass + try: + yield + finally: + for name in names: + try: + del sys.modules[name] + except KeyError: + pass + + +@contextlib.contextmanager +def temp_module(name, content='', *, pkg=False): + conflicts = [n for n in sys.modules if n.partition('.')[0] == name] + with support.temp_cwd(None) as cwd: + with uncache(name, *conflicts): + with support.DirsOnSysPath(cwd): + invalidate_caches() + + location = os.path.join(cwd, name) + if pkg: + modpath = os.path.join(location, '__init__.py') + os.mkdir(name) + else: + modpath = location + '.py' + if content is None: + # Make sure the module file gets created. + content = '' + if content is not None: + # not a namespace package + with open(modpath, 'w') as modfile: + modfile.write(content) + yield location + + +@contextlib.contextmanager +def import_state(**kwargs): + """Context manager to manage the various importers and stored state in the + sys module. + + The 'modules' attribute is not supported as the interpreter state stores a + pointer to the dict that the interpreter uses internally; + reassigning to sys.modules does not have the desired effect. + + """ + originals = {} + try: + for attr, default in (('meta_path', []), ('path', []), + ('path_hooks', []), + ('path_importer_cache', {})): + originals[attr] = getattr(sys, attr) + if attr in kwargs: + new_value = kwargs[attr] + del kwargs[attr] + else: + new_value = default + setattr(sys, attr, new_value) + if len(kwargs): + raise ValueError( + 'unrecognized arguments: {0}'.format(kwargs.keys())) + yield + finally: + for attr, value in originals.items(): + setattr(sys, attr, value) + + +class _ImporterMock: + + """Base class to help with creating importer mocks.""" + + def __init__(self, *names, module_code={}): + self.modules = {} + self.module_code = {} + for name in names: + if not name.endswith('.__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + if '.' not in name: + package = None + elif import_name == name: + package = name.rsplit('.', 1)[0] + else: + package = import_name + module = types.ModuleType(import_name) + module.__loader__ = self + module.__file__ = '' + module.__package__ = package + module.attr = name + if import_name != name: + module.__path__ = [''] + self.modules[import_name] = module + if import_name in module_code: + self.module_code[import_name] = module_code[import_name] + + def __getitem__(self, name): + return self.modules[name] + + def __enter__(self): + self._uncache = uncache(*self.modules.keys()) + self._uncache.__enter__() + return self + + def __exit__(self, *exc_info): + self._uncache.__exit__(None, None, None) + + +class mock_modules(_ImporterMock): + + """Importer mock using PEP 302 APIs.""" + + def find_module(self, fullname, path=None): + if fullname not in self.modules: + return None + else: + return self + + def load_module(self, fullname): + if fullname not in self.modules: + raise ImportError + else: + sys.modules[fullname] = self.modules[fullname] + if fullname in self.module_code: + try: + self.module_code[fullname]() + except Exception: + del sys.modules[fullname] + raise + return self.modules[fullname] + + +class mock_spec(_ImporterMock): + + """Importer mock using PEP 451 APIs.""" + + def find_spec(self, fullname, path=None, parent=None): + try: + module = self.modules[fullname] + except KeyError: + return None + spec = util.spec_from_file_location( + fullname, module.__file__, loader=self, + submodule_search_locations=getattr(module, '__path__', None)) + return spec + + def create_module(self, spec): + if spec.name not in self.modules: + raise ImportError + return self.modules[spec.name] + + def exec_module(self, module): + try: + self.module_code[module.__spec__.name]() + except KeyError: + pass + + +def writes_bytecode_files(fxn): + """Decorator to protect sys.dont_write_bytecode from mutation and to skip + tests that require it to be set to False.""" + if sys.dont_write_bytecode: + return lambda *args, **kwargs: None + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + original = sys.dont_write_bytecode + sys.dont_write_bytecode = False + try: + to_return = fxn(*args, **kwargs) + finally: + sys.dont_write_bytecode = original + return to_return + return wrapper + + +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +@contextlib.contextmanager +def temporary_pycache_prefix(prefix): + """Adjust and restore sys.pycache_prefix.""" + _orig_prefix = sys.pycache_prefix + sys.pycache_prefix = prefix + try: + yield + finally: + sys.pycache_prefix = _orig_prefix + + +@contextlib.contextmanager +def create_modules(*names): + """Temporarily create each named module with an attribute (named 'attr') + that contains the name passed into the context manager that caused the + creation of the module. + + All files are created in a temporary directory returned by + tempfile.mkdtemp(). This directory is inserted at the beginning of + sys.path. When the context manager exits all created files (source and + bytecode) are explicitly deleted. + + No magic is performed when creating packages! This means that if you create + a module within a package you must also create the package's __init__ as + well. + + """ + source = 'attr = {0!r}' + created_paths = [] + mapping = {} + state_manager = None + uncache_manager = None + try: + temp_dir = tempfile.mkdtemp() + mapping['.root'] = temp_dir + import_names = set() + for name in names: + if not name.endswith('__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + import_names.add(import_name) + if import_name in sys.modules: + del sys.modules[import_name] + name_parts = name.split('.') + file_path = temp_dir + for directory in name_parts[:-1]: + file_path = os.path.join(file_path, directory) + if not os.path.exists(file_path): + os.mkdir(file_path) + created_paths.append(file_path) + file_path = os.path.join(file_path, name_parts[-1] + '.py') + with open(file_path, 'w') as file: + file.write(source.format(name)) + created_paths.append(file_path) + mapping[name] = file_path + uncache_manager = uncache(*import_names) + uncache_manager.__enter__() + state_manager = import_state(path=[temp_dir]) + state_manager.__enter__() + yield mapping + finally: + if state_manager is not None: + state_manager.__exit__(None, None, None) + if uncache_manager is not None: + uncache_manager.__exit__(None, None, None) + support.rmtree(temp_dir) + + +def mock_path_hook(*entries, importer): + """A mock sys.path_hooks entry.""" + def hook(entry): + if entry not in entries: + raise ImportError + return importer + return hook + + +class CASEOKTestBase: + + def caseok_env_changed(self, *, should_exist): + possibilities = b'PYTHONCASEOK', 'PYTHONCASEOK' + if any(x in self.importlib._bootstrap_external._os.environ + for x in possibilities) != should_exist: + self.skipTest('os.environ changes not reflected in _os.environ') + + +def create_package(file, path, is_package=True, contents=()): + class Reader(ResourceReader): + def get_resource_reader(self, package): + return self + + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + def is_resource(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + for entry in contents: + parts = entry.split('/') + if len(parts) == 1 and parts[0] == path_: + return True + return False + + def contents(self): + if isinstance(path, Exception): + raise path + # There's no yield from in baseball, er, Python 2. + for entry in contents: + yield entry + + name = 'testingpackage' + # Unforunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + loader = Reader() + spec = machinery.ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader + return module + + +class CommonResourceTests(abc.ABC): + @abc.abstractmethod + def execute(self, package, path): + raise NotImplementedError + + def test_package_name(self): + # Passing in the package name should succeed. + self.execute(data01.__name__, 'utf-8.file') + + def test_package_object(self): + # Passing in the package itself should succeed. + self.execute(data01, 'utf-8.file') + + def test_string_path(self): + # Passing in a string for the path should succeed. + path = 'utf-8.file' + self.execute(data01, path) + + @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support') + def test_pathlib_path(self): + # Passing in a pathlib.PurePath object for the path should succeed. + path = PurePath('utf-8.file') + self.execute(data01, path) + + def test_absolute_path(self): + # An absolute path is a ValueError. + path = Path(__file__) + full_path = path.parent/'utf-8.file' + with self.assertRaises(ValueError): + self.execute(data01, full_path) + + def test_relative_path(self): + # A reative path is a ValueError. + with self.assertRaises(ValueError): + self.execute(data01, '../data01/utf-8.file') + + def test_importing_module_as_side_effect(self): + # The anchor package can already be imported. + del sys.modules[data01.__name__] + self.execute(data01.__name__, 'utf-8.file') + + def test_non_package_by_name(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + self.execute(__name__, 'utf-8.file') + + def test_non_package_by_package(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + module = sys.modules['test.test_importlib.util'] + self.execute(module, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_opener(self): + bytes_data = io.BytesIO(b'Hello, world!') + package = create_package(file=bytes_data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_path(self): + bytes_data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=bytes_data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + + +class ZipSetupBase: + ZIP_MODULE = None + + @classmethod + def setUpClass(cls): + data_path = Path(cls.ZIP_MODULE.__file__) + data_dir = data_path.parent + cls._zip_path = str(data_dir / 'ziptestdata.zip') + sys.path.append(cls._zip_path) + cls.data = importlib.import_module('ziptestdata') + + @classmethod + def tearDownClass(cls): + try: + sys.path.remove(cls._zip_path) + except ValueError: + pass + + try: + del sys.path_importer_cache[cls._zip_path] + del sys.modules[cls.data.__name__] + except KeyError: + pass + + try: + del cls.data + del cls._zip_path + except AttributeError: + pass + + def setUp(self): + modules = support.modules_setup() + self.addCleanup(support.modules_cleanup, *modules) + + +class ZipSetup(ZipSetupBase): + ZIP_MODULE = zipdata01 # type: ignore diff --git a/Lib/test/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d8fa97f199cc29f6905404ea05f88926658ee2b GIT binary patch literal 876 zcmWIWW@Zs#0D-o$9&a!MN{9pLs?36t)Z&tq#F9k)`1s7c%#!$cy@JXB6ivcyJG)tc zia{7%Q&MJLVo{}DT4qiv10xeNs>aRTN;Wz`O(2Y}v9u&j*U${C@&7*$hE#?eAj}6U zRbU8Z2w`wz&}A@WFaz?+fucn~xfBLP1}+9v3r_zyGz;W85EcVsWbc41umGFyk(!f} zucHte;-+h%P@Z3ulcLClY7S5<9H;&?E(z02lENC@Ig!?SR;Polaa10BMBT zX9RKpPypEhj7%cTxC0L88!*_?2%>NXBW^=L0SN Date: Sat, 6 Jun 2020 18:14:27 +0300 Subject: [PATCH 031/209] Mark unsupported py_compile tests --- .../test_importlib/extension/test_finder.py | 2 + .../test_importlib/extension/test_loader.py | 6 ++ Lib/test/test_importlib/frozen/test_finder.py | 8 +++ Lib/test/test_importlib/frozen/test_loader.py | 5 ++ .../import_/test___package__.py | 4 ++ .../test_importlib/import_/test_caching.py | 1 + .../test_importlib/import_/test_meta_path.py | 3 + Lib/test/test_importlib/import_/test_path.py | 2 + .../test_importlib/source/test_file_loader.py | 71 ++++++++++--------- Lib/test/test_importlib/source/test_finder.py | 1 + .../source/test_source_encoding.py | 6 ++ Lib/test/test_importlib/test_abc.py | 3 + Lib/test/test_importlib/test_api.py | 4 ++ Lib/test/test_importlib/test_lazy.py | 1 + Lib/test/test_importlib/test_locks.py | 13 ++-- Lib/test/test_importlib/test_open.py | 3 + Lib/test/test_importlib/test_path.py | 1 + Lib/test/test_importlib/test_read.py | 3 + Lib/test/test_importlib/test_resource.py | 1 + Lib/test/test_importlib/test_util.py | 6 +- Lib/test/test_importlib/util.py | 6 ++ 21 files changed, 111 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index c9b4a3772..89855d686 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -19,6 +19,8 @@ class FinderTests(abc.FinderTests): warnings.simplefilter('ignore', DeprecationWarning) return importer.find_module(fullname) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_module(self): self.assertTrue(self.find_module(util.EXTENSIONS.name)) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 9ad05fade..6707fc1ba 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -22,6 +22,7 @@ class LoaderTests(abc.LoaderTests): def load_module(self, fullname): return self.loader.load_module(fullname) + @unittest.skip("TODO: RUSTPYTHON") def test_load_module_API(self): # Test the default argument for load_module(). self.loader.load_module() @@ -39,6 +40,8 @@ class LoaderTests(abc.LoaderTests): util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_module(self): with util.uncache(util.EXTENSIONS.name): module = self.load_module(util.EXTENSIONS.name) @@ -56,6 +59,7 @@ class LoaderTests(abc.LoaderTests): # No extension module in a package available for testing. test_lacking_parent = None + @unittest.skip("TODO: RUSTPYTHON") def test_module_reuse(self): with util.uncache(util.EXTENSIONS.name): module1 = self.load_module(util.EXTENSIONS.name) @@ -71,6 +75,7 @@ class LoaderTests(abc.LoaderTests): self.load_module(name) self.assertEqual(cm.exception.name, name) + @unittest.skip("TODO: RUSTPYTHON") def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: @@ -82,6 +87,7 @@ class LoaderTests(abc.LoaderTests): Source_LoaderTests ) = util.test_both(LoaderTests, machinery=machinery) +@unittest.skip("TODO: RUSTPYTHON") class MultiPhaseExtensionModuleTests(abc.LoaderTests): """Test loading extension modules with multi-phase initialization (PEP 489) """ diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 519aa027d..4c224cc66 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -19,10 +19,14 @@ class FindSpecTests(abc.FinderTests): spec = self.find(name) self.assertEqual(spec.origin, 'frozen') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_package(self): spec = self.find('__phello__') self.assertIsNotNone(spec) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_module_in_package(self): spec = self.find('__phello__.spam', ['__phello__']) self.assertIsNotNone(spec) @@ -56,10 +60,14 @@ class FinderTests(abc.FinderTests): loader = self.find(name) self.assertTrue(hasattr(loader, 'load_module')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_package(self): loader = self.find('__phello__') self.assertTrue(hasattr(loader, 'load_module')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_module_in_package(self): loader = self.find('__phello__.spam', ['__phello__']) self.assertTrue(hasattr(loader, 'load_module')) diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 29ecff177..a8e55286a 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -34,6 +34,7 @@ class ExecModuleTests(abc.LoaderTests): self.assertEqual(output, 'Hello world!\n') self.assertTrue(hasattr(module, '__spec__')) + @unittest.skip("TODO: RUSTPYTHON") def test_package(self): name = '__phello__' module, output = self.exec_module(name) @@ -46,6 +47,7 @@ class ExecModuleTests(abc.LoaderTests): expected=value)) self.assertEqual(output, 'Hello world!\n') + @unittest.skip("TODO: RUSTPYTHON") def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): @@ -105,6 +107,7 @@ class LoaderTests(abc.LoaderTests): self.assertEqual(stdout.getvalue(), 'Hello world!\n') self.assertFalse(hasattr(module, '__file__')) + @unittest.skip("TODO: RUSTPYTHON") def test_package(self): with util.uncache('__phello__'), captured_stdout() as stdout: with warnings.catch_warnings(): @@ -123,6 +126,7 @@ class LoaderTests(abc.LoaderTests): self.assertEqual(stdout.getvalue(), 'Hello world!\n') self.assertFalse(hasattr(module, '__file__')) + @unittest.skip("TODO: RUSTPYTHON") def test_lacking_parent(self): with util.uncache('__phello__', '__phello__.spam'), \ captured_stdout() as stdout: @@ -200,6 +204,7 @@ class InspectLoaderTests: result = self.machinery.FrozenImporter.get_source('__hello__') self.assertIsNone(result) + @unittest.skip("TODO: RUSTPYTHON") def test_is_package(self): # Should be able to tell what is a package. test_for = (('__hello__', False), ('__phello__', True), diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 761b256b3..4b1766452 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -56,6 +56,8 @@ class Using__package__: '__path__': []}) self.assertEqual(module.__name__, 'pkg') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_warn_when_using___name__(self): with self.assertWarns(ImportWarning): self.import_module({'__name__': 'pkg.fake', '__path__': []}) @@ -73,6 +75,8 @@ class Using__package__: module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) self.assertEqual(module.__name__, 'pkg') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_warn_when_package_and_spec_disagree(self): # Raise an ImportWarning if __package__ != __spec__.parent. with self.assertWarns(ImportWarning): diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index 8079add5b..0630e10bd 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -44,6 +44,7 @@ class UseCache: ) = util.test_both(UseCache, __import__=util.__import__) +@unittest.skip("TODO: RUSTPYTHON") class ImportlibUseCache(UseCache, unittest.TestCase): # Pertinent only to PEP 302; exec_module() doesn't return a module. diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 5a41e8968..469684886 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -30,6 +30,8 @@ class CallingOrder: with util.import_state(meta_path=[first, second]): self.assertIs(self.__import__(mod_name), second.modules[mod_name]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_empty(self): # Raise an ImportWarning if sys.meta_path is empty. module_name = 'nothing' @@ -51,6 +53,7 @@ class CallingOrder: ) = util.test_both(CallingOrder, __import__=util.__import__) +@unittest.skip("TODO: RUSTPYTHON") class CallSignature: """If there is no __path__ entry on the parent module, then 'path' is None diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 18c81dda4..7beaee1ca 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -68,6 +68,8 @@ class FinderTests: self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_empty_path_hooks(self): # Test that if sys.path_hooks is empty a warning is raised, # sys.path_importer_cache gets None set, and PathFinder returns None. diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 3ffb2aa1e..79187e696 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -192,6 +192,7 @@ class SimpleTest(abc.LoaderTests): if os.path.exists(pycache): shutil.rmtree(pycache) + @unittest.skip("TODO: RUSTPYTHON") @util.writes_bytecode_files def test_timestamp_overflow(self): # When a modification timestamp is larger than 2**32, it should be @@ -238,6 +239,7 @@ class SimpleTest(abc.LoaderTests): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module('bad name') + @unittest.skip("TODO: RUSTPYTHON") @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -269,6 +271,7 @@ class SimpleTest(abc.LoaderTests): data[8:16], ) + @unittest.skip("TODO: RUSTPYTHON") @util.writes_bytecode_files def test_overridden_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ @@ -294,6 +297,7 @@ class SimpleTest(abc.LoaderTests): loader.exec_module(mod) self.assertEqual(mod.state, 'old') + @unittest.skip("TODO: RUSTPYTHON") @util.writes_bytecode_files def test_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -324,6 +328,7 @@ class SimpleTest(abc.LoaderTests): data[8:16], ) + @unittest.skip("TODO: RUSTPYTHON") @util.writes_bytecode_files def test_overiden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ @@ -671,28 +676,29 @@ class SourceLoaderBadBytecodeTest: os.chmod(bytecode_path, stat.S_IWUSR) -class SourceLoaderBadBytecodeTestPEP451( - SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): - pass +# TODO: RustPython +# class SourceLoaderBadBytecodeTestPEP451( +# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): +# pass -(Frozen_SourceBadBytecodePEP451, - Source_SourceBadBytecodePEP451 - ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) +# (Frozen_SourceBadBytecodePEP451, +# Source_SourceBadBytecodePEP451 +# ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, +# machinery=machinery, abc=importlib_abc, +# util=importlib_util) -class SourceLoaderBadBytecodeTestPEP302( - SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): - pass +# class SourceLoaderBadBytecodeTestPEP302( +# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): +# pass -(Frozen_SourceBadBytecodePEP302, - Source_SourceBadBytecodePEP302 - ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) +# (Frozen_SourceBadBytecodePEP302, +# Source_SourceBadBytecodePEP302 +# ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, +# machinery=machinery, abc=importlib_abc, +# util=importlib_util) class SourcelessLoaderBadBytecodeTest: @@ -769,28 +775,29 @@ class SourcelessLoaderBadBytecodeTest: self._test_non_code_marshal(del_source=True) -class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, - BadBytecodeTestPEP451): - pass +# TODO: RustPython +# class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, +# BadBytecodeTestPEP451): +# pass -(Frozen_SourcelessBadBytecodePEP451, - Source_SourcelessBadBytecodePEP451 - ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) +# (Frozen_SourcelessBadBytecodePEP451, +# Source_SourcelessBadBytecodePEP451 +# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, +# machinery=machinery, abc=importlib_abc, +# util=importlib_util) -class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, - BadBytecodeTestPEP302): - pass +# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, +# BadBytecodeTestPEP302): +# pass -(Frozen_SourcelessBadBytecodePEP302, - Source_SourcelessBadBytecodePEP302 - ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) +# (Frozen_SourcelessBadBytecodePEP302, +# Source_SourcelessBadBytecodePEP302 +# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, +# machinery=machinery, abc=importlib_abc, +# util=importlib_util) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index f372b850d..1a76fc02d 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -178,6 +178,7 @@ class FinderTests(abc.FinderTests): found = self._find(finder, 'doesnotexist') self.assertEqual(found, self.NOT_FOUND) + @unittest.skip("TODO: RUSTPYTHON") def test_ignore_file(self): # If a directory got changed to a file from underneath us, then don't # worry about looking for submodules. diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 980855fe1..f2cfa3c98 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -57,30 +57,36 @@ class EncodingTest: source = "{0}\na=42\n".format(encoding_line).encode("koi8-r") self.run_test(source) + @unittest.skip("TODO: RUSTPYTHON") # [default encoding] def test_default_encoding(self): self.run_test(self.source_line.encode('utf-8')) # [encoding first line] + @unittest.skip("TODO: RUSTPYTHON") def test_encoding_on_first_line(self): encoding = 'Latin-1' source = self.create_source(encoding) self.run_test(source) # [encoding second line] + @unittest.skip("TODO: RUSTPYTHON") def test_encoding_on_second_line(self): source = b"#/usr/bin/python\n" + self.create_source('Latin-1') self.run_test(source) + @unittest.skip("TODO: RUSTPYTHON") # [BOM] def test_bom(self): self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8')) + @unittest.skip("TODO: RUSTPYTHON") # [BOM and utf-8] def test_bom_and_utf_8(self): source = codecs.BOM_UTF8 + self.create_source('utf-8') self.run_test(source) + @unittest.skip("TODO: RUSTPYTHON") # [BOM conflict] def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 05608bbb8..11250ab06 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -789,6 +789,7 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): """ + @unittest.skip("TODO: RUSTPYTHON") def test_get_source(self): # Verify the source code is returned as a string. # If an OSError is raised by get_data then raise ImportError. @@ -843,6 +844,7 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): self.verify_module(module) self.assertFalse(hasattr(module, '__path__')) + @unittest.skip("TODO: RUSTPYTHON") def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise # specified by an encoding line. @@ -958,6 +960,7 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness): loader_mock=SPLIT_SL) +@unittest.skip("TODO: RUSTPYTHON") class SourceLoaderGetSourceTests: """Tests for importlib.abc.SourceLoader.get_source().""" diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index edb745c2c..7b4137652 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -242,6 +242,8 @@ class ReloadTests: self.assertIs(reloaded, types) self.assertIs(sys.modules['types'], types) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_reload_location_changed(self): name = 'spam' with support.temp_cwd(None) as cwd: @@ -293,6 +295,8 @@ class ReloadTests: self.maxDiff = None self.assertEqual(ns, expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_reload_namespace_changed(self): name = 'spam' with support.temp_cwd(None) as cwd: diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index 28608e95d..4b3be72f3 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -52,6 +52,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): self.loaded = module +@unittest.skip("TODO: RUSTPYTHON") class LazyLoaderTests(unittest.TestCase): def test_init(self): diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 21794d911..db494c339 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -132,10 +132,10 @@ class LifetimeTests: self.assertEqual(0, len(self.bootstrap._module_locks), self.bootstrap._module_locks) - -(Frozen_LifetimeTests, - Source_LifetimeTests - ) = test_util.test_both(LifetimeTests, init=init) +# TODO: RustPython +# (Frozen_LifetimeTests, +# Source_LifetimeTests +# ) = test_util.test_both(LifetimeTests, init=init) @support.reap_threads @@ -144,8 +144,9 @@ def test_main(): Source_ModuleLockAsRLockTests, Frozen_DeadlockAvoidanceTests, Source_DeadlockAvoidanceTests, - Frozen_LifetimeTests, - Source_LifetimeTests) + # Frozen_LifetimeTests, + # Source_LifetimeTests + ) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py index fd6e84b70..4c740095e 100644 --- a/Lib/test/test_importlib/test_open.py +++ b/Lib/test/test_importlib/test_open.py @@ -23,17 +23,20 @@ class OpenTests: result = fp.read() self.assertEqual(result, b'\x00\x01\x02\x03') + @unittest.skip("TODO: RUSTPYTHON") def test_open_text_default_encoding(self): with resources.open_text(self.data, 'utf-8.file') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-8 world!\n') + @unittest.skip("TODO: RUSTPYTHON") def test_open_text_given_encoding(self): with resources.open_text( self.data, 'utf-16.file', 'utf-16', 'strict') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-16 world!\n') + @unittest.skip("TODO: RUSTPYTHON") def test_open_text_with_errors(self): # Raises UnicodeError without the 'errors' argument. with resources.open_text( diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py index 2d3dcda7e..562a566e0 100644 --- a/Lib/test/test_importlib/test_path.py +++ b/Lib/test/test_importlib/test_path.py @@ -12,6 +12,7 @@ class CommonTests(util.CommonResourceTests, unittest.TestCase): class PathTests: + @unittest.skip("TODO: RUSTPYTHON") def test_reading(self): # Path should be readable. # Test also implicitly verifies the returned object is a pathlib.Path diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py index ff78d0b29..52cc24501 100644 --- a/Lib/test/test_importlib/test_read.py +++ b/Lib/test/test_importlib/test_read.py @@ -20,15 +20,18 @@ class ReadTests: result = resources.read_binary(self.data, 'binary.file') self.assertEqual(result, b'\0\1\2\3') + @unittest.skip("TODO: RUSTPYTHON") def test_read_text_default_encoding(self): result = resources.read_text(self.data, 'utf-8.file') self.assertEqual(result, 'Hello, UTF-8 world!\n') + @unittest.skip("TODO: RUSTPYTHON") def test_read_text_given_encoding(self): result = resources.read_text( self.data, 'utf-16.file', encoding='utf-16') self.assertEqual(result, 'Hello, UTF-16 world!\n') + @unittest.skip("TODO: RUSTPYTHON") def test_read_text_with_errors(self): # Raises UnicodeError without the 'errors' argument. self.assertRaises( diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index f88d92d15..4de6d9ac3 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -48,6 +48,7 @@ class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): pass +@unittest.skip("TODO: RUSTPYTHON") class ResourceLoaderTests(unittest.TestCase): def test_resource_contents(self): package = util.create_package( diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index db0899aff..df7125bfb 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -1,4 +1,4 @@ -from . import util +from . import util abc = util.import_importlib('importlib.abc') init = util.import_importlib('importlib') machinery = util.import_importlib('importlib.machinery') @@ -17,6 +17,7 @@ import unittest.mock import warnings +@unittest.skip("TODO: RUSTPYTHON") class DecodeSourceBytesTests: source = "string ='ü'" @@ -543,6 +544,7 @@ class MagicNumberTests: # Should be 4 bytes. self.assertEqual(len(self.util.MAGIC_NUMBER), 4) + @unittest.skip("TODO: RUSTPYTHON") def test_incorporates_rn(self): # The magic number uses \r\n to come out wrong when splitting on lines. self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) @@ -584,6 +586,8 @@ class PEP3147Tests: self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cache_from_source_debug_override(self): # Given the path to a .py file, return the path to its PEP 3147/PEP 488 # defined .pyc file (i.e. under __pycache__). diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 196ea1c9d..6ecf4f419 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -461,14 +461,17 @@ class CommonResourceTests(abc.ABC): def execute(self, package, path): raise NotImplementedError + @unittest.skip("TODO: RUSTPYTHON") def test_package_name(self): # Passing in the package name should succeed. self.execute(data01.__name__, 'utf-8.file') + @unittest.skip("TODO: RUSTPYTHON") def test_package_object(self): # Passing in the package itself should succeed. self.execute(data01, 'utf-8.file') + @unittest.skip("TODO: RUSTPYTHON") def test_string_path(self): # Passing in a string for the path should succeed. path = 'utf-8.file' @@ -492,6 +495,7 @@ class CommonResourceTests(abc.ABC): with self.assertRaises(ValueError): self.execute(data01, '../data01/utf-8.file') + @unittest.skip("TODO: RUSTPYTHON") def test_importing_module_as_side_effect(self): # The anchor package can already be imported. del sys.modules[data01.__name__] @@ -509,6 +513,7 @@ class CommonResourceTests(abc.ABC): self.execute(module, 'utf-8.file') @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + @unittest.skip("TODO: RUSTPYTHON") def test_resource_opener(self): bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) @@ -516,6 +521,7 @@ class CommonResourceTests(abc.ABC): self.assertEqual(package.__loader__._path, 'utf-8.file') @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + @unittest.skip("TODO: RUSTPYTHON") def test_resource_path(self): bytes_data = io.BytesIO(b'Hello, world!') path = __file__ From 4fe66e2b9a9fc87a086a7e6104ff7558bafc7a9f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Jun 2020 11:05:44 +0300 Subject: [PATCH 032/209] Remove bad_coding2.py --- Lib/test/bad_coding2.py | 2 -- Lib/test/test_py_compile.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 Lib/test/bad_coding2.py diff --git a/Lib/test/bad_coding2.py b/Lib/test/bad_coding2.py deleted file mode 100644 index bb2bb7e1e..000000000 --- a/Lib/test/bad_coding2.py +++ /dev/null @@ -1,2 +0,0 @@ -#coding: utf8 -print('我') diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 912f0f0ea..29d108871 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -129,6 +129,7 @@ class PyCompileTestsBase: finally: os.chmod(self.directory, mode.st_mode) + @unittest.skip("TODO: RUSTPYTHON") def test_bad_coding(self): bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') with support.captured_stderr(): From eba1d722eaf6f9a252702fdb2af2f220b8dd5c83 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Jun 2020 11:16:30 +0300 Subject: [PATCH 033/209] Remove global import lock in WASM --- vm/src/stdlib/imp.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 1cbc24b85..7f15e2eb4 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -5,19 +5,26 @@ use crate::obj::objmodule::PyModuleRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; +#[cfg(not(target_arch = "wasm32"))] use crate::stdlib::thread::RawRMutex; use crate::vm::VirtualMachine; +#[cfg(not(target_arch = "wasm32"))] static IMP_LOCK: RawRMutex = RawRMutex::INIT; fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_list(vec![])) } +#[cfg(not(target_arch = "wasm32"))] fn imp_acquire_lock(_vm: &VirtualMachine) { IMP_LOCK.lock() } +#[cfg(target_arch = "wasm32")] +fn imp_acquire_lock(_vm: &VirtualMachine) {} + +#[cfg(not(target_arch = "wasm32"))] fn imp_release_lock(vm: &VirtualMachine) -> PyResult<()> { if !IMP_LOCK.is_locked() { Err(vm.new_runtime_error("Global import lock not held".to_owned())) @@ -27,10 +34,19 @@ fn imp_release_lock(vm: &VirtualMachine) -> PyResult<()> { } } +#[cfg(target_arch = "wasm32")] +fn imp_release_lock(_vm: &VirtualMachine) {} + +#[cfg(not(target_arch = "wasm32"))] fn imp_lock_held(_vm: &VirtualMachine) -> bool { IMP_LOCK.is_locked() } +#[cfg(target_arch = "wasm32")] +fn imp_lock_held(_vm: &VirtualMachine) -> bool { + false +} + fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { vm.state.stdlib_inits.contains_key(name.as_str()) } From fa76d1048915611cad7e92dabd06edaf4c808fae Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 13 Jun 2020 09:22:42 +0300 Subject: [PATCH 034/209] Skip macos failing test --- Lib/test/test_py_compile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 29d108871..8dc52c29f 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -107,6 +107,7 @@ class PyCompileTestsBase: self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) + @unittest.skip("TODO: RUSTPYTHON") def test_relative_path(self): py_compile.compile(os.path.relpath(self.source_path), os.path.relpath(self.pyc_path)) From 4ba6b476aa4b0739946feda1318b921d7eefc795 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 18 Jul 2020 17:26:05 +0300 Subject: [PATCH 035/209] Wrap unlock in unsfae --- vm/src/stdlib/imp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 7f15e2eb4..c11b27431 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -29,7 +29,7 @@ fn imp_release_lock(vm: &VirtualMachine) -> PyResult<()> { if !IMP_LOCK.is_locked() { Err(vm.new_runtime_error("Global import lock not held".to_owned())) } else { - IMP_LOCK.unlock(); + unsafe { IMP_LOCK.unlock() }; Ok(()) } } From 8bee6203d0ee49570142655b6f2ceff816230378 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Jun 2020 21:16:09 +0900 Subject: [PATCH 036/209] Build dependencies with opt-level 3 always --- .github/workflows/ci.yaml | 6 +++--- Cargo.toml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4eaf4c247..7724a0fc1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }} + key: ${{ runner.os }}-debug-opt3-${{ hashFiles('Cargo.lock') }} - name: run rust tests uses: actions-rs/cargo@v1 with: @@ -135,8 +135,8 @@ jobs: target key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }} - ${{ runner.os }}-debug- + ${{ runner.os }}-debug-opt3-${{ hashFiles('Cargo.lock') }} + ${{ runner.os }}-debug-opt3 - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver diff --git a/Cargo.toml b/Cargo.toml index 629001891..78a179ade 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,9 @@ version = "0.2" name = "rustpython" path = "src/main.rs" +[profile.dev.package."*"] +opt-level = 3 + [patch.crates-io] # REDOX START, Uncommment when you want to compile/check with redoxer # # following patches are just waiting on a new version to be released to crates.io From 53a116f9f10c2e953ba49a27f8a54f83d918d252 Mon Sep 17 00:00:00 2001 From: clemado1 Date: Sun, 19 Jul 2020 12:52:59 +0900 Subject: [PATCH 037/209] Implement remove method of array --- vm/src/stdlib/array.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 5eaf7e2a6..0f6dde5f2 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -105,6 +105,20 @@ macro_rules! def_array_enum { } } + fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ + match self { + $(ArrayContentType::$n(v) => { + let val = $t::try_from_object(vm, obj)?; + if let Some(pos) = v.iter().position(|&a| a == val) { + v.remove(pos); + } else { + return Err(vm.new_value_error("array.remove(x): x not in array".to_owned())); + } + })* + } + Ok(()) + } + fn frombytes(&mut self, b: &[u8]) { match self { $(ArrayContentType::$n(v) => { @@ -298,6 +312,11 @@ impl PyArray { self.borrow_value().count(x, vm) } + #[pymethod] + fn remove(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.borrow_value_mut().remove(x, vm) + } + #[pymethod] fn extend(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult<()> { let mut array = self.borrow_value_mut(); From 4387d96ffea34377d4d1a8446282c85ca59ebecf Mon Sep 17 00:00:00 2001 From: hyperbora Date: Sun, 19 Jul 2020 22:12:10 +0900 Subject: [PATCH 038/209] add test_urlparse from CPython 3.8.4 --- Lib/test/test_urlparse.py | 1259 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1259 insertions(+) create mode 100644 Lib/test/test_urlparse.py diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py new file mode 100644 index 000000000..4ae6ed338 --- /dev/null +++ b/Lib/test/test_urlparse.py @@ -0,0 +1,1259 @@ +import sys +import unicodedata +import unittest +import urllib.parse + +RFC1808_BASE = "http://a/b/c/d;p?q#f" +RFC2396_BASE = "http://a/b/c/d;p?q" +RFC3986_BASE = 'http://a/b/c/d;p?q' +SIMPLE_BASE = 'http://a/b/c/d' + +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. + +parse_qsl_test_cases = [ + ("", []), + ("&", []), + ("&&", []), + ("=", [('', '')]), + ("=a", [('', 'a')]), + ("a", [('a', '')]), + ("a=", [('a', '')]), + ("&a=b", [('a', 'b')]), + ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), + ("a=1&a=2", [('a', '1'), ('a', '2')]), + (b"", []), + (b"&", []), + (b"&&", []), + (b"=", [(b'', b'')]), + (b"=a", [(b'', b'a')]), + (b"a", [(b'a', b'')]), + (b"a=", [(b'a', b'')]), + (b"&a=b", [(b'a', b'b')]), + (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), + (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), + (";", []), + (";;", []), + (";a=b", [('a', 'b')]), + ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]), + ("a=1;a=2", [('a', '1'), ('a', '2')]), + (b";", []), + (b";;", []), + (b";a=b", [(b'a', b'b')]), + (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), + (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), +] + +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + +parse_qs_test_cases = [ + ("", {}), + ("&", {}), + ("&&", {}), + ("=", {'': ['']}), + ("=a", {'': ['a']}), + ("a", {'a': ['']}), + ("a=", {'a': ['']}), + ("&a=b", {'a': ['b']}), + ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=1&a=2", {'a': ['1', '2']}), + (b"", {}), + (b"&", {}), + (b"&&", {}), + (b"=", {b'': [b'']}), + (b"=a", {b'': [b'a']}), + (b"a", {b'a': [b'']}), + (b"a=", {b'a': [b'']}), + (b"&a=b", {b'a': [b'b']}), + (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), + (b"a=1&a=2", {b'a': [b'1', b'2']}), + (";", {}), + (";;", {}), + (";a=b", {'a': ['b']}), + ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=1;a=2", {'a': ['1', '2']}), + (b";", {}), + (b";;", {}), + (b";a=b", {b'a': [b'b']}), + (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), + (b"a=1;a=2", {b'a': [b'1', b'2']}), +] + +class UrlParseTestCase(unittest.TestCase): + + def checkRoundtrips(self, url, parsed, split): + result = urllib.parse.urlparse(url) + self.assertEqual(result, parsed) + t = (result.scheme, result.netloc, result.path, + result.params, result.query, result.fragment) + self.assertEqual(t, parsed) + # put it back together and it should be the same + result2 = urllib.parse.urlunparse(result) + self.assertEqual(result2, url) + self.assertEqual(result2, result.geturl()) + + # the result of geturl() is a fixpoint; we can always parse it + # again to get the same result: + result3 = urllib.parse.urlparse(result.geturl()) + self.assertEqual(result3.geturl(), result.geturl()) + self.assertEqual(result3, result) + self.assertEqual(result3.scheme, result.scheme) + self.assertEqual(result3.netloc, result.netloc) + self.assertEqual(result3.path, result.path) + self.assertEqual(result3.params, result.params) + self.assertEqual(result3.query, result.query) + self.assertEqual(result3.fragment, result.fragment) + self.assertEqual(result3.username, result.username) + self.assertEqual(result3.password, result.password) + self.assertEqual(result3.hostname, result.hostname) + self.assertEqual(result3.port, result.port) + + # check the roundtrip using urlsplit() as well + result = urllib.parse.urlsplit(url) + self.assertEqual(result, split) + t = (result.scheme, result.netloc, result.path, + result.query, result.fragment) + self.assertEqual(t, split) + result2 = urllib.parse.urlunsplit(result) + self.assertEqual(result2, url) + self.assertEqual(result2, result.geturl()) + + # check the fixpoint property of re-parsing the result of geturl() + result3 = urllib.parse.urlsplit(result.geturl()) + self.assertEqual(result3.geturl(), result.geturl()) + self.assertEqual(result3, result) + self.assertEqual(result3.scheme, result.scheme) + self.assertEqual(result3.netloc, result.netloc) + self.assertEqual(result3.path, result.path) + self.assertEqual(result3.query, result.query) + self.assertEqual(result3.fragment, result.fragment) + self.assertEqual(result3.username, result.username) + self.assertEqual(result3.password, result.password) + self.assertEqual(result3.hostname, result.hostname) + self.assertEqual(result3.port, result.port) + + def test_qsl(self): + for orig, expect in parse_qsl_test_cases: + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect, "Error parsing %r" % orig) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks, + "Error parsing %r" % orig) + + def test_qs(self): + for orig, expect in parse_qs_test_cases: + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect, "Error parsing %r" % orig) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks, + "Error parsing %r" % orig) + + def test_roundtrips(self): + str_cases = [ + ('file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('imap://mail.python.org/mbox1', + ('imap', 'mail.python.org', '/mbox1', '', '', ''), + ('imap', 'mail.python.org', '/mbox1', '', '')), + ('mms://wms.sys.hinet.net/cts/Drama/09006251100.asf', + ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf', + '', '', ''), + ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf', + '', '')), + ('nfs://server/path/to/file.txt', + ('nfs', 'server', '/path/to/file.txt', '', '', ''), + ('nfs', 'server', '/path/to/file.txt', '', '')), + ('svn+ssh://svn.zope.org/repos/main/ZConfig/trunk/', + ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', + '', '', ''), + ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', + '', '')), + ('git+ssh://git@github.com/user/project.git', + ('git+ssh', 'git@github.com','/user/project.git', + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), + ] + def _encode(t): + return (t[0].encode('ascii'), + tuple(x.encode('ascii') for x in t[1]), + tuple(x.encode('ascii') for x in t[2])) + bytes_cases = [_encode(x) for x in str_cases] + for url, parsed, split in str_cases + bytes_cases: + self.checkRoundtrips(url, parsed, split) + + def test_http_roundtrips(self): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + str_cases = [ + ('://www.python.org', + ('www.python.org', '', '', '', ''), + ('www.python.org', '', '', '')), + ('://www.python.org#abc', + ('www.python.org', '', '', '', 'abc'), + ('www.python.org', '', '', 'abc')), + ('://www.python.org?q=abc', + ('www.python.org', '', '', 'q=abc', ''), + ('www.python.org', '', 'q=abc', '')), + ('://www.python.org/#abc', + ('www.python.org', '/', '', '', 'abc'), + ('www.python.org', '/', '', 'abc')), + ('://a/b/c/d;p?q#f', + ('a', '/b/c/d', 'p', 'q', 'f'), + ('a', '/b/c/d;p', 'q', 'f')), + ] + def _encode(t): + return (t[0].encode('ascii'), + tuple(x.encode('ascii') for x in t[1]), + tuple(x.encode('ascii') for x in t[2])) + bytes_cases = [_encode(x) for x in str_cases] + str_schemes = ('http', 'https') + bytes_schemes = (b'http', b'https') + str_tests = str_schemes, str_cases + bytes_tests = bytes_schemes, bytes_cases + for schemes, test_cases in (str_tests, bytes_tests): + for scheme in schemes: + for url, parsed, split in test_cases: + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) + + def checkJoin(self, base, relurl, expected): + str_components = (base, relurl, expected) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + bytes_components = baseb, relurlb, expectedb = [ + x.encode('ascii') for x in str_components] + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + def test_unparse_parse(self): + str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] + bytes_cases = [x.encode('ascii') for x in str_cases] + for u in str_cases + bytes_cases: + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + + def test_RFC1808(self): + # "normal" cases from RFC 1808: + self.checkJoin(RFC1808_BASE, 'g:h', 'g:h') + self.checkJoin(RFC1808_BASE, 'g', 'http://a/b/c/g') + self.checkJoin(RFC1808_BASE, './g', 'http://a/b/c/g') + self.checkJoin(RFC1808_BASE, 'g/', 'http://a/b/c/g/') + self.checkJoin(RFC1808_BASE, '/g', 'http://a/g') + self.checkJoin(RFC1808_BASE, '//g', 'http://g') + self.checkJoin(RFC1808_BASE, 'g?y', 'http://a/b/c/g?y') + self.checkJoin(RFC1808_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x') + self.checkJoin(RFC1808_BASE, '#s', 'http://a/b/c/d;p?q#s') + self.checkJoin(RFC1808_BASE, 'g#s', 'http://a/b/c/g#s') + self.checkJoin(RFC1808_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x') + self.checkJoin(RFC1808_BASE, 'g?y#s', 'http://a/b/c/g?y#s') + self.checkJoin(RFC1808_BASE, 'g;x', 'http://a/b/c/g;x') + self.checkJoin(RFC1808_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s') + self.checkJoin(RFC1808_BASE, '.', 'http://a/b/c/') + self.checkJoin(RFC1808_BASE, './', 'http://a/b/c/') + self.checkJoin(RFC1808_BASE, '..', 'http://a/b/') + self.checkJoin(RFC1808_BASE, '../', 'http://a/b/') + self.checkJoin(RFC1808_BASE, '../g', 'http://a/b/g') + self.checkJoin(RFC1808_BASE, '../..', 'http://a/') + self.checkJoin(RFC1808_BASE, '../../', 'http://a/') + self.checkJoin(RFC1808_BASE, '../../g', 'http://a/g') + + # "abnormal" cases from RFC 1808: + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'g.', 'http://a/b/c/g.') + self.checkJoin(RFC1808_BASE, '.g', 'http://a/b/c/.g') + self.checkJoin(RFC1808_BASE, 'g..', 'http://a/b/c/g..') + self.checkJoin(RFC1808_BASE, '..g', 'http://a/b/c/..g') + self.checkJoin(RFC1808_BASE, './../g', 'http://a/b/g') + self.checkJoin(RFC1808_BASE, './g/.', 'http://a/b/c/g/') + self.checkJoin(RFC1808_BASE, 'g/./h', 'http://a/b/c/g/h') + self.checkJoin(RFC1808_BASE, 'g/../h', 'http://a/b/c/h') + + # RFC 1808 and RFC 1630 disagree on these (according to RFC 1808), + # so we'll not actually run these tests (which expect 1808 behavior). + #self.checkJoin(RFC1808_BASE, 'http:g', 'http:g') + #self.checkJoin(RFC1808_BASE, 'http:', 'http:') + + # XXX: The following tests are no longer compatible with RFC3986 + # self.checkJoin(RFC1808_BASE, '../../../g', 'http://a/../g') + # self.checkJoin(RFC1808_BASE, '../../../../g', 'http://a/../../g') + # self.checkJoin(RFC1808_BASE, '/./g', 'http://a/./g') + # self.checkJoin(RFC1808_BASE, '/../g', 'http://a/../g') + + + def test_RFC2368(self): + # Issue 11467: path that starts with a number is not parsed correctly + self.assertEqual(urllib.parse.urlparse('mailto:1337@example.org'), + ('mailto', '', '1337@example.org', '', '', '')) + + def test_RFC2396(self): + # cases from RFC 2396 + + self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') + self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') + self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') + self.checkJoin(RFC2396_BASE, 'g/', 'http://a/b/c/g/') + self.checkJoin(RFC2396_BASE, '/g', 'http://a/g') + self.checkJoin(RFC2396_BASE, '//g', 'http://g') + self.checkJoin(RFC2396_BASE, 'g?y', 'http://a/b/c/g?y') + self.checkJoin(RFC2396_BASE, '#s', 'http://a/b/c/d;p?q#s') + self.checkJoin(RFC2396_BASE, 'g#s', 'http://a/b/c/g#s') + self.checkJoin(RFC2396_BASE, 'g?y#s', 'http://a/b/c/g?y#s') + self.checkJoin(RFC2396_BASE, 'g;x', 'http://a/b/c/g;x') + self.checkJoin(RFC2396_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s') + self.checkJoin(RFC2396_BASE, '.', 'http://a/b/c/') + self.checkJoin(RFC2396_BASE, './', 'http://a/b/c/') + self.checkJoin(RFC2396_BASE, '..', 'http://a/b/') + self.checkJoin(RFC2396_BASE, '../', 'http://a/b/') + self.checkJoin(RFC2396_BASE, '../g', 'http://a/b/g') + self.checkJoin(RFC2396_BASE, '../..', 'http://a/') + self.checkJoin(RFC2396_BASE, '../../', 'http://a/') + self.checkJoin(RFC2396_BASE, '../../g', 'http://a/g') + self.checkJoin(RFC2396_BASE, '', RFC2396_BASE) + self.checkJoin(RFC2396_BASE, 'g.', 'http://a/b/c/g.') + self.checkJoin(RFC2396_BASE, '.g', 'http://a/b/c/.g') + self.checkJoin(RFC2396_BASE, 'g..', 'http://a/b/c/g..') + self.checkJoin(RFC2396_BASE, '..g', 'http://a/b/c/..g') + self.checkJoin(RFC2396_BASE, './../g', 'http://a/b/g') + self.checkJoin(RFC2396_BASE, './g/.', 'http://a/b/c/g/') + self.checkJoin(RFC2396_BASE, 'g/./h', 'http://a/b/c/g/h') + self.checkJoin(RFC2396_BASE, 'g/../h', 'http://a/b/c/h') + self.checkJoin(RFC2396_BASE, 'g;x=1/./y', 'http://a/b/c/g;x=1/y') + self.checkJoin(RFC2396_BASE, 'g;x=1/../y', 'http://a/b/c/y') + self.checkJoin(RFC2396_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x') + self.checkJoin(RFC2396_BASE, 'g?y/../x', 'http://a/b/c/g?y/../x') + self.checkJoin(RFC2396_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x') + self.checkJoin(RFC2396_BASE, 'g#s/../x', 'http://a/b/c/g#s/../x') + + # XXX: The following tests are no longer compatible with RFC3986 + # self.checkJoin(RFC2396_BASE, '../../../g', 'http://a/../g') + # self.checkJoin(RFC2396_BASE, '../../../../g', 'http://a/../../g') + # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') + # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') + + def test_RFC3986(self): + self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') + self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC3986_BASE, 'g:h','g:h') + self.checkJoin(RFC3986_BASE, 'g','http://a/b/c/g') + self.checkJoin(RFC3986_BASE, './g','http://a/b/c/g') + self.checkJoin(RFC3986_BASE, 'g/','http://a/b/c/g/') + self.checkJoin(RFC3986_BASE, '/g','http://a/g') + self.checkJoin(RFC3986_BASE, '//g','http://g') + self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') + self.checkJoin(RFC3986_BASE, 'g?y','http://a/b/c/g?y') + self.checkJoin(RFC3986_BASE, '#s','http://a/b/c/d;p?q#s') + self.checkJoin(RFC3986_BASE, 'g#s','http://a/b/c/g#s') + self.checkJoin(RFC3986_BASE, 'g?y#s','http://a/b/c/g?y#s') + self.checkJoin(RFC3986_BASE, ';x','http://a/b/c/;x') + self.checkJoin(RFC3986_BASE, 'g;x','http://a/b/c/g;x') + self.checkJoin(RFC3986_BASE, 'g;x?y#s','http://a/b/c/g;x?y#s') + self.checkJoin(RFC3986_BASE, '','http://a/b/c/d;p?q') + self.checkJoin(RFC3986_BASE, '.','http://a/b/c/') + self.checkJoin(RFC3986_BASE, './','http://a/b/c/') + self.checkJoin(RFC3986_BASE, '..','http://a/b/') + self.checkJoin(RFC3986_BASE, '../','http://a/b/') + self.checkJoin(RFC3986_BASE, '../g','http://a/b/g') + self.checkJoin(RFC3986_BASE, '../..','http://a/') + self.checkJoin(RFC3986_BASE, '../../','http://a/') + self.checkJoin(RFC3986_BASE, '../../g','http://a/g') + self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') + + # Abnormal Examples + + # The 'abnormal scenarios' are incompatible with RFC2986 parsing + # Tests are here for reference. + + self.checkJoin(RFC3986_BASE, '../../../g','http://a/g') + self.checkJoin(RFC3986_BASE, '../../../../g','http://a/g') + self.checkJoin(RFC3986_BASE, '/./g','http://a/g') + self.checkJoin(RFC3986_BASE, '/../g','http://a/g') + self.checkJoin(RFC3986_BASE, 'g.','http://a/b/c/g.') + self.checkJoin(RFC3986_BASE, '.g','http://a/b/c/.g') + self.checkJoin(RFC3986_BASE, 'g..','http://a/b/c/g..') + self.checkJoin(RFC3986_BASE, '..g','http://a/b/c/..g') + self.checkJoin(RFC3986_BASE, './../g','http://a/b/g') + self.checkJoin(RFC3986_BASE, './g/.','http://a/b/c/g/') + self.checkJoin(RFC3986_BASE, 'g/./h','http://a/b/c/g/h') + self.checkJoin(RFC3986_BASE, 'g/../h','http://a/b/c/h') + self.checkJoin(RFC3986_BASE, 'g;x=1/./y','http://a/b/c/g;x=1/y') + self.checkJoin(RFC3986_BASE, 'g;x=1/../y','http://a/b/c/y') + self.checkJoin(RFC3986_BASE, 'g?y/./x','http://a/b/c/g?y/./x') + self.checkJoin(RFC3986_BASE, 'g?y/../x','http://a/b/c/g?y/../x') + self.checkJoin(RFC3986_BASE, 'g#s/./x','http://a/b/c/g#s/./x') + self.checkJoin(RFC3986_BASE, 'g#s/../x','http://a/b/c/g#s/../x') + #self.checkJoin(RFC3986_BASE, 'http:g','http:g') # strict parser + self.checkJoin(RFC3986_BASE, 'http:g','http://a/b/c/g') #relaxed parser + + # Test for issue9721 + self.checkJoin('http://a/b/c/de', ';x','http://a/b/c/;x') + + def test_urljoins(self): + self.checkJoin(SIMPLE_BASE, 'g:h','g:h') + self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') + self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') + self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') + self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') + self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') + self.checkJoin(SIMPLE_BASE, '/g','http://a/g') + self.checkJoin(SIMPLE_BASE, '//g','http://g') + self.checkJoin(SIMPLE_BASE, '?y','http://a/b/c/d?y') + self.checkJoin(SIMPLE_BASE, 'g?y','http://a/b/c/g?y') + self.checkJoin(SIMPLE_BASE, 'g?y/./x','http://a/b/c/g?y/./x') + self.checkJoin(SIMPLE_BASE, '.','http://a/b/c/') + self.checkJoin(SIMPLE_BASE, './','http://a/b/c/') + self.checkJoin(SIMPLE_BASE, '..','http://a/b/') + self.checkJoin(SIMPLE_BASE, '../','http://a/b/') + self.checkJoin(SIMPLE_BASE, '../g','http://a/b/g') + self.checkJoin(SIMPLE_BASE, '../..','http://a/') + self.checkJoin(SIMPLE_BASE, '../../g','http://a/g') + self.checkJoin(SIMPLE_BASE, './../g','http://a/b/g') + self.checkJoin(SIMPLE_BASE, './g/.','http://a/b/c/g/') + self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') + self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') + self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') + self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') + self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') + self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') + self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') + self.checkJoin('http:///', '..','http:///') + self.checkJoin('', 'http://a/b/c/g?y/./x','http://a/b/c/g?y/./x') + self.checkJoin('', 'http://a/./g', 'http://a/./g') + self.checkJoin('svn://pathtorepo/dir1', 'dir2', 'svn://pathtorepo/dir2') + self.checkJoin('svn+ssh://pathtorepo/dir1', 'dir2', 'svn+ssh://pathtorepo/dir2') + self.checkJoin('ws://a/b','g','ws://a/g') + self.checkJoin('wss://a/b','g','wss://a/g') + + # XXX: The following tests are no longer compatible with RFC3986 + # self.checkJoin(SIMPLE_BASE, '../../../g','http://a/../g') + # self.checkJoin(SIMPLE_BASE, '/./g','http://a/./g') + + # test for issue22118 duplicate slashes + self.checkJoin(SIMPLE_BASE + '/', 'foo', SIMPLE_BASE + '/foo') + + # Non-RFC-defined tests, covering variations of base and trailing + # slashes + self.checkJoin('http://a/b/c/d/e/', '../../f/g/', 'http://a/b/c/f/g/') + self.checkJoin('http://a/b/c/d/e', '../../f/g/', 'http://a/b/f/g/') + self.checkJoin('http://a/b/c/d/e/', '/../../f/g/', 'http://a/f/g/') + self.checkJoin('http://a/b/c/d/e', '/../../f/g/', 'http://a/f/g/') + self.checkJoin('http://a/b/c/d/e/', '../../f/g', 'http://a/b/c/f/g') + self.checkJoin('http://a/b/', '../../f/g/', 'http://a/f/g/') + + # issue 23703: don't duplicate filename + self.checkJoin('a', 'b', 'b') + + def test_RFC2732(self): + str_cases = [ + ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), + ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), + ('http://[::1]:5432/foo/', '::1', 5432), + ('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432), + ('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432), + ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/', + 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432), + ('http://[::12.34.56.78]:5432/foo/', '::12.34.56.78', 5432), + ('http://[::ffff:12.34.56.78]:5432/foo/', + '::ffff:12.34.56.78', 5432), + ('http://Test.python.org/foo/', 'test.python.org', None), + ('http://12.34.56.78/foo/', '12.34.56.78', None), + ('http://[::1]/foo/', '::1', None), + ('http://[dead:beef::1]/foo/', 'dead:beef::1', None), + ('http://[dead:beef::]/foo/', 'dead:beef::', None), + ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/', + 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None), + ('http://[::12.34.56.78]/foo/', '::12.34.56.78', None), + ('http://[::ffff:12.34.56.78]/foo/', + '::ffff:12.34.56.78', None), + ('http://Test.python.org:/foo/', 'test.python.org', None), + ('http://12.34.56.78:/foo/', '12.34.56.78', None), + ('http://[::1]:/foo/', '::1', None), + ('http://[dead:beef::1]:/foo/', 'dead:beef::1', None), + ('http://[dead:beef::]:/foo/', 'dead:beef::', None), + ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:/foo/', + 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None), + ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), + ('http://[::ffff:12.34.56.78]:/foo/', + '::ffff:12.34.56.78', None), + ] + def _encode(t): + return t[0].encode('ascii'), t[1].encode('ascii'), t[2] + bytes_cases = [_encode(x) for x in str_cases] + for url, hostname, port in str_cases + bytes_cases: + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) + + str_cases = [ + 'http://::12.34.56.78]/', + 'http://[::1/foo/', + 'ftp://[::1/foo/bad]/bad', + 'http://[::1/foo/bad]/bad', + 'http://[::ffff:12.34.56.78'] + bytes_cases = [x.encode('ascii') for x in str_cases] + for invalid_url in str_cases + bytes_cases: + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + + def test_urldefrag(self): + str_cases = [ + ('http://python.org#frag', 'http://python.org', 'frag'), + ('http://python.org', 'http://python.org', ''), + ('http://python.org/#frag', 'http://python.org/', 'frag'), + ('http://python.org/', 'http://python.org/', ''), + ('http://python.org/?q#frag', 'http://python.org/?q', 'frag'), + ('http://python.org/?q', 'http://python.org/?q', ''), + ('http://python.org/p#frag', 'http://python.org/p', 'frag'), + ('http://python.org/p?q', 'http://python.org/p?q', ''), + (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), + (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), + ] + def _encode(t): + return type(t)(x.encode('ascii') for x in t) + bytes_cases = [_encode(x) for x in str_cases] + for url, defrag, frag in str_cases + bytes_cases: + result = urllib.parse.urldefrag(url) + self.assertEqual(result.geturl(), url) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) + + def test_urlsplit_scoped_IPv6(self): + p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') + self.assertEqual(p.hostname, "fe80::822a:a8ff:fe49:470c%tESt") + self.assertEqual(p.netloc, '[FE80::822a:a8ff:fe49:470c%tESt]:1234') + + p = urllib.parse.urlsplit(b'http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') + self.assertEqual(p.hostname, b"fe80::822a:a8ff:fe49:470c%tESt") + self.assertEqual(p.netloc, b'[FE80::822a:a8ff:fe49:470c%tESt]:1234') + + def test_urlsplit_attributes(self): + url = "HTTP://WWW.PYTHON.ORG/doc/#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "WWW.PYTHON.ORG") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, None) + # geturl() won't return exactly the original URL in this case + # since the scheme is always case-normalized + # We handle this by ignoring the first 4 characters of the URL + self.assertEqual(p.geturl()[4:], url[4:]) + + url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "User:Pass@www.python.org:080") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query=yes") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, "User") + self.assertEqual(p.password, "Pass") + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), url) + + # Addressing issue1698, which suggests Username can contain + # "@" characters. Though not RFC compliant, many ftp sites allow + # and request email addresses as usernames. + + url = "http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "User@example.com:Pass@www.python.org:080") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query=yes") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, "User@example.com") + self.assertEqual(p.password, "Pass") + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), url) + + # And check them all again, only with bytes this time + url = b"HTTP://WWW.PYTHON.ORG/doc/#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"WWW.PYTHON.ORG") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, None) + self.assertEqual(p.geturl()[4:], url[4:]) + + url = b"http://User:Pass@www.python.org:080/doc/?query=yes#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"query=yes") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, b"User") + self.assertEqual(p.password, b"Pass") + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), url) + + url = b"http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"User@example.com:Pass@www.python.org:080") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"query=yes") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, b"User@example.com") + self.assertEqual(p.password, b"Pass") + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), url) + + # Verify an illegal port raises ValueError + url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" + p = urllib.parse.urlsplit(url) + with self.assertRaisesRegex(ValueError, "out of range"): + p.port + + def test_attributes_bad_port(self): + """Check handling of invalid ports.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for port in ("foo", "1.5", "-1", "0x10"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port + url = "http://" + netloc + if bytes: + netloc = netloc.encode("ascii") + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port + + def test_attributes_without_netloc(self): + # This example is straight from RFC 3261. It looks like it + # should allow the username, hostname, and port to be filled + # in, but doesn't. Since it's a URI and doesn't use the + # scheme://netloc syntax, the netloc and related attributes + # should be left empty. + uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15" + p = urllib.parse.urlsplit(uri) + self.assertEqual(p.netloc, "") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, None) + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), uri) + + p = urllib.parse.urlparse(uri) + self.assertEqual(p.netloc, "") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, None) + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), uri) + + # You guessed it, repeating the test with bytes input + uri = b"sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15" + p = urllib.parse.urlsplit(uri) + self.assertEqual(p.netloc, b"") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, None) + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), uri) + + p = urllib.parse.urlparse(uri) + self.assertEqual(p.netloc, b"") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, None) + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), uri) + + def test_noslash(self): + # Issue 1637: http://foo.com?query is legal + self.assertEqual(urllib.parse.urlparse("http://example.com?blahblah=/foo"), + ('http', 'example.com', '', '', 'blahblah=/foo', '')) + self.assertEqual(urllib.parse.urlparse(b"http://example.com?blahblah=/foo"), + (b'http', b'example.com', b'', b'', b'blahblah=/foo', b'')) + + def test_withoutscheme(self): + # Test urlparse without scheme + # Issue 754016: urlparse goes wrong with IP:port without scheme + # RFC 1808 specifies that netloc should start with //, urlparse expects + # the same, otherwise it classifies the portion of url as path. + self.assertEqual(urllib.parse.urlparse("path"), + ('','','path','','','')) + self.assertEqual(urllib.parse.urlparse("//www.python.org:80"), + ('','www.python.org:80','','','','')) + self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"), + ('http','www.python.org:80','','','','')) + # Repeat for bytes input + self.assertEqual(urllib.parse.urlparse(b"path"), + (b'',b'',b'path',b'',b'',b'')) + self.assertEqual(urllib.parse.urlparse(b"//www.python.org:80"), + (b'',b'www.python.org:80',b'',b'',b'',b'')) + self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"), + (b'http',b'www.python.org:80',b'',b'',b'',b'')) + + def test_portseparator(self): + # Issue 754016 makes changes for port separator ':' from scheme separator + self.assertEqual(urllib.parse.urlparse("path:80"), + ('','','path:80','','','')) + self.assertEqual(urllib.parse.urlparse("http:"),('http','','','','','')) + self.assertEqual(urllib.parse.urlparse("https:"),('https','','','','','')) + self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"), + ('http','www.python.org:80','','','','')) + # As usual, need to check bytes input as well + self.assertEqual(urllib.parse.urlparse(b"path:80"), + (b'',b'',b'path:80',b'',b'',b'')) + self.assertEqual(urllib.parse.urlparse(b"http:"),(b'http',b'',b'',b'',b'',b'')) + self.assertEqual(urllib.parse.urlparse(b"https:"),(b'https',b'',b'',b'',b'',b'')) + self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"), + (b'http',b'www.python.org:80',b'',b'',b'',b'')) + + def test_usingsys(self): + # Issue 3314: sys module is used in the error + self.assertRaises(TypeError, urllib.parse.urlencode, "foo") + + def test_anyscheme(self): + # Issue 7904: s3://foo.com/stuff has netloc "foo.com". + self.assertEqual(urllib.parse.urlparse("s3://foo.com/stuff"), + ('s3', 'foo.com', '/stuff', '', '', '')) + self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff"), + ('x-newscheme', 'foo.com', '/stuff', '', '', '')) + self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query#fragment"), + ('x-newscheme', 'foo.com', '/stuff', '', 'query', 'fragment')) + self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query"), + ('x-newscheme', 'foo.com', '/stuff', '', 'query', '')) + + # And for bytes... + self.assertEqual(urllib.parse.urlparse(b"s3://foo.com/stuff"), + (b's3', b'foo.com', b'/stuff', b'', b'', b'')) + self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff"), + (b'x-newscheme', b'foo.com', b'/stuff', b'', b'', b'')) + self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query#fragment"), + (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'fragment')) + self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), + (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) + + def test_default_scheme(self): + # Exercise the scheme parameter of urlparse() and urlsplit() + for func in (urllib.parse.urlparse, urllib.parse.urlsplit): + with self.subTest(function=func): + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") + + def test_parse_fragments(self): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + tests = ( + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), + ) + for url, attr, expected_frag in tests: + for func in (urllib.parse.urlparse, urllib.parse.urlsplit): + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + with self.subTest(url=url, function=func): + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) + self.assertEqual(func(url, "", False).fragment, "") + + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) + + def test_mixed_types_rejected(self): + # Several functions that process either strings or ASCII encoded bytes + # accept multiple arguments. Check they reject mixed type input + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlparse("www.python.org", b"http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlparse(b"www.python.org", "http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlsplit("www.python.org", b"http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlsplit(b"www.python.org", "http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunparse(( b"http", "www.python.org","","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunparse(("http", b"www.python.org","","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunsplit((b"http", "www.python.org","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunsplit(("http", b"www.python.org","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urljoin("http://python.org", b"http://python.org") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urljoin(b"http://python.org", "http://python.org") + + def _check_result_type(self, str_type): + num_args = len(str_type._fields) + bytes_type = str_type._encoded_counterpart + self.assertIs(bytes_type._decoded_counterpart, str_type) + str_args = ('',) * num_args + bytes_args = (b'',) * num_args + str_result = str_type(*str_args) + bytes_result = bytes_type(*bytes_args) + encoding = 'ascii' + errors = 'strict' + self.assertEqual(str_result, str_args) + self.assertEqual(bytes_result.decode(), str_args) + self.assertEqual(bytes_result.decode(), str_result) + self.assertEqual(bytes_result.decode(encoding), str_args) + self.assertEqual(bytes_result.decode(encoding), str_result) + self.assertEqual(bytes_result.decode(encoding, errors), str_args) + self.assertEqual(bytes_result.decode(encoding, errors), str_result) + self.assertEqual(bytes_result, bytes_args) + self.assertEqual(str_result.encode(), bytes_args) + self.assertEqual(str_result.encode(), bytes_result) + self.assertEqual(str_result.encode(encoding), bytes_args) + self.assertEqual(str_result.encode(encoding), bytes_result) + self.assertEqual(str_result.encode(encoding, errors), bytes_args) + self.assertEqual(str_result.encode(encoding, errors), bytes_result) + + def test_result_pairs(self): + # Check encoding and decoding between result pairs + result_types = [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ] + for result_type in result_types: + self._check_result_type(result_type) + + def test_parse_qs_encoding(self): + result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") + self.assertEqual(result, {'key': ['\u0141\xE9']}) + result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="utf-8") + self.assertEqual(result, {'key': ['\u0141\xE9']}) + result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="ascii") + self.assertEqual(result, {'key': ['\u0141\ufffd\ufffd']}) + result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii") + self.assertEqual(result, {'key': ['\u0141\ufffd-']}) + result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii", + errors="ignore") + self.assertEqual(result, {'key': ['\u0141-']}) + + def test_parse_qsl_encoding(self): + result = urllib.parse.parse_qsl("key=\u0141%E9", encoding="latin-1") + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="utf-8") + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="ascii") + self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')]) + result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii") + self.assertEqual(result, [('key', '\u0141\ufffd-')]) + result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii", + errors="ignore") + self.assertEqual(result, [('key', '\u0141-')]) + + def test_parse_qsl_max_num_fields(self): + with self.assertRaises(ValueError): + urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) + with self.assertRaises(ValueError): + urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + + def test_urlencode_sequences(self): + # Other tests incidentally urlencode things; test non-covered cases: + # Sequence and object values. + result = urllib.parse.urlencode({'a': [1, 2], 'b': (3, 4, 5)}, True) + # we cannot rely on ordering here + assert set(result.split('&')) == {'a=1', 'a=2', 'b=3', 'b=4', 'b=5'} + + class Trivial: + def __str__(self): + return 'trivial' + + result = urllib.parse.urlencode({'a': Trivial()}, True) + self.assertEqual(result, 'a=trivial') + + def test_urlencode_quote_via(self): + result = urllib.parse.urlencode({'a': 'some value'}) + self.assertEqual(result, "a=some+value") + result = urllib.parse.urlencode({'a': 'some value/another'}, + quote_via=urllib.parse.quote) + self.assertEqual(result, "a=some%20value%2Fanother") + result = urllib.parse.urlencode({'a': 'some value/another'}, + safe='/', quote_via=urllib.parse.quote) + self.assertEqual(result, "a=some%20value/another") + + def test_quote_from_bytes(self): + self.assertRaises(TypeError, urllib.parse.quote_from_bytes, 'foo') + result = urllib.parse.quote_from_bytes(b'archaeological arcana') + self.assertEqual(result, 'archaeological%20arcana') + result = urllib.parse.quote_from_bytes(b'') + self.assertEqual(result, '') + + def test_unquote_to_bytes(self): + result = urllib.parse.unquote_to_bytes('abc%20def') + self.assertEqual(result, b'abc def') + result = urllib.parse.unquote_to_bytes('') + self.assertEqual(result, b'') + + def test_quote_errors(self): + self.assertRaises(TypeError, urllib.parse.quote, b'foo', + encoding='utf-8') + self.assertRaises(TypeError, urllib.parse.quote, b'foo', errors='strict') + + def test_issue14072(self): + p1 = urllib.parse.urlsplit('tel:+31-641044153') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '+31-641044153') + p2 = urllib.parse.urlsplit('tel:+31641044153') + self.assertEqual(p2.scheme, 'tel') + self.assertEqual(p2.path, '+31641044153') + # assert the behavior for urlparse + p1 = urllib.parse.urlparse('tel:+31-641044153') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '+31-641044153') + p2 = urllib.parse.urlparse('tel:+31641044153') + self.assertEqual(p2.scheme, 'tel') + self.assertEqual(p2.path, '+31641044153') + + def test_port_casting_failure_message(self): + message = "Port could not be cast to integer value as 'oracle'" + p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p1.port + + p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle') + with self.assertRaisesRegex(ValueError, message): + p2.port + + def test_telurl_params(self): + p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '123-4') + self.assertEqual(p1.params, 'phone-context=+1-650-516') + + p1 = urllib.parse.urlparse('tel:+1-201-555-0123') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '+1-201-555-0123') + self.assertEqual(p1.params, '') + + p1 = urllib.parse.urlparse('tel:7042;phone-context=example.com') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '7042') + self.assertEqual(p1.params, 'phone-context=example.com') + + p1 = urllib.parse.urlparse('tel:863-1234;phone-context=+1-914-555') + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '863-1234') + self.assertEqual(p1.params, 'phone-context=+1-914-555') + + def test_Quoter_repr(self): + quoter = urllib.parse.Quoter(urllib.parse._ALWAYS_SAFE) + self.assertIn('Quoter', repr(quoter)) + + def test_all(self): + expected = [] + undocumented = { + 'splitattr', 'splithost', 'splitnport', 'splitpasswd', + 'splitport', 'splitquery', 'splittag', 'splittype', 'splituser', + 'splitvalue', + 'Quoter', 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap', + } + for name in dir(urllib.parse): + if name.startswith('_') or name in undocumented: + continue + object = getattr(urllib.parse, name) + if getattr(object, '__module__', None) == 'urllib.parse': + expected.append(name) + self.assertCountEqual(urllib.parse.__all__, expected) + + def test_urlsplit_normalization(self): + # Certain characters should never occur in the netloc, + # including under normalization. + # Ensure that ALL of them are detected and cause an error + illegal_chars = '/:#?@' + hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars} + denorm_chars = [ + c for c in map(chr, range(128, sys.maxunicode)) + if (hex_chars & set(unicodedata.decomposition(c).split())) + and c not in illegal_chars + ] + # Sanity check that we found at least one such character + self.assertIn('\u2100', denorm_chars) + self.assertIn('\uFF03', denorm_chars) + + # bpo-36742: Verify port separators are ignored when they + # existed prior to decomposition + urllib.parse.urlsplit('http://\u30d5\u309a:80') + with self.assertRaises(ValueError): + urllib.parse.urlsplit('http://\u30d5\u309a\ufe1380') + + for scheme in ["http", "https", "ftp"]: + for netloc in ["netloc{}false.netloc", "n{}user@netloc"]: + for c in denorm_chars: + url = "{}://{}/path".format(scheme, netloc.format(c)) + with self.subTest(url=url, char='{:04X}'.format(ord(c))): + with self.assertRaises(ValueError): + urllib.parse.urlsplit(url) + +class Utility_Tests(unittest.TestCase): + """Testcase to test the various utility functions in the urllib.""" + # In Python 2 this test class was in test_urllib. + + def test_splittype(self): + splittype = urllib.parse._splittype + self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring')) + self.assertEqual(splittype('opaquestring'), (None, 'opaquestring')) + self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring')) + self.assertEqual(splittype('type:'), ('type', '')) + self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) + + def test_splithost(self): + splithost = urllib.parse._splithost + self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), + ('www.example.org:80', '/foo/bar/baz.html')) + self.assertEqual(splithost('//www.example.org:80'), + ('www.example.org:80', '')) + self.assertEqual(splithost('/foo/bar/baz.html'), + (None, '/foo/bar/baz.html')) + + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + + def test_splituser(self): + splituser = urllib.parse._splituser + self.assertEqual(splituser('User:Pass@www.python.org:080'), + ('User:Pass', 'www.python.org:080')) + self.assertEqual(splituser('@www.python.org:080'), + ('', 'www.python.org:080')) + self.assertEqual(splituser('www.python.org:080'), + (None, 'www.python.org:080')) + self.assertEqual(splituser('User:Pass@'), + ('User:Pass', '')) + self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'), + ('User@example.com:Pass', 'www.python.org:080')) + + def test_splitpasswd(self): + # Some of the password examples are not sensible, but it is added to + # confirming to RFC2617 and addressing issue4675. + splitpasswd = urllib.parse._splitpasswd + self.assertEqual(splitpasswd('user:ab'), ('user', 'ab')) + self.assertEqual(splitpasswd('user:a\nb'), ('user', 'a\nb')) + self.assertEqual(splitpasswd('user:a\tb'), ('user', 'a\tb')) + self.assertEqual(splitpasswd('user:a\rb'), ('user', 'a\rb')) + self.assertEqual(splitpasswd('user:a\fb'), ('user', 'a\fb')) + self.assertEqual(splitpasswd('user:a\vb'), ('user', 'a\vb')) + self.assertEqual(splitpasswd('user:a:b'), ('user', 'a:b')) + self.assertEqual(splitpasswd('user:a b'), ('user', 'a b')) + self.assertEqual(splitpasswd('user 2:ab'), ('user 2', 'ab')) + self.assertEqual(splitpasswd('user+1:a+b'), ('user+1', 'a+b')) + self.assertEqual(splitpasswd('user:'), ('user', '')) + self.assertEqual(splitpasswd('user'), ('user', None)) + self.assertEqual(splitpasswd(':ab'), ('', 'ab')) + + def test_splitport(self): + splitport = urllib.parse._splitport + self.assertEqual(splitport('parrot:88'), ('parrot', '88')) + self.assertEqual(splitport('parrot'), ('parrot', None)) + self.assertEqual(splitport('parrot:'), ('parrot', None)) + self.assertEqual(splitport('127.0.0.1'), ('127.0.0.1', None)) + self.assertEqual(splitport('parrot:cheese'), ('parrot:cheese', None)) + self.assertEqual(splitport('[::1]:88'), ('[::1]', '88')) + self.assertEqual(splitport('[::1]'), ('[::1]', None)) + self.assertEqual(splitport(':88'), ('', '88')) + + def test_splitnport(self): + splitnport = urllib.parse._splitnport + self.assertEqual(splitnport('parrot:88'), ('parrot', 88)) + self.assertEqual(splitnport('parrot'), ('parrot', -1)) + self.assertEqual(splitnport('parrot', 55), ('parrot', 55)) + self.assertEqual(splitnport('parrot:'), ('parrot', -1)) + self.assertEqual(splitnport('parrot:', 55), ('parrot', 55)) + self.assertEqual(splitnport('127.0.0.1'), ('127.0.0.1', -1)) + self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) + self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) + self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + + def test_splitquery(self): + # Normal cases are exercised by other tests; ensure that we also + # catch cases with no port specified (testcase ensuring coverage) + splitquery = urllib.parse._splitquery + self.assertEqual(splitquery('http://python.org/fake?foo=bar'), + ('http://python.org/fake', 'foo=bar')) + self.assertEqual(splitquery('http://python.org/fake?foo=bar?'), + ('http://python.org/fake?foo=bar', '')) + self.assertEqual(splitquery('http://python.org/fake'), + ('http://python.org/fake', None)) + self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar')) + + def test_splittag(self): + splittag = urllib.parse._splittag + self.assertEqual(splittag('http://example.com?foo=bar#baz'), + ('http://example.com?foo=bar', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar#'), + ('http://example.com?foo=bar', '')) + self.assertEqual(splittag('#baz'), ('', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar'), + ('http://example.com?foo=bar', None)) + self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'), + ('http://example.com?foo=bar#baz', 'boo')) + + def test_splitattr(self): + splitattr = urllib.parse._splitattr + self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'), + ('/path', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path;'), ('/path', [''])) + self.assertEqual(splitattr(';attr1=value1;attr2=value2'), + ('', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path'), ('/path', [])) + + def test_splitvalue(self): + # Normal cases are exercised by other tests; test pathological cases + # with no key/value pairs. (testcase ensuring coverage) + splitvalue = urllib.parse._splitvalue + self.assertEqual(splitvalue('foo=bar'), ('foo', 'bar')) + self.assertEqual(splitvalue('foo='), ('foo', '')) + self.assertEqual(splitvalue('=bar'), ('', 'bar')) + self.assertEqual(splitvalue('foobar'), ('foobar', None)) + self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz')) + + def test_to_bytes(self): + result = urllib.parse._to_bytes('http://www.python.org') + self.assertEqual(result, 'http://www.python.org') + self.assertRaises(UnicodeError, urllib.parse._to_bytes, + 'http://www.python.org/medi\u00e6val') + + def test_unwrap(self): + for wrapped_url in ('', '', + 'URL:scheme://host/path', 'scheme://host/path'): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') + + +class DeprecationTest(unittest.TestCase): + + def test_splittype_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splittype('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splittype() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splithost_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splithost('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splithost() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splituser_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splituser('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splituser() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitpasswd_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitpasswd('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitpasswd() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitport_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitport('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitport() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitnport_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitnport('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitnport() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitquery_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitquery('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitquery() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splittag_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splittag('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splittag() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitattr_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitattr('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitattr() is deprecated as of 3.8, ' + 'use urllib.parse.urlparse() instead') + + def test_splitvalue_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.splitvalue('') + self.assertEqual(str(cm.warning), + 'urllib.parse.splitvalue() is deprecated as of 3.8, ' + 'use urllib.parse.parse_qsl() instead') + + def test_to_bytes_deprecation(self): + with self.assertWarns(DeprecationWarning) as cm: + urllib.parse.to_bytes('') + self.assertEqual(str(cm.warning), + 'urllib.parse.to_bytes() is deprecated as of 3.8') + + +if __name__ == "__main__": + unittest.main() From b132de110fbdf5b12f3b77b2a38594593073503d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Jul 2020 20:41:55 +0900 Subject: [PATCH 039/209] Fix Cargo.lock and add dependency for wasm --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7724a0fc1..e857d1177 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-debug-opt3-${{ hashFiles('Cargo.lock') }} + key: ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }} - name: run rust tests uses: actions-rs/cargo@v1 with: @@ -56,7 +56,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-release-${{ hashFiles('Cargo.lock') }} + key: ${{ runner.os }}-release-${{ hashFiles('**/Cargo.lock') }} - name: build rustpython uses: actions-rs/cargo@v1 with: @@ -123,6 +123,7 @@ jobs: wasm: name: Check the WASM package and demo + needs: rust_tests runs-on: ubuntu-latest steps: - uses: actions/checkout@master @@ -133,10 +134,9 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-wasm_opt3-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-debug-opt3-${{ hashFiles('Cargo.lock') }} - ${{ runner.os }}-debug-opt3 + ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }} - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver From 7629a2d2025f6a7e828ceee8bdae1561299d6c9e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Jul 2020 23:27:06 +0900 Subject: [PATCH 040/209] Run rust&python ilnt in same flow --- .github/workflows/ci.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e857d1177..8d01a9b17 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,7 +86,7 @@ jobs: RUSTPYTHONPATH: ${{ github.workspace }}/Lib if: runner.os == 'macOS' - format: + lint: name: Check Rust code with rustfmt and clippy runs-on: ubuntu-latest steps: @@ -107,12 +107,6 @@ jobs: with: command: clippy args: ${{ env.CARGO_ARGS }} -- -Dwarnings - - lint: - name: Lint Python code with flake8 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - uses: actions/setup-python@v1 with: python-version: 3.8 From 62277faa44817671e1b1494e0482051082d2f8cf Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Apr 2020 04:00:04 +0900 Subject: [PATCH 041/209] Share str.splitlines and bytes.splitlines implementaion --- vm/src/obj/objbyteinner.rs | 54 ++++++++++++++++---------------------- vm/src/obj/objstr.rs | 38 ++++++++++++++++----------- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 02139637b..309278105 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -962,45 +962,35 @@ impl PyByteInner { } pub fn splitlines(&self, options: pystr::SplitLinesArgs) -> Vec<&[u8]> { - let mut res = vec![]; - - if self.elements.is_empty() { - return vec![]; - } - - let mut prev_index = 0; - let mut index = 0; let keep = if options.keepends { 1 } else { 0 }; - let slice = &self.elements; - - while index < slice.len() { - match slice[index] { - b'\n' => { - res.push(&slice[prev_index..index + keep]); - index += 1; - prev_index = index; - } + let value = &self.elements; + let mut elements = Vec::new(); + let mut last_i = 0; + let mut chars = value.iter().enumerate().peekable(); + while let Some((i, ch)) = chars.next() { + let (end_len, i_diff) = match ch { + b'\n' => (keep, 1), b'\r' => { - if index + 2 <= slice.len() && slice[index + 1] == b'\n' { - res.push(&slice[prev_index..index + keep + keep]); - index += 2; + let is_rn = chars.peek().map_or(false, |(_, ch)| *ch == &b'\n'); + if is_rn { + let _ = chars.next(); + (keep + keep, 2) } else { - res.push(&slice[prev_index..index + keep]); - index += 1; + (keep, 1) } - prev_index = index; } - _x => { - if index == slice.len() - 1 { - res.push(&slice[prev_index..=index]); - break; - } - index += 1 + _ => { + continue; } - } + }; + let range = last_i..i + end_len; + last_i = i + i_diff; + elements.push(&value[range]); } - - res + if last_i != value.len() { + elements.push(&value[last_i..]); + } + elements } pub fn zfill(&self, width: isize) -> Vec { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index d9998a50d..5ddfb5868 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -803,25 +803,33 @@ impl PyString { #[pymethod] fn splitlines(&self, args: pystr::SplitLinesArgs, vm: &VirtualMachine) -> PyObjectRef { - let mut elements = vec![]; - let mut curr = "".to_owned(); - let mut chars = self.value.chars().peekable(); - while let Some(ch) = chars.next() { - if ch == '\n' || ch == '\r' { - if args.keepends { - curr.push(ch); + let keep = if args.keepends { 1 } else { 0 }; + let value = &self.value; + let mut elements = Vec::new(); + let mut last_i = 0; + let mut chars = value.chars().enumerate().peekable(); + while let Some((i, ch)) = chars.next() { + let (end_len, i_diff) = match ch { + '\n' => (keep, 1), + '\r' => { + let is_rn = chars.peek().map_or(false, |(_, ch)| *ch == '\n'); + if is_rn { + let _ = chars.next(); + (keep + keep, 2) + } else { + (keep, 1) + } } - if ch == '\r' && chars.peek() == Some(&'\n') { + _ => { continue; } - elements.push(vm.ctx.new_str(curr.clone())); - curr.clear(); - } else { - curr.push(ch); - } + }; + let range = last_i..i + end_len; + last_i = i + i_diff; + elements.push(vm.ctx.new_str(&value[range])); } - if !curr.is_empty() { - elements.push(vm.ctx.new_str(curr)); + if last_i != value.len() { + elements.push(vm.ctx.new_str(&value[last_i..])); } vm.ctx.new_list(elements) } From 5cfdf1db2c06cb5e502866058e4e382e1af343ce Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 19 Jul 2020 03:02:06 +0900 Subject: [PATCH 042/209] pystr::py_splitlines --- vm/src/obj/objbytearray.rs | 9 +++------ vm/src/obj/objbyteinner.rs | 39 +++++++++----------------------------- vm/src/obj/objbytes.rs | 9 +++------ vm/src/obj/objstr.rs | 35 ++++++---------------------------- vm/src/obj/pystr.rs | 35 ++++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 71 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 36d032e21..64d311427 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -451,13 +451,10 @@ impl PyByteArray { #[pymethod(name = "splitlines")] fn splitlines(&self, options: pystr::SplitLinesArgs, vm: &VirtualMachine) -> PyResult { - let as_bytes = self + let lines = self .borrow_value() - .splitlines(options) - .iter() - .map(|x| vm.ctx.new_bytearray(x.to_vec())) - .collect::>(); - Ok(vm.ctx.new_list(as_bytes)) + .splitlines(options, |x| vm.ctx.new_bytearray(x.to_vec())); + Ok(vm.ctx.new_list(lines)) } #[pymethod(name = "zfill")] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 309278105..b2049aac6 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -961,36 +961,11 @@ impl PyByteInner { res } - pub fn splitlines(&self, options: pystr::SplitLinesArgs) -> Vec<&[u8]> { - let keep = if options.keepends { 1 } else { 0 }; - let value = &self.elements; - let mut elements = Vec::new(); - let mut last_i = 0; - let mut chars = value.iter().enumerate().peekable(); - while let Some((i, ch)) = chars.next() { - let (end_len, i_diff) = match ch { - b'\n' => (keep, 1), - b'\r' => { - let is_rn = chars.peek().map_or(false, |(_, ch)| *ch == &b'\n'); - if is_rn { - let _ = chars.next(); - (keep + keep, 2) - } else { - (keep, 1) - } - } - _ => { - continue; - } - }; - let range = last_i..i + end_len; - last_i = i + i_diff; - elements.push(&value[range]); - } - if last_i != value.len() { - elements.push(&value[last_i..]); - } - elements + pub fn splitlines(&self, options: pystr::SplitLinesArgs, into_wrapper: FW) -> Vec + where + FW: Fn(&[u8]) -> W, + { + self.elements.py_splitlines(options, into_wrapper) } pub fn zfill(&self, width: isize) -> Vec { @@ -1319,6 +1294,10 @@ impl PyCommonString for [u8] { Vec::with_capacity(capacity) } + fn as_bytes(&self) -> &[u8] { + self + } + fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { &self[range] } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index b64656de0..19d9f71e7 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -418,13 +418,10 @@ impl PyBytes { #[pymethod(name = "splitlines")] fn splitlines(&self, options: pystr::SplitLinesArgs, vm: &VirtualMachine) -> PyResult { - let as_bytes = self + let lines = self .inner - .splitlines(options) - .iter() - .map(|x| vm.ctx.new_bytes(x.to_vec())) - .collect::>(); - Ok(vm.ctx.new_list(as_bytes)) + .splitlines(options, |x| vm.ctx.new_bytes(x.to_vec())); + Ok(vm.ctx.new_list(lines)) } #[pymethod(name = "zfill")] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 5ddfb5868..dc1959d3d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -803,35 +803,8 @@ impl PyString { #[pymethod] fn splitlines(&self, args: pystr::SplitLinesArgs, vm: &VirtualMachine) -> PyObjectRef { - let keep = if args.keepends { 1 } else { 0 }; - let value = &self.value; - let mut elements = Vec::new(); - let mut last_i = 0; - let mut chars = value.chars().enumerate().peekable(); - while let Some((i, ch)) = chars.next() { - let (end_len, i_diff) = match ch { - '\n' => (keep, 1), - '\r' => { - let is_rn = chars.peek().map_or(false, |(_, ch)| *ch == '\n'); - if is_rn { - let _ = chars.next(); - (keep + keep, 2) - } else { - (keep, 1) - } - } - _ => { - continue; - } - }; - let range = last_i..i + end_len; - last_i = i + i_diff; - elements.push(vm.ctx.new_str(&value[range])); - } - if last_i != value.len() { - elements.push(vm.ctx.new_str(&value[last_i..])); - } - vm.ctx.new_list(elements) + vm.ctx + .new_list(self.value.py_splitlines(args, |s| vm.new_str(s.to_owned()))) } #[pymethod] @@ -1760,6 +1733,10 @@ impl PyCommonString for str { String::with_capacity(capacity) } + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { &self[range] } diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 1e09386b4..6ac9e83fb 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -129,6 +129,7 @@ pub trait PyCommonString { type Container; fn with_capacity(capacity: usize) -> Self::Container; + fn as_bytes(&self) -> &[u8]; fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self; // FIXME: get_chars is expensive for str fn get_chars<'a>(&'a self, range: std::ops::Range) -> &'a Self; @@ -297,4 +298,38 @@ pub trait PyCommonString { &self } } + + fn py_splitlines(&self, options: SplitLinesArgs, into_wrapper: FW) -> Vec + where + FW: Fn(&Self) -> W, + { + let keep = if options.keepends { 1 } else { 0 }; + let mut elements = Vec::new(); + let mut last_i = 0; + let mut enumerated = self.as_bytes().iter().enumerate().peekable(); + while let Some((i, ch)) = enumerated.next() { + let (end_len, i_diff) = match *ch { + b'\n' => (keep, 1), + b'\r' => { + let is_rn = enumerated.peek().map_or(false, |(_, ch)| **ch == b'\n'); + if is_rn { + let _ = enumerated.next(); + (keep + keep, 2) + } else { + (keep, 1) + } + } + _ => { + continue; + } + }; + let range = last_i..i + end_len; + last_i = i + i_diff; + elements.push(into_wrapper(self.get_bytes(range))); + } + if last_i != self.bytes_len() { + elements.push(into_wrapper(self.get_bytes(last_i..self.bytes_len()))); + } + elements + } } From e674b9f99e2747080d01d02bdd33410277e86f1e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 03:20:55 +0900 Subject: [PATCH 043/209] Use .to_usize().unwrap_or(0) rather than .max(0) --- vm/src/obj/objlist.rs | 2 +- vm/src/sequence.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index c5b992cac..30b5b27f0 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -122,7 +122,7 @@ impl PyList { position }; // Bound it by [0, vec.len()] - let position = unbounded_position.max(0).min(vec_len).to_usize().unwrap(); + let position = unbounded_position.min(vec_len).to_usize().unwrap_or(0); elements.insert(position, element.clone()); } diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 2f062b545..1672be6e4 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -1,5 +1,6 @@ use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use num_traits::cast::ToPrimitive; use parking_lot::{RwLockReadGuard, RwLockWriteGuard}; use std::ops::Deref; @@ -187,7 +188,7 @@ impl<'a> Iterator for SeqMul<'a> { pub(crate) fn seq_mul(seq: &impl SimpleSeq, repetitions: isize) -> SeqMul { SeqMul { seq, - repetitions: repetitions.max(0) as usize, + repetitions: repetitions.to_usize().unwrap_or(0), iter: None, } } From f40b19927fb8464c09eb64812037564eb3451f8b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 04:45:55 +0900 Subject: [PATCH 044/209] Simplify SimpleSeq --- vm/src/obj/objlist.rs | 4 +-- vm/src/sequence.rs | 66 ++++-------------------------------- vm/src/stdlib/collections.rs | 29 +++++++++++++--- 3 files changed, 33 insertions(+), 66 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 30b5b27f0..dc66341eb 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -438,7 +438,7 @@ impl PyList { #[pymethod(name = "__mul__")] fn mul(&self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = sequence::seq_mul(&self.borrow_elements(), counter) + let new_elements = sequence::seq_mul(&*self.borrow_elements(), counter) .cloned() .collect(); vm.ctx.new_list(new_elements) @@ -453,7 +453,7 @@ impl PyList { fn imul(zelf: PyRef, counter: isize) -> PyRef { let mut elements = zelf.borrow_elements_mut(); let mut new_elements: Vec = - sequence::seq_mul(&elements, counter).cloned().collect(); + sequence::seq_mul(&*elements, counter).cloned().collect(); std::mem::swap(elements.deref_mut(), &mut new_elements); zelf.clone() } diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 1672be6e4..38bc843fe 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -1,82 +1,28 @@ use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; use num_traits::cast::ToPrimitive; -use parking_lot::{RwLockReadGuard, RwLockWriteGuard}; -use std::ops::Deref; -type DynPyIter<'a> = Box + 'a>; +pub(super) type DynPyIter<'a> = Box + 'a>; #[allow(clippy::len_without_is_empty)] -pub trait SimpleSeq { +pub(crate) trait SimpleSeq { fn len(&self) -> usize; fn iter(&self) -> DynPyIter; } -// impl SimpleSeq for &[PyObjectRef] { -// fn len(&self) -> usize { -// (&**self).len() -// } -// fn iter(&self) -> DynPyIter { -// Box::new((&**self).iter()) -// } -// } - -impl SimpleSeq for Vec { - fn len(&self) -> usize { - self.len() - } - fn iter(&self) -> DynPyIter { - Box::new(self.as_slice().iter()) - } -} - -impl SimpleSeq for std::collections::VecDeque { - fn len(&self) -> usize { - self.len() - } - fn iter(&self) -> DynPyIter { - Box::new(self.iter()) - } -} - -impl SimpleSeq for std::cell::Ref<'_, T> +impl<'a, D> SimpleSeq for D where - T: SimpleSeq, + D: 'a + std::ops::Deref, { fn len(&self) -> usize { self.deref().len() } + fn iter(&self) -> DynPyIter { - self.deref().iter() + Box::new(self.deref().iter()) } } -impl SimpleSeq for RwLockReadGuard<'_, T> -where - T: SimpleSeq, -{ - fn len(&self) -> usize { - self.deref().len() - } - fn iter(&self) -> DynPyIter { - self.deref().iter() - } -} - -impl SimpleSeq for RwLockWriteGuard<'_, T> -where - T: SimpleSeq, -{ - fn len(&self) -> usize { - self.deref().len() - } - fn iter(&self) -> DynPyIter { - self.deref().iter() - } -} - -// impl<'a, I> - pub(crate) fn eq( vm: &VirtualMachine, zelf: &impl SimpleSeq, diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index 411a28d97..90bd579c5 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -47,6 +47,24 @@ mod _collections { } } + struct SimpleSeqDeque<'a>(RwLockReadGuard<'a, VecDeque>); + + impl sequence::SimpleSeq for SimpleSeqDeque<'_> { + fn len(&self) -> usize { + self.0.len() + } + + fn iter(&self) -> sequence::DynPyIter { + Box::new(self.0.iter()) + } + } + + impl<'a> From>> for SimpleSeqDeque<'a> { + fn from(from: RwLockReadGuard<'a, VecDeque>) -> Self { + Self { 0: from } + } + } + #[pyimpl(flags(BASETYPE))] impl PyDeque { #[pyslot] @@ -267,10 +285,13 @@ mod _collections { vm: &VirtualMachine, ) -> PyResult where - F: Fn(&VecDeque, &VecDeque) -> PyResult, + F: Fn(&SimpleSeqDeque, &SimpleSeqDeque) -> PyResult, { let r = if let Some(other) = other.payload_if_subclass::(vm) { - Implemented(op(&*self.borrow_deque(), &*other.borrow_deque())?) + Implemented(op( + &self.borrow_deque().into(), + &other.borrow_deque().into(), + )?) } else { NotImplemented }; @@ -353,8 +374,8 @@ mod _collections { #[pymethod(name = "__mul__")] fn mul(&self, n: isize) -> Self { - let deque: &VecDeque<_> = &self.borrow_deque(); - let mul = sequence::seq_mul(deque, n); + let deque: SimpleSeqDeque = self.borrow_deque().into(); + let mul = sequence::seq_mul(&deque, n); let skipped = if let Some(maxlen) = self.maxlen.load() { mul.len() - maxlen } else { From 826dc900f8c18541eba8aedb47cc549a8730176a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 05:49:04 +0900 Subject: [PATCH 045/209] SeqMul check non-zero length only once --- vm/src/sequence.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 38bc843fe..394e47992 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -108,9 +108,6 @@ impl ExactSizeIterator for SeqMul<'_> {} impl<'a> Iterator for SeqMul<'a> { type Item = &'a PyObjectRef; fn next(&mut self) -> Option { - if self.seq.len() == 0 { - return None; - } match self.iter.as_mut().and_then(Iterator::next) { Some(item) => Some(item), None => { @@ -132,9 +129,14 @@ impl<'a> Iterator for SeqMul<'a> { } pub(crate) fn seq_mul(seq: &impl SimpleSeq, repetitions: isize) -> SeqMul { + let repetitions = if seq.len() > 0 { + repetitions.to_usize().unwrap_or(0) + } else { + 0 + }; SeqMul { seq, - repetitions: repetitions.to_usize().unwrap_or(0), + repetitions, iter: None, } } From 91e8695c935f55f4d0b85e8c502fda3aa3774c29 Mon Sep 17 00:00:00 2001 From: hyperbora Date: Mon, 20 Jul 2020 20:51:11 +0900 Subject: [PATCH 046/209] Mark unsupported tests --- Lib/test/test_urlparse.py | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 4ae6ed338..95d14fe13 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -522,6 +522,8 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(result.url, defrag) self.assertEqual(result.fragment, frag) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') self.assertEqual(p.hostname, "fe80::822a:a8ff:fe49:470c%tESt") @@ -881,6 +883,8 @@ class UrlParseTestCase(unittest.TestCase): errors="ignore") self.assertEqual(result, [('key', '\u0141-')]) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) @@ -945,6 +949,8 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') @@ -996,6 +1002,8 @@ class UrlParseTestCase(unittest.TestCase): expected.append(name) self.assertCountEqual(urllib.parse.__all__, expected) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_urlsplit_normalization(self): # Certain characters should never occur in the netloc, # including under normalization. @@ -1029,6 +1037,8 @@ class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" # In Python 2 this test class was in test_urllib. + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splittype(self): splittype = urllib.parse._splittype self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring')) @@ -1037,6 +1047,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splittype('type:'), ('type', '')) self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splithost(self): splithost = urllib.parse._splithost self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), @@ -1066,6 +1078,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splithost("//example.net/file#"), ('example.net', '/file#')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splituser(self): splituser = urllib.parse._splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), @@ -1079,6 +1093,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'), ('User@example.com:Pass', 'www.python.org:080')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitpasswd(self): # Some of the password examples are not sensible, but it is added to # confirming to RFC2617 and addressing issue4675. @@ -1097,6 +1113,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitpasswd('user'), ('user', None)) self.assertEqual(splitpasswd(':ab'), ('', 'ab')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitport(self): splitport = urllib.parse._splitport self.assertEqual(splitport('parrot:88'), ('parrot', '88')) @@ -1108,6 +1126,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitport('[::1]'), ('[::1]', None)) self.assertEqual(splitport(':88'), ('', '88')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitnport(self): splitnport = urllib.parse._splitnport self.assertEqual(splitnport('parrot:88'), ('parrot', 88)) @@ -1120,6 +1140,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitquery(self): # Normal cases are exercised by other tests; ensure that we also # catch cases with no port specified (testcase ensuring coverage) @@ -1132,6 +1154,8 @@ class Utility_Tests(unittest.TestCase): ('http://python.org/fake', None)) self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splittag(self): splittag = urllib.parse._splittag self.assertEqual(splittag('http://example.com?foo=bar#baz'), @@ -1144,6 +1168,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'), ('http://example.com?foo=bar#baz', 'boo')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitattr(self): splitattr = urllib.parse._splitattr self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'), @@ -1153,6 +1179,8 @@ class Utility_Tests(unittest.TestCase): ('', ['attr1=value1', 'attr2=value2'])) self.assertEqual(splitattr('/path'), ('/path', [])) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_splitvalue(self): # Normal cases are exercised by other tests; test pathological cases # with no key/value pairs. (testcase ensuring coverage) @@ -1163,6 +1191,8 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitvalue('foobar'), ('foobar', None)) self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz')) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON") def test_to_bytes(self): result = urllib.parse._to_bytes('http://www.python.org') self.assertEqual(result, 'http://www.python.org') @@ -1178,6 +1208,8 @@ class Utility_Tests(unittest.TestCase): class DeprecationTest(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') @@ -1185,6 +1217,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splittype() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splithost_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splithost('') @@ -1192,6 +1226,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splithost() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splituser_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splituser('') @@ -1199,6 +1235,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splituser() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitpasswd_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitpasswd('') @@ -1206,6 +1244,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitpasswd() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitport('') @@ -1213,6 +1253,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitport() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitnport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitnport('') @@ -1220,6 +1262,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitnport() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitquery_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitquery('') @@ -1227,6 +1271,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitquery() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splittag_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittag('') @@ -1234,6 +1280,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splittag() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitattr_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitattr('') @@ -1241,6 +1289,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitattr() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_splitvalue_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitvalue('') @@ -1248,6 +1298,8 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitvalue() is deprecated as of 3.8, ' 'use urllib.parse.parse_qsl() instead') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_to_bytes_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.to_bytes('') From 05bd893e3d03180ccd1bf4fffc1d3b1689bb4749 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 12:47:03 -0700 Subject: [PATCH 047/209] Partial support for __init_subclass__ dunder method. This doesn't support passing kwargs to __init_subclass__. --- vm/src/obj/objobject.rs | 5 +++++ vm/src/obj/objtype.rs | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index cd04817a1..c64fa6883 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -138,6 +138,11 @@ impl PyBaseObject { Ok(vm.ctx.not_implemented()) } + #[pyclassmethod(magic)] + fn init_subclass(_cls: PyClassRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.none()) + } + #[pymethod(magic)] pub fn dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { let attributes: PyAttributes = obj.class().get_attributes(); diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index efffead6c..cd7e8a56b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -2,6 +2,7 @@ use parking_lot::RwLock; use std::collections::{HashMap, HashSet}; use std::fmt; +use super::objclassmethod::PyClassMethod; use super::objdict::PyDictRef; use super::objlist::PyList; use super::objmappingproxy::PyMappingProxy; @@ -14,7 +15,7 @@ use crate::pyobject::{ IdProtocol, PyAttributes, PyClassImpl, PyContext, PyIterable, PyLease, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; -use crate::slots::{PyClassSlots, PyTpFlags}; +use crate::slots::{PyClassSlots, PyTpFlags, SlotDescriptor}; use crate::vm::VirtualMachine; use itertools::Itertools; use std::ops::Deref; @@ -301,6 +302,12 @@ impl PyClassRef { } } + if let Some(f) = attributes.get_mut("__init_subclass__") { + if f.class().is(&vm.ctx.function_type()) { + *f = PyClassMethod::new(f.clone()).into_ref(vm).into_object(); + } + } + let typ = new(metatype, name.as_str(), base.clone(), bases, attributes) .map_err(|e| vm.new_type_error(e))?; @@ -327,6 +334,20 @@ impl PyClassRef { } } + if let Some(initter) = typ.get_super_attr("__init_subclass__") { + let initter = PyClassMethod::descr_get( + vm, + initter, + None, + OptionalArg::Present(typ.clone().into_object()), + )?; + let init_args = PyFuncArgs { + args: Vec::new(), + kwargs: args.kwargs, + }; + vm.invoke(&initter, init_args)?; + }; + Ok(typ.into_object()) } From 8e52c29e3b1f5376ee911cdcaa1723b15958e9a2 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 20:12:43 -0700 Subject: [PATCH 048/209] Add test_subclassinit.py from CPython 3.8.5 --- Lib/test/test_subclassinit.py | 284 ++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 Lib/test/test_subclassinit.py diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py new file mode 100644 index 000000000..c007476e0 --- /dev/null +++ b/Lib/test/test_subclassinit.py @@ -0,0 +1,284 @@ +import types +import unittest + + +class Test(unittest.TestCase): + def test_init_subclass(self): + class A: + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_dict(self): + class A(dict): + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_kwargs(self): + class A: + def __init_subclass__(cls, **kwargs): + cls.kwargs = kwargs + + class B(A, x=3): + pass + + self.assertEqual(B.kwargs, dict(x=3)) + + def test_init_subclass_error(self): + class A: + def __init_subclass__(cls): + raise RuntimeError + + with self.assertRaises(RuntimeError): + class B(A): + pass + + def test_init_subclass_wrong(self): + class A: + def __init_subclass__(cls, whatever): + pass + + with self.assertRaises(TypeError): + class B(A): + pass + + def test_init_subclass_skipped(self): + class BaseWithInit: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.initialized = cls + + class BaseWithoutInit(BaseWithInit): + pass + + class A(BaseWithoutInit): + pass + + self.assertIs(A.initialized, A) + self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit) + + def test_init_subclass_diamond(self): + class Base: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls = [] + + class Left(Base): + pass + + class Middle: + def __init_subclass__(cls, middle, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [middle] + + class Right(Base): + def __init_subclass__(cls, right="right", **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [right] + + class A(Left, Middle, Right, middle="middle"): + pass + + self.assertEqual(A.calls, ["right", "middle"]) + self.assertEqual(Left.calls, []) + self.assertEqual(Right.calls, []) + + def test_set_name(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A: + d = Descriptor() + + self.assertEqual(A.d.name, "d") + self.assertIs(A.d.owner, A) + + def test_set_name_metaclass(self): + class Meta(type): + def __new__(cls, name, bases, ns): + ret = super().__new__(cls, name, bases, ns) + self.assertEqual(ret.d.name, "d") + self.assertIs(ret.d.owner, ret) + return 0 + + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A(metaclass=Meta): + d = Descriptor() + self.assertEqual(A, 0) + + def test_set_name_error(self): + class Descriptor: + def __set_name__(self, owner, name): + 1/0 + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, ZeroDivisionError) + + def test_set_name_wrong(self): + class Descriptor: + def __set_name__(self): + pass + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, TypeError) + + def test_set_name_lookup(self): + resolved = [] + class NonDescriptor: + def __getattr__(self, name): + resolved.append(name) + + class A: + d = NonDescriptor() + + self.assertNotIn('__set_name__', resolved, + '__set_name__ is looked up in instance dict') + + def test_set_name_init_subclass(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class Meta(type): + def __new__(cls, name, bases, ns): + self = super().__new__(cls, name, bases, ns) + self.meta_owner = self.owner + self.meta_name = self.name + return self + + class A: + def __init_subclass__(cls): + cls.owner = cls.d.owner + cls.name = cls.d.name + + class B(A, metaclass=Meta): + d = Descriptor() + + self.assertIs(B.owner, B) + self.assertEqual(B.name, 'd') + self.assertIs(B.meta_owner, B) + self.assertEqual(B.name, 'd') + + def test_set_name_modifying_dict(self): + notified = [] + class Descriptor: + def __set_name__(self, owner, name): + setattr(owner, name + 'x', None) + notified.append(name) + + class A: + a = Descriptor() + b = Descriptor() + c = Descriptor() + d = Descriptor() + e = Descriptor() + + self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e']) + + def test_errors(self): + class MyMeta(type): + pass + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + with self.assertRaises(TypeError): + types.new_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + types.prepare_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + + class MyMeta(type): + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + self.otherarg = otherarg + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_errors_changed_pep487(self): + # These tests failed before Python 3.6, PEP 487 + class MyMeta(type): + def __new__(cls, name, bases, namespace): + return super().__new__(cls, name=name, bases=bases, + dict=namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + self = super().__new__(cls, name, bases, namespace) + self.otherarg = otherarg + return self + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_type(self): + t = type('NewClass', (object,), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, 'NewClass') + + with self.assertRaises(TypeError): + type(name='NewClass', bases=(object,), dict={}) + + +if __name__ == "__main__": + unittest.main() + From bb433bfd513beedbeabee9d98c7567d068ba962d Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 20:13:49 -0700 Subject: [PATCH 049/209] Mark expected failures and skipped tests --- Lib/test/test_subclassinit.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py index c007476e0..9579fde93 100644 --- a/Lib/test/test_subclassinit.py +++ b/Lib/test/test_subclassinit.py @@ -112,6 +112,8 @@ class Test(unittest.TestCase): self.assertEqual(A.d.name, "d") self.assertIs(A.d.owner, A) + #TODO: RUSTPYTHON + @unittest.expectedFailure def test_set_name_metaclass(self): class Meta(type): def __new__(cls, name, bases, ns): @@ -197,6 +199,8 @@ class Test(unittest.TestCase): self.assertIs(B.meta_owner, B) self.assertEqual(B.name, 'd') + #TODO: RUSTPYTHON + @unittest.skip("infinite loops") def test_set_name_modifying_dict(self): notified = [] class Descriptor: @@ -213,6 +217,8 @@ class Test(unittest.TestCase): self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e']) + #TODO: RUSTPYTHON + @unittest.expectedFailure def test_errors(self): class MyMeta(type): pass From 8cccf13c806d39a1dd85605d0cba1654a7b1d77e Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 15:41:17 -0700 Subject: [PATCH 050/209] Implemented kwargs on __init_subclass__ --- vm/src/function.rs | 9 +++++++++ vm/src/obj/objtype.rs | 9 +++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index 81881768e..76b0fad40 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -66,6 +66,15 @@ impl From<(&Args, &KwArgs)> for PyFuncArgs { } } +impl From for PyFuncArgs { + fn from(kwargs: KwArgs) -> Self { + PyFuncArgs { + args: Vec::new(), + kwargs: kwargs.into_iter().collect(), + } + } +} + impl FromArgs for PyFuncArgs { fn from_args(_vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { Ok(mem::take(args)) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index cd7e8a56b..db235823b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -10,7 +10,7 @@ use super::objstaticmethod::PyStaticMethod; use super::objstr::PyStringRef; use super::objtuple::PyTuple; use super::objweakref::PyWeak; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, PyAttributes, PyClassImpl, PyContext, PyIterable, PyLease, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, @@ -258,7 +258,7 @@ impl PyClassRef { })); } - let (name, bases, dict): (PyStringRef, PyIterable, PyDictRef) = + let (name, bases, dict, kwargs): (PyStringRef, PyIterable, PyDictRef, KwArgs) = args.clone().bind(vm)?; let bases: Vec = bases.iter(vm)?.collect::, _>>()?; @@ -341,10 +341,7 @@ impl PyClassRef { None, OptionalArg::Present(typ.clone().into_object()), )?; - let init_args = PyFuncArgs { - args: Vec::new(), - kwargs: args.kwargs, - }; + let init_args = PyFuncArgs::from(kwargs); vm.invoke(&initter, init_args)?; }; From 94bafbb1ea9be43224b2cacd0b2795c1bcacbc01 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 19:24:19 -0700 Subject: [PATCH 051/209] Refactored unecessary code out and calls __get__ descriptor when needed --- vm/src/obj/objtype.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index db235823b..0b6dc4560 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -15,7 +15,7 @@ use crate::pyobject::{ IdProtocol, PyAttributes, PyClassImpl, PyContext, PyIterable, PyLease, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; -use crate::slots::{PyClassSlots, PyTpFlags, SlotDescriptor}; +use crate::slots::{PyClassSlots, PyTpFlags}; use crate::vm::VirtualMachine; use itertools::Itertools; use std::ops::Deref; @@ -335,14 +335,10 @@ impl PyClassRef { } if let Some(initter) = typ.get_super_attr("__init_subclass__") { - let initter = PyClassMethod::descr_get( - vm, - initter, - None, - OptionalArg::Present(typ.clone().into_object()), - )?; - let init_args = PyFuncArgs::from(kwargs); - vm.invoke(&initter, init_args)?; + let initter = vm + .call_get_descriptor_specific(initter, None, Some(typ.clone().into_object())) + .unwrap()?; + vm.invoke(&initter, kwargs)?; }; Ok(typ.into_object()) From 3f7ad38458c7a47743c881432e67915a845c5ac1 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Tue, 21 Jul 2020 02:41:31 +0000 Subject: [PATCH 052/209] Update vm/src/obj/objtype.rs Co-authored-by: Noah <33094578+coolreader18@users.noreply.github.com> --- vm/src/obj/objtype.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 0b6dc4560..3a0b73efd 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -336,8 +336,8 @@ impl PyClassRef { if let Some(initter) = typ.get_super_attr("__init_subclass__") { let initter = vm - .call_get_descriptor_specific(initter, None, Some(typ.clone().into_object())) - .unwrap()?; + .call_get_descriptor_specific(initter.clone(), None, Some(typ.clone().into_object())) + .unwrap_or(Ok(initter))?; vm.invoke(&initter, kwargs)?; }; From dd9eb28eb865d2e8ca420abdf54668ce6a445bf5 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 20 Jul 2020 19:46:11 -0700 Subject: [PATCH 053/209] formatting --- vm/src/obj/objtype.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 3a0b73efd..1dff7f790 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -336,7 +336,11 @@ impl PyClassRef { if let Some(initter) = typ.get_super_attr("__init_subclass__") { let initter = vm - .call_get_descriptor_specific(initter.clone(), None, Some(typ.clone().into_object())) + .call_get_descriptor_specific( + initter.clone(), + None, + Some(typ.clone().into_object()), + ) .unwrap_or(Ok(initter))?; vm.invoke(&initter, kwargs)?; }; From b677b88bddefe6a31e076150b441ccd2cc7675a4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 21 Jul 2020 04:40:04 +0900 Subject: [PATCH 054/209] sequence::cmp take DynPyIter instead of SimpleSeq to avoid duplicated code generation for Box operation --- vm/src/obj/objlist.rs | 6 ++-- vm/src/obj/objtuple.rs | 13 +++++--- vm/src/sequence.rs | 60 +++++++++++++----------------------- vm/src/stdlib/collections.rs | 10 +++--- 4 files changed, 37 insertions(+), 52 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index dc66341eb..70139ab45 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -530,12 +530,12 @@ impl PyList { #[inline] fn cmp(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult where - F: Fn(&Vec, &Vec) -> PyResult, + F: Fn(sequence::DynPyIter, sequence::DynPyIter) -> PyResult, { let r = if let Some(other) = other.payload_if_subclass::(vm) { Implemented(op( - &self.borrow_elements().clone(), - &other.borrow_elements().clone(), + self.borrow_elements().boxed_iter(), + other.borrow_elements().boxed_iter(), )?) } else { NotImplemented diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 1b3099473..43570b386 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -11,7 +11,7 @@ use crate::pyobject::{ PyArithmaticValue::{self, *}, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; -use crate::sequence; +use crate::sequence::{self, SimpleSeq}; use crate::vm::{ReprGuard, VirtualMachine}; /// tuple() -> empty tuple @@ -81,10 +81,13 @@ impl PyTuple { #[inline] fn cmp(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult where - F: Fn(&Vec, &Vec) -> PyResult, + F: Fn(sequence::DynPyIter, sequence::DynPyIter) -> PyResult, { let r = if let Some(other) = other.payload_if_subclass::(vm) { - Implemented(op(&self.elements, &other.elements)?) + Implemented(op( + self.as_slice().boxed_iter(), + other.as_slice().boxed_iter(), + )?) } else { NotImplemented }; @@ -116,8 +119,8 @@ impl PyTuple { if let Some(other) = other.payload_if_subclass::(vm) { let elements: Vec<_> = self .elements - .iter() - .chain(other.as_slice().iter()) + .boxed_iter() + .chain(other.as_slice().boxed_iter()) .cloned() .collect(); Implemented(elements.into()) diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 394e47992..557569218 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -7,7 +7,7 @@ pub(super) type DynPyIter<'a> = Box usize; - fn iter(&self) -> DynPyIter; + fn boxed_iter(&self) -> DynPyIter; } impl<'a, D> SimpleSeq for D @@ -18,18 +18,14 @@ where self.deref().len() } - fn iter(&self) -> DynPyIter { + fn boxed_iter(&self) -> DynPyIter { Box::new(self.deref().iter()) } } -pub(crate) fn eq( - vm: &VirtualMachine, - zelf: &impl SimpleSeq, - other: &impl SimpleSeq, -) -> PyResult { +pub(crate) fn eq(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { if zelf.len() == other.len() { - for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { + for (a, b) in Iterator::zip(zelf, other) { if a.is(b) { continue; } @@ -43,58 +39,44 @@ pub(crate) fn eq( } } -pub(crate) fn lt( - vm: &VirtualMachine, - zelf: &impl SimpleSeq, - other: &impl SimpleSeq, -) -> PyResult { - for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { +pub(crate) fn lt(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + let fallback = zelf.len() < other.len(); + for (a, b) in Iterator::zip(zelf, other) { if let Some(v) = vm.bool_seq_lt(a.clone(), b.clone())? { return Ok(v); } } - Ok(zelf.len() < other.len()) + Ok(fallback) } -pub(crate) fn gt( - vm: &VirtualMachine, - zelf: &impl SimpleSeq, - other: &impl SimpleSeq, -) -> PyResult { - for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { +pub(crate) fn gt(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + let fallback = zelf.len() > other.len(); + for (a, b) in Iterator::zip(zelf, other) { if let Some(v) = vm.bool_seq_gt(a.clone(), b.clone())? { return Ok(v); } } - Ok(zelf.len() > other.len()) + Ok(fallback) } -pub(crate) fn ge( - vm: &VirtualMachine, - zelf: &impl SimpleSeq, - other: &impl SimpleSeq, -) -> PyResult { - for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { +pub(crate) fn ge(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + let fallback = zelf.len() >= other.len(); + for (a, b) in Iterator::zip(zelf, other) { if let Some(v) = vm.bool_seq_gt(a.clone(), b.clone())? { return Ok(v); } } - - Ok(zelf.len() >= other.len()) + Ok(fallback) } -pub(crate) fn le( - vm: &VirtualMachine, - zelf: &impl SimpleSeq, - other: &impl SimpleSeq, -) -> PyResult { - for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { +pub(crate) fn le(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + let fallback = zelf.len() <= other.len(); + for (a, b) in Iterator::zip(zelf, other) { if let Some(v) = vm.bool_seq_lt(a.clone(), b.clone())? { return Ok(v); } } - - Ok(zelf.len() <= other.len()) + Ok(fallback) } pub(crate) struct SeqMul<'a> { @@ -115,7 +97,7 @@ impl<'a> Iterator for SeqMul<'a> { None } else { self.repetitions -= 1; - self.iter = Some(self.seq.iter()); + self.iter = Some(self.seq.boxed_iter()); self.next() } } diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index 90bd579c5..a959970b8 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -8,7 +8,7 @@ mod _collections { IdProtocol, PyArithmaticValue::*, PyClassImpl, PyComparisonValue, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, }; - use crate::sequence; + use crate::sequence::{self, SimpleSeq}; use crate::vm::ReprGuard; use crate::VirtualMachine; use itertools::Itertools; @@ -54,7 +54,7 @@ mod _collections { self.0.len() } - fn iter(&self) -> sequence::DynPyIter { + fn boxed_iter(&self) -> sequence::DynPyIter { Box::new(self.0.iter()) } } @@ -285,12 +285,12 @@ mod _collections { vm: &VirtualMachine, ) -> PyResult where - F: Fn(&SimpleSeqDeque, &SimpleSeqDeque) -> PyResult, + F: Fn(sequence::DynPyIter, sequence::DynPyIter) -> PyResult, { let r = if let Some(other) = other.payload_if_subclass::(vm) { Implemented(op( - &self.borrow_deque().into(), - &other.borrow_deque().into(), + SimpleSeqDeque::from(self.borrow_deque()).boxed_iter(), + SimpleSeqDeque::from(other.borrow_deque()).boxed_iter(), )?) } else { NotImplemented From 7f7e0ac7700bb5369e42360467d7bd416df653d0 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2020 03:36:09 +0900 Subject: [PATCH 055/209] unify sequence::cmp --- vm/src/sequence.rs | 58 +++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 557569218..db9152f77 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -39,44 +39,34 @@ pub(crate) fn eq(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyRe } } +fn cmp(zelf: DynPyIter, other: DynPyIter, len_cmp: L, obj_cmp: O) -> PyResult +where + L: Fn(usize, usize) -> bool, + O: Fn(PyObjectRef, PyObjectRef) -> PyResult>, +{ + let fallback = len_cmp(zelf.len(), other.len()); + for (a, b) in Iterator::zip(zelf, other) { + if let Some(v) = obj_cmp(a.clone(), b.clone())? { + return Ok(v); + } + } + Ok(fallback) +} + pub(crate) fn lt(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { - let fallback = zelf.len() < other.len(); - for (a, b) in Iterator::zip(zelf, other) { - if let Some(v) = vm.bool_seq_lt(a.clone(), b.clone())? { - return Ok(v); - } - } - Ok(fallback) -} - -pub(crate) fn gt(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { - let fallback = zelf.len() > other.len(); - for (a, b) in Iterator::zip(zelf, other) { - if let Some(v) = vm.bool_seq_gt(a.clone(), b.clone())? { - return Ok(v); - } - } - Ok(fallback) -} - -pub(crate) fn ge(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { - let fallback = zelf.len() >= other.len(); - for (a, b) in Iterator::zip(zelf, other) { - if let Some(v) = vm.bool_seq_gt(a.clone(), b.clone())? { - return Ok(v); - } - } - Ok(fallback) + cmp(zelf, other, |a, b| a < b, |a, b| vm.bool_seq_lt(a, b)) } pub(crate) fn le(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { - let fallback = zelf.len() <= other.len(); - for (a, b) in Iterator::zip(zelf, other) { - if let Some(v) = vm.bool_seq_lt(a.clone(), b.clone())? { - return Ok(v); - } - } - Ok(fallback) + cmp(zelf, other, |a, b| a <= b, |a, b| vm.bool_seq_lt(a, b)) +} + +pub(crate) fn gt(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + cmp(zelf, other, |a, b| a > b, |a, b| vm.bool_seq_gt(a, b)) +} + +pub(crate) fn ge(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { + cmp(zelf, other, |a, b| a >= b, |a, b| vm.bool_seq_gt(a, b)) } pub(crate) struct SeqMul<'a> { From 93c80ebe7d2166cc2a64bf6e9d16a141c3a0ba8f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2020 03:36:16 +0900 Subject: [PATCH 056/209] Remove redundant clone --- vm/src/vm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index f1f28fe13..c629d8f23 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1469,7 +1469,7 @@ impl VirtualMachine { pub fn bool_seq_lt(&self, a: PyObjectRef, b: PyObjectRef) -> PyResult> { let value = if objbool::boolval(self, self._lt(a.clone(), b.clone())?)? { Some(true) - } else if !objbool::boolval(self, self._eq(a.clone(), b.clone())?)? { + } else if !objbool::boolval(self, self._eq(a, b)?)? { Some(false) } else { None @@ -1480,7 +1480,7 @@ impl VirtualMachine { pub fn bool_seq_gt(&self, a: PyObjectRef, b: PyObjectRef) -> PyResult> { let value = if objbool::boolval(self, self._gt(a.clone(), b.clone())?)? { Some(true) - } else if !objbool::boolval(self, self._eq(a.clone(), b.clone())?)? { + } else if !objbool::boolval(self, self._eq(a, b)?)? { Some(false) } else { None From d52309255e9041ddc70062be2073096d40e5dc79 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 20 Jul 2020 19:54:48 -0500 Subject: [PATCH 057/209] Fix compilation on mingw/msys --- vm/src/stdlib/os.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b66a284d4..c9ff3c116 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1413,7 +1413,7 @@ pub type Offset = libc::off_t; #[cfg(windows)] pub type Offset = libc::c_longlong; -#[cfg(windows)] +#[cfg(all(windows, target_env = "msvc"))] type InvalidParamHandler = extern "C" fn( *const libc::wchar_t, *const libc::wchar_t, @@ -1421,7 +1421,7 @@ type InvalidParamHandler = extern "C" fn( libc::c_uint, libc::uintptr_t, ); -#[cfg(windows)] +#[cfg(all(windows, target_env = "msvc"))] extern "C" { #[doc(hidden)] pub fn _set_thread_local_invalid_parameter_handler( @@ -1429,7 +1429,7 @@ extern "C" { ) -> InvalidParamHandler; } -#[cfg(windows)] +#[cfg(all(windows, target_env = "msvc"))] #[doc(hidden)] pub extern "C" fn silent_iph_handler( _: *const libc::wchar_t, @@ -1443,7 +1443,7 @@ pub extern "C" fn silent_iph_handler( #[macro_export] macro_rules! suppress_iph { ($e:expr) => {{ - #[cfg(windows)] + #[cfg(all(windows, target_env = "msvc"))] { let old = $crate::stdlib::os::_set_thread_local_invalid_parameter_handler( $crate::stdlib::os::silent_iph_handler, @@ -1452,7 +1452,7 @@ macro_rules! suppress_iph { $crate::stdlib::os::_set_thread_local_invalid_parameter_handler(old); ret } - #[cfg(not(windows))] + #[cfg(not(all(windows, target_env = "msvc")))] { $e } From eeaf1d5092de7aba1288aab85008fd16733d215e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 02:27:20 +0900 Subject: [PATCH 058/209] {str,bytes}.join with pystr --- vm/src/obj/objbyteinner.rs | 38 ++++++++++++++++++++++++-------------- vm/src/obj/objstr.rs | 35 +++++++++++++++++++++-------------- vm/src/obj/pystr.rs | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b2049aac6..77fc193a0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -13,7 +13,7 @@ use super::objnone::PyNoneRef; use super::objsequence::{PySliceableSequence, SequenceIndex}; use super::objslice::PySliceRef; use super::objstr::{self, PyString, PyStringRef}; -use super::pystr::{self, PyCommonString, PyCommonStringWrapper}; +use super::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyhash; use crate::pyobject::{ @@ -739,17 +739,13 @@ impl PyByteInner { .py_count(needle.as_slice(), range, |h, n| h.find_iter(n).count())) } - pub fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult> { - let mut refs = Vec::new(); - for v in iter.iter(vm)? { - let v = v?; - if !refs.is_empty() { - refs.extend(&self.elements); - } - refs.extend(v.elements); - } - - Ok(refs) + pub fn join( + &self, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult> { + let iter = iterable.iter(vm)?; + self.elements.py_join(iter) } #[inline] @@ -1285,13 +1281,27 @@ impl PyCommonStringWrapper<[u8]> for PyByteInner { } } +impl PyCommonStringContainer<[u8]> for Vec { + fn new() -> Self { + Vec::new() + } + + fn with_capacity(capacity: usize) -> Self { + Vec::with_capacity(capacity) + } + + fn push_str(&mut self, other: &[u8]) { + self.extend(other) + } +} + const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; impl PyCommonString for [u8] { type Container = Vec; - fn with_capacity(capacity: usize) -> Self::Container { - Vec::with_capacity(capacity) + fn to_container(&self) -> Self::Container { + self.to_vec() } fn as_bytes(&self) -> &[u8] { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index dc1959d3d..ed6ad00a2 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -22,7 +22,9 @@ use super::objnone::PyNone; use super::objsequence::{PySliceableSequence, SequenceIndex}; use super::objtuple; use super::objtype::{self, PyClassRef}; -use super::pystr::{self, adjust_indices, PyCommonString, PyCommonStringWrapper}; +use super::pystr::{ + self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, +}; use crate::cformat::{ CFormatPart, CFormatPreconversor, CFormatQuantity, CFormatSpec, CFormatString, CFormatType, CNumberType, @@ -809,17 +811,8 @@ impl PyString { #[pymethod] fn join(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut joined = String::new(); - - for (idx, elem) in iterable.iter(vm)?.enumerate() { - let elem = elem?; - if idx != 0 { - joined.push_str(&self.value); - } - joined.push_str(&elem.value) - } - - Ok(joined) + let iter = iterable.iter(vm)?; + self.value.py_join(iter) } #[inline] @@ -1726,11 +1719,25 @@ impl PyCommonStringWrapper for PyStringRef { } } +impl PyCommonStringContainer for String { + fn new() -> Self { + String::new() + } + + fn with_capacity(capacity: usize) -> Self { + String::with_capacity(capacity) + } + + fn push_str(&mut self, other: &str) { + String::push_str(self, other) + } +} + impl PyCommonString for str { type Container = String; - fn with_capacity(capacity: usize) -> Self::Container { - String::with_capacity(capacity) + fn to_container(&self) -> Self::Container { + self.to_owned() } fn as_bytes(&self) -> &[u8] { diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 6ac9e83fb..1a32033b2 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -1,6 +1,6 @@ use crate::function::{single_or_tuple_any, OptionalOption}; use crate::obj::objint::PyIntRef; -use crate::pyobject::{PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use crate::pyobject::{PyIterator, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use num_traits::{cast::ToPrimitive, sign::Signed}; @@ -125,10 +125,22 @@ where fn as_ref(&self) -> &S; } -pub trait PyCommonString { +pub trait PyCommonStringContainer +where + S: ?Sized, +{ + fn new() -> Self; + fn with_capacity(capacity: usize) -> Self; + fn push_str(&mut self, s: &S); +} + +pub trait PyCommonString +where + Self::Container: PyCommonStringContainer, +{ type Container; - fn with_capacity(capacity: usize) -> Self::Container; + fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self; // FIXME: get_chars is expensive for str @@ -266,6 +278,23 @@ pub trait PyCommonString { self.py_pad(width - self.chars_len(), 0, fillchar) } + fn py_join<'a>( + &self, + mut iter: PyIterator<'a, impl PyCommonStringWrapper + TryFromObject>, + ) -> PyResult { + let mut joined = if let Some(elem) = iter.next() { + elem?.as_ref().to_container() + } else { + return Ok(Self::Container::new()); + }; + for elem in iter { + let elem = elem?; + joined.push_str(self); + joined.push_str(elem.as_ref()); + } + Ok(joined) + } + fn py_removeprefix( &self, prefix: &Self::Container, From 9ac868737336cef3786b0111cc35752898aa236f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 02:42:05 +0900 Subject: [PATCH 059/209] Use to_owned when its intension is not display --- vm/src/obj/objstr.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ed6ad00a2..c0696e05b 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -537,7 +537,7 @@ impl PyString { self.value .as_str() .py_removeprefix(&pref.value, pref.value.len(), |s, p| s.starts_with(p)) - .to_string() + .to_owned() } /// removesuffix(self, prefix, /) @@ -552,7 +552,7 @@ impl PyString { self.value .as_str() .py_removesuffix(&suff.value, suff.value.len(), |s, p| s.ends_with(p)) - .to_string() + .to_owned() } #[pymethod] From 26a43f9a639566a84c31203e8c3d1b72dbf45264 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 03:00:41 +0900 Subject: [PATCH 060/209] zfill into pystr --- vm/src/obj/objbyteinner.rs | 20 +------------------- vm/src/obj/objstr.rs | 9 ++------- vm/src/obj/pystr.rs | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 77fc193a0..b0ba626ca 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -965,7 +965,7 @@ impl PyByteInner { } pub fn zfill(&self, width: isize) -> Vec { - bytes_zfill(&self.elements, width.to_usize().unwrap_or(0)) + self.elements.py_zfill(width) } // len(self)>=1, from="", len(to)>=1, maxcount>=1 @@ -1257,24 +1257,6 @@ impl PyBytesLike { } } -pub fn bytes_zfill(bytes: &[u8], width: usize) -> Vec { - if width <= bytes.len() { - bytes.to_vec() - } else { - let (sign, s) = match bytes.first() { - Some(_sign @ b'+') | Some(_sign @ b'-') => { - (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) - } - _ => (&b""[..], bytes), - }; - let mut filled = Vec::new(); - filled.extend_from_slice(sign); - filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); - filled.extend_from_slice(s); - filled - } -} - impl PyCommonStringWrapper<[u8]> for PyByteInner { fn as_ref(&self) -> &[u8] { &self.elements diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index c0696e05b..1abf673d1 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -927,13 +927,8 @@ impl PyString { #[pymethod] fn zfill(&self, width: isize) -> String { - use super::objbyteinner::bytes_zfill; - unsafe { - String::from_utf8_unchecked(bytes_zfill( - self.value.as_bytes(), - width.to_usize().unwrap_or(0), - )) - } + // this is safe-guaranteed because the original self.value is valid utf8 + unsafe { String::from_utf8_unchecked(self.value.py_zfill(width)) } } fn get_fill_char(fillchar: OptionalArg, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 1a32033b2..d1e370d1b 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -361,4 +361,24 @@ where } elements } + + fn py_zfill(&self, width: isize) -> Vec { + let bytes = self.as_bytes(); + let width = width.to_usize().unwrap_or(0); + if width <= bytes.len() { + bytes.to_vec() + } else { + let (sign, s) = match bytes.first() { + Some(_sign @ b'+') | Some(_sign @ b'-') => { + (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) + } + _ => (&b""[..], bytes), + }; + let mut filled = Vec::new(); + filled.extend_from_slice(sign); + filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); + filled.extend_from_slice(s); + filled + } + } } From 59beca2626d008c34f638f8fbe2c14657735c4e5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 20 Jul 2020 03:10:58 +0900 Subject: [PATCH 061/209] use BStr::repeat instead of original implementation --- vm/src/obj/objbytearray.rs | 23 +++++++++++------------ vm/src/obj/objbyteinner.rs | 17 ++--------------- vm/src/obj/objbytes.rs | 2 +- vm/src/obj/objstr.rs | 2 +- 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 64d311427..aea9e6372 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,8 +1,8 @@ //! Implementation of the python bytearray object. use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; +use num_traits::cast::ToPrimitive; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use std::convert::TryFrom; use std::mem::size_of; use std::str::FromStr; @@ -506,8 +506,10 @@ impl PyByteArray { #[pymethod(name = "insert")] fn insert(&self, mut index: isize, x: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { let bytes = &mut self.borrow_value_mut().elements; - let len = isize::try_from(bytes.len()) - .map_err(|_e| vm.new_overflow_error("bytearray too big".to_owned()))?; + let len = bytes + .len() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("bytearray too big".to_owned()))?; let x = x.as_bigint().byte_or(vm)?; @@ -521,8 +523,9 @@ impl PyByteArray { index = index.max(0); } - let index = usize::try_from(index) - .map_err(|_e| vm.new_overflow_error("overflow in index calculation".to_owned()))?; + let index = index + .to_usize() + .ok_or_else(|| vm.new_overflow_error("overflow in index calculation".to_owned()))?; bytes.insert(index, x); @@ -543,17 +546,13 @@ impl PyByteArray { } #[pymethod(name = "__mul__")] - fn repeat(&self, n: isize) -> PyByteArray { + #[pymethod(name = "__rmul__")] + fn mul(&self, n: isize) -> PyByteArray { self.borrow_value().repeat(n).into() } - #[pymethod(name = "__rmul__")] - fn rmul(&self, n: isize) -> PyByteArray { - self.repeat(n) - } - #[pymethod(name = "__imul__")] - fn irepeat(&self, n: isize) { + fn imul(&self, n: isize) { self.borrow_value_mut().irepeat(n) } diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b0ba626ca..ef01269cf 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,7 +1,6 @@ use bstr::ByteSlice; use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; -use std::convert::TryFrom; use std::ops::Range; use super::objbytearray::{PyByteArray, PyByteArrayRef}; @@ -1157,19 +1156,7 @@ impl PyByteInner { } pub fn repeat(&self, n: isize) -> Vec { - if self.elements.is_empty() || n <= 0 { - // We can multiple an empty vector by any integer, even if it doesn't fit in an isize. - Vec::new() - } else { - let n = usize::try_from(n).unwrap(); - - let mut new_value = Vec::with_capacity(n * self.elements.len()); - for _ in 0..n { - new_value.extend(&self.elements); - } - - new_value - } + self.elements.repeat(n.to_usize().unwrap_or(0)) } pub fn irepeat(&mut self, n: isize) { @@ -1181,7 +1168,7 @@ impl PyByteInner { if n <= 0 { self.elements.clear(); } else { - let n = usize::try_from(n).unwrap(); + let n = n.to_usize().unwrap(); // always positive by outer if condition let old = self.elements.clone(); diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 19d9f71e7..e5ed355ff 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -447,7 +447,7 @@ impl PyBytes { #[pymethod(name = "__mul__")] #[pymethod(name = "__rmul__")] - fn repeat(&self, value: isize, vm: &VirtualMachine) -> PyResult { + fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult { if value > 0 && self.inner.len() as isize > std::isize::MAX / value { return Err(vm.new_overflow_error("repeated bytes are too long".to_owned())); } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 1abf673d1..daf225ac5 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -314,7 +314,7 @@ impl PyString { #[pymethod(name = "__mul__")] #[pymethod(name = "__rmul__")] fn mul(&self, value: isize) -> String { - self.value.repeat(value.max(0) as usize) + self.value.repeat(value.to_usize().unwrap_or(0)) } #[pymethod(name = "__str__")] From 56850163a028e93b742ecf4465245575d5abdfb1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2020 03:57:13 +0900 Subject: [PATCH 062/209] pystr py_iscase --- vm/src/obj/objbyteinner.rs | 35 +++++++++++----------------------- vm/src/obj/objstr.rs | 31 +++++++++--------------------- vm/src/obj/pystr.rs | 39 +++++++++++++++++++++++++++++++------- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index ef01269cf..4aa81874f 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -229,7 +229,7 @@ impl ByteInnerTranslateOptions { } } -pub type ByteInnerSplitOptions = pystr::SplitArgs; +pub type ByteInnerSplitOptions<'a> = pystr::SplitArgs<'a, PyByteInner, [u8], u8>; #[derive(FromArgs)] pub struct ByteInnerSplitlinesOptions { @@ -557,31 +557,13 @@ impl PyByteInner { } pub fn islower(&self) -> bool { - // CPython _Py_bytes_islower - let mut cased = false; - for b in self.elements.iter() { - let c = *b as char; - if c.is_uppercase() { - return false; - } else if !cased && c.is_lowercase() { - cased = true - } - } - cased + self.elements + .py_iscase(char::is_lowercase, char::is_uppercase) } pub fn isupper(&self) -> bool { - // CPython _Py_bytes_isupper - let mut cased = false; - for b in self.elements.iter() { - let c = *b as char; - if c.is_lowercase() { - return false; - } else if !cased && c.is_uppercase() { - cased = true - } - } - cased + self.elements + .py_iscase(char::is_uppercase, char::is_lowercase) } pub fn isspace(&self) -> bool { @@ -1266,8 +1248,9 @@ impl PyCommonStringContainer<[u8]> for Vec { const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; -impl PyCommonString for [u8] { +impl<'s> PyCommonString<'s, u8> for [u8] { type Container = Vec; + type CharIter = std::iter::Map, fn(&u8) -> char>; fn to_container(&self) -> Self::Container { self.to_vec() @@ -1277,6 +1260,10 @@ impl PyCommonString for [u8] { self } + fn chars(&'s self) -> Self::CharIter { + self.iter().map(|c| *c as char) + } + fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { &self[range] } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index daf225ac5..f4191f547 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -771,31 +771,13 @@ impl PyString { // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise. #[pymethod] fn islower(&self) -> bool { - // CPython unicode_islower_impl - let mut cased = false; - for c in self.value.chars() { - if c.is_uppercase() { - return false; - } else if !cased && c.is_lowercase() { - cased = true - } - } - cased + self.value.py_iscase(char::is_lowercase, char::is_uppercase) } // Return true if all cased characters in the string are uppercase and there is at least one cased character, false otherwise. #[pymethod] fn isupper(&self) -> bool { - // CPython unicode_isupper_impl - let mut cased = false; - for c in self.value.chars() { - if c.is_lowercase() { - return false; - } else if !cased && c.is_uppercase() { - cased = true - } - } - cased + self.value.py_iscase(char::is_uppercase, char::is_lowercase) } #[pymethod] @@ -1208,7 +1190,7 @@ impl TryFromObject for std::ffi::CString { } } -type SplitArgs = pystr::SplitArgs; +type SplitArgs<'a> = pystr::SplitArgs<'a, PyStringRef, str, char>; #[derive(FromArgs)] pub struct FindArgs { @@ -1728,8 +1710,9 @@ impl PyCommonStringContainer for String { } } -impl PyCommonString for str { +impl<'s> PyCommonString<'s, char> for str { type Container = String; + type CharIter = std::str::Chars<'s>; fn to_container(&self) -> Self::Container { self.to_owned() @@ -1739,6 +1722,10 @@ impl PyCommonString for str { self.as_bytes() } + fn chars(&'s self) -> Self::CharIter { + str::chars(self) + } + fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { &self[range] } diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index d1e370d1b..6ba59692f 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -5,23 +5,23 @@ use crate::vm::VirtualMachine; use num_traits::{cast::ToPrimitive, sign::Signed}; #[derive(FromArgs)] -pub struct SplitArgs +pub struct SplitArgs<'a, T, S, E> where T: TryFromObject + PyCommonStringWrapper, - S: ?Sized + PyCommonString, + S: ?Sized + PyCommonString<'a, E>, { #[pyarg(positional_or_keyword, default = "None")] sep: Option, #[pyarg(positional_or_keyword, default = "-1")] maxsplit: isize, - _phantom1: std::marker::PhantomData, + _phantom1: std::marker::PhantomData<&'a S>, _phantom2: std::marker::PhantomData, } -impl SplitArgs +impl<'a, T, S, E> SplitArgs<'a, T, S, E> where T: TryFromObject + PyCommonStringWrapper, - S: ?Sized + PyCommonString, + S: ?Sized + PyCommonString<'a, E>, { pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option, isize)> { let sep = if let Some(s) = self.sep { @@ -134,14 +134,18 @@ where fn push_str(&mut self, s: &S); } -pub trait PyCommonString +pub trait PyCommonString<'s, E> where + Self: 's, Self::Container: PyCommonStringContainer, + Self::CharIter: 's + std::iter::Iterator, { type Container; + type CharIter; fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; + fn chars(&'s self) -> Self::CharIter; fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self; // FIXME: get_chars is expensive for str fn get_chars<'a>(&'a self, range: std::ops::Range) -> &'a Self; @@ -151,7 +155,7 @@ where fn py_split( &self, - args: SplitArgs, + args: SplitArgs<'s, T, Self, E>, vm: &VirtualMachine, split: SP, splitn: SN, @@ -381,4 +385,25 @@ where filled } } + + fn py_iscase(&'s self, is_case: F, is_opposite: G) -> bool + where + F: Fn(char) -> bool, + G: Fn(char) -> bool, + { + // Unified form of CPython functions: + // _Py_bytes_islower + // Py_bytes_isupper + // unicode_islower_impl + // unicode_isupper_impl + let mut cased = false; + for c in self.chars() { + if is_opposite(c) { + return false; + } else if !cased && is_case(c) { + cased = true + } + } + cased + } } From 5054615ac5eb7e72ef006cb70aaeb49515007fd4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2020 05:02:23 +0900 Subject: [PATCH 063/209] simplify str.isascii --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index f4191f547..577e3903d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -782,7 +782,7 @@ impl PyString { #[pymethod] fn isascii(&self) -> bool { - self.value.chars().all(|c| c.is_ascii()) + self.value.is_ascii() } #[pymethod] From 4ab05249b104524d71b9ed2432d67d7e60da73ea Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 03:41:53 +0900 Subject: [PATCH 064/209] pystr py_add --- vm/src/obj/objbyteinner.rs | 6 +----- vm/src/obj/objstr.rs | 9 +++++---- vm/src/obj/pystr.rs | 7 +++++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4aa81874f..4e66c878a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -308,11 +308,7 @@ impl PyByteInner { } pub fn add(&self, other: PyByteInner) -> Vec { - self.elements - .iter() - .chain(other.elements.iter()) - .cloned() - .collect::>() + self.elements.py_add(&other.elements) } pub fn contains( diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 577e3903d..ded9aff50 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -207,12 +207,13 @@ impl PyString { PyString::from(string.as_str()).into_ref_with_type(vm, cls) } } + #[pymethod(name = "__add__")] - fn add(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&rhs, &vm.ctx.types.str_type) { - Ok(format!("{}{}", self.value, borrow_value(&rhs))) + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&other, &vm.ctx.types.str_type) { + Ok(self.value.py_add(borrow_value(&other))) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot add {} and {}", self, other))) } } diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 6ba59692f..7b37dc7e9 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -153,6 +153,13 @@ where fn chars_len(&self) -> usize; fn is_empty(&self) -> bool; + fn py_add(&self, other: &Self) -> Self::Container { + let mut new = Self::Container::with_capacity(self.bytes_len() + other.bytes_len()); + new.push_str(self); + new.push_str(other); + new + } + fn py_split( &self, args: SplitArgs<'s, T, Self, E>, From 0535d7a78d9900e916a7998d36348ac344d00a56 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 04:09:11 +0900 Subject: [PATCH 065/209] unify pystr py_pad --- vm/src/obj/objbyteinner.rs | 21 +++++++++++---------- vm/src/obj/objstr.rs | 17 +++++++++-------- vm/src/obj/pystr.rs | 20 ++++++++++++++++++-- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4e66c878a..8b9d10c44 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1246,7 +1246,12 @@ const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; impl<'s> PyCommonString<'s, u8> for [u8] { type Container = Vec; - type CharIter = std::iter::Map, fn(&u8) -> char>; + type CharIter = bstr::Chars<'s>; + type ElementIter = std::iter::Copied>; + + fn element_bytes_len(_: u8) -> usize { + 1 + } fn to_container(&self) -> Self::Container { self.to_vec() @@ -1257,7 +1262,11 @@ impl<'s> PyCommonString<'s, u8> for [u8] { } fn chars(&'s self) -> Self::CharIter { - self.iter().map(|c| *c as char) + bstr::ByteSlice::chars(self) + } + + fn elements(&'s self) -> Self::ElementIter { + self.iter().copied() } fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { @@ -1325,12 +1334,4 @@ impl<'s> PyCommonString<'s, u8> for [u8] { } splited } - - fn py_pad(&self, left: usize, right: usize, fill: u8) -> Self::Container { - let mut u = Vec::with_capacity(left + self.len() + right); - u.extend(std::iter::repeat(fill).take(left)); - u.extend_from_slice(self); - u.extend(std::iter::repeat(fill).take(right)); - u - } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ded9aff50..5afe219b0 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1714,6 +1714,11 @@ impl PyCommonStringContainer for String { impl<'s> PyCommonString<'s, char> for str { type Container = String; type CharIter = std::str::Chars<'s>; + type ElementIter = std::str::Chars<'s>; + + fn element_bytes_len(c: char) -> usize { + c.len_utf8() + } fn to_container(&self) -> Self::Container { self.to_owned() @@ -1727,6 +1732,10 @@ impl<'s> PyCommonString<'s, char> for str { str::chars(self) } + fn elements(&'s self) -> Self::ElementIter { + str::chars(self) + } + fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self { &self[range] } @@ -1807,12 +1816,4 @@ impl<'s> PyCommonString<'s, char> for str { } splited } - - fn py_pad(&self, left: usize, right: usize, fill: char) -> Self::Container { - let mut u = String::with_capacity((left + right) * fill.len_utf8() + self.len()); - u.extend(std::iter::repeat(fill).take(left)); - u.push_str(self); - u.extend(std::iter::repeat(fill).take(right)); - u - } } diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 7b37dc7e9..68a745e28 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -9,6 +9,7 @@ pub struct SplitArgs<'a, T, S, E> where T: TryFromObject + PyCommonStringWrapper, S: ?Sized + PyCommonString<'a, E>, + E: Copy, { #[pyarg(positional_or_keyword, default = "None")] sep: Option, @@ -22,6 +23,7 @@ impl<'a, T, S, E> SplitArgs<'a, T, S, E> where T: TryFromObject + PyCommonStringWrapper, S: ?Sized + PyCommonString<'a, E>, + E: Copy, { pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option, isize)> { let sep = if let Some(s) = self.sep { @@ -136,16 +138,22 @@ where pub trait PyCommonString<'s, E> where + E: Copy, Self: 's, - Self::Container: PyCommonStringContainer, + Self::Container: PyCommonStringContainer + std::iter::Extend, Self::CharIter: 's + std::iter::Iterator, + Self::ElementIter: 's + std::iter::Iterator, { type Container; type CharIter; + type ElementIter; + + fn element_bytes_len(c: E) -> usize; fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; fn chars(&'s self) -> Self::CharIter; + fn elements(&'s self) -> Self::ElementIter; fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self; // FIXME: get_chars is expensive for str fn get_chars<'a>(&'a self, range: std::ops::Range) -> &'a Self; @@ -273,7 +281,15 @@ where } } - fn py_pad(&self, left: usize, right: usize, fillchar: E) -> Self::Container; + fn py_pad(&self, left: usize, right: usize, fillchar: E) -> Self::Container { + let mut u = Self::Container::with_capacity( + (left + right) * Self::element_bytes_len(fillchar) + self.bytes_len(), + ); + u.extend(std::iter::repeat(fillchar).take(left)); + u.push_str(self); + u.extend(std::iter::repeat(fillchar).take(right)); + u + } fn py_center(&self, width: usize, fillchar: E) -> Self::Container { let marg = width - self.chars_len(); From d8876a1f7a1902ae596acc9868e87ac93961feb4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 04:15:39 +0900 Subject: [PATCH 066/209] Remove pystr::chars_len due to inaccessibility of cache --- vm/src/obj/objbyteinner.rs | 8 ++------ vm/src/obj/objstr.rs | 8 ++------ vm/src/obj/pystr.rs | 14 +++++++------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8b9d10c44..9893156e0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -674,14 +674,14 @@ impl PyByteInner { fn pad( &self, options: ByteInnerPaddingOptions, - pad: fn(&[u8], usize, u8) -> Vec, + pad: fn(&[u8], usize, u8, usize) -> Vec, vm: &VirtualMachine, ) -> PyResult> { let (width, fillchar) = options.get_value("center", vm)?; Ok(if self.len() as isize >= width { Vec::from(&self.elements[..]) } else { - pad(&self.elements, width as usize, fillchar) + pad(&self.elements, width as usize, fillchar, self.len()) }) } @@ -1285,10 +1285,6 @@ impl<'s> PyCommonString<'s, u8> for [u8] { Self::len(self) } - fn chars_len(&self) -> usize { - Self::len(self) - } - fn py_split_whitespace(&self, maxsplit: isize, convert: F) -> Vec where F: Fn(&Self) -> PyObjectRef, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 5afe219b0..5c93fc532 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -930,14 +930,14 @@ impl PyString { &self, width: isize, fillchar: OptionalArg, - pad: fn(&str, usize, char) -> String, + pad: fn(&str, usize, char, usize) -> String, vm: &VirtualMachine, ) -> PyResult { let fillchar = Self::get_fill_char(fillchar, vm)?; Ok(if self.len() as isize >= width { String::from(&self.value) } else { - pad(&self.value, width as usize, fillchar) + pad(&self.value, width as usize, fillchar, self.len()) }) } @@ -1761,10 +1761,6 @@ impl<'s> PyCommonString<'s, char> for str { Self::len(self) } - fn chars_len(&self) -> usize { - self.chars().count() - } - fn py_split_whitespace(&self, maxsplit: isize, convert: F) -> Vec where F: Fn(&Self) -> PyObjectRef, diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 68a745e28..c37ea0b68 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -158,7 +158,7 @@ where // FIXME: get_chars is expensive for str fn get_chars<'a>(&'a self, range: std::ops::Range) -> &'a Self; fn bytes_len(&self) -> usize; - fn chars_len(&self) -> usize; + // fn chars_len(&self) -> usize; // cannot access to cache here fn is_empty(&self) -> bool; fn py_add(&self, other: &Self) -> Self::Container { @@ -291,18 +291,18 @@ where u } - fn py_center(&self, width: usize, fillchar: E) -> Self::Container { - let marg = width - self.chars_len(); + fn py_center(&self, width: usize, fillchar: E, len: usize) -> Self::Container { + let marg = width - len; let left = marg / 2 + (marg & width & 1); self.py_pad(left, marg - left, fillchar) } - fn py_ljust(&self, width: usize, fillchar: E) -> Self::Container { - self.py_pad(0, width - self.chars_len(), fillchar) + fn py_ljust(&self, width: usize, fillchar: E, len: usize) -> Self::Container { + self.py_pad(0, width - len, fillchar) } - fn py_rjust(&self, width: usize, fillchar: E) -> Self::Container { - self.py_pad(width - self.chars_len(), 0, fillchar) + fn py_rjust(&self, width: usize, fillchar: E, len: usize) -> Self::Container { + self.py_pad(width - len, 0, fillchar) } fn py_join<'a>( From 73877bcbbe421fef6d44721775ecad84c44683e3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 05:00:24 +0900 Subject: [PATCH 067/209] refactor small codes --- vm/src/builtins.rs | 7 ++----- vm/src/obj/objstr.rs | 20 ++++++-------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a11db6084..c6a073ebe 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -6,9 +6,6 @@ use crate::vm::VirtualMachine; #[pymodule(name = "builtins")] mod decl { - use std::char; - use std::str; - use num_bigint::Sign; use num_traits::{Signed, ToPrimitive, Zero}; #[cfg(feature = "rustpython-compiler")] @@ -95,7 +92,7 @@ mod decl { #[pyfunction] fn chr(i: u32, vm: &VirtualMachine) -> PyResult { - match char::from_u32(i) { + match std::char::from_u32(i) { Some(value) => Ok(value.to_string()), None => Err(vm.new_value_error("chr() arg not in range(0x110000)".to_owned())), } @@ -124,7 +121,7 @@ mod decl { // TODO: compile::compile should probably get bytes let source = match &args.source { Either::A(string) => string.as_str(), - Either::B(bytes) => str::from_utf8(bytes).unwrap(), + Either::B(bytes) => std::str::from_utf8(bytes).unwrap(), }; let mode_str = args.mode.as_str(); diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 5c93fc532..cdd0579d6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -506,7 +506,7 @@ impl PyString { #[pymethod] fn endswith(&self, args: pystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { - self.value.as_str().py_startsendswith( + self.value.py_startsendswith( args, "endswith", "str", @@ -517,7 +517,7 @@ impl PyString { #[pymethod] fn startswith(&self, args: pystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { - self.value.as_str().py_startsendswith( + self.value.py_startsendswith( args, "startswith", "str", @@ -536,7 +536,6 @@ impl PyString { #[pymethod] fn removeprefix(&self, pref: PyStringRef) -> String { self.value - .as_str() .py_removeprefix(&pref.value, pref.value.len(), |s, p| s.starts_with(p)) .to_owned() } @@ -551,7 +550,6 @@ impl PyString { #[pymethod] fn removesuffix(&self, suff: PyStringRef) -> String { self.value - .as_str() .py_removesuffix(&suff.value, suff.value.len(), |s, p| s.ends_with(p)) .to_owned() } @@ -573,23 +571,17 @@ impl PyString { 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, ]; - if self.value.is_empty() { - false - } else { - self.value + !self.value.is_empty() + && self + .value .chars() .filter(|c| !c.is_digit(10)) .all(|c| valid_unicodes.contains(&(c as u16))) - } } #[pymethod] fn isdecimal(&self) -> bool { - if self.value.is_empty() { - false - } else { - self.value.chars().all(|c| c.is_ascii_digit()) - } + !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_digit()) } #[pymethod(name = "__mod__")] From 4dc39da128ef08f179ca523e7a91eba6fc9a7a5b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 05:01:51 +0900 Subject: [PATCH 068/209] cformat.rs owns related codes --- vm/src/cformat.rs | 237 ++++++++++++++++++++++++++++++++++++- vm/src/obj/objbytearray.rs | 16 +-- vm/src/obj/objbytes.rs | 20 +--- vm/src/obj/objstr.rs | 237 +------------------------------------ 4 files changed, 244 insertions(+), 266 deletions(-) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 8fa85a8ee..b45137de2 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,13 +1,17 @@ /// Implementation of Printf-Style string formatting /// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] + +use crate::format::get_num_digits; +use crate::obj::{objfloat, objint, objstr, objtuple, objtype}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::vm::VirtualMachine; use num_bigint::{BigInt, Sign}; +use num_traits::cast::ToPrimitive; use num_traits::Signed; use std::cmp; use std::fmt; use std::str::FromStr; -use crate::format::get_num_digits; - #[derive(Debug, PartialEq)] pub enum CFormatErrorType { UnmatchedKeyParentheses, @@ -113,7 +117,7 @@ impl CFormatSpec { .collect::() } - pub fn fill_string( + fn fill_string( &self, string: String, fill_char: char, @@ -154,7 +158,7 @@ impl CFormatSpec { self.fill_string(string, ' ', None) } - pub fn format_number(&self, num: &BigInt) -> String { + fn format_number(&self, num: &BigInt) -> String { use CFormatCase::{Lowercase, Uppercase}; use CNumberType::*; let magnitude = num.abs(); @@ -263,6 +267,89 @@ impl CFormatSpec { Ok(self.fill_string(format!("{}{}", sign_string, magnitude_string), ' ', None)) } } + + fn format(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + use CNumberType::*; + + // do the formatting by type + let format_type = &self.format_type; + + match format_type { + CFormatType::String(preconversor) => { + let result = match preconversor { + CFormatPreconversor::Str => vm.call_method(&obj.clone(), "__str__", vec![])?, + CFormatPreconversor::Repr => { + vm.call_method(&obj.clone(), "__repr__", vec![])? + } + CFormatPreconversor::Ascii => { + vm.call_method(&obj.clone(), "__repr__", vec![])? + } + CFormatPreconversor::Bytes => vm.call_method(&obj.clone(), "decode", vec![])?, + }; + Ok(self.format_string(objstr::clone_value(&result))) + } + CFormatType::Number(_) => { + if !objtype::isinstance(&obj, &vm.ctx.types.int_type) { + let required_type_string = match format_type { + CFormatType::Number(Decimal) => "a number", + CFormatType::Number(_) => "an integer", + _ => unreachable!(), + }; + return Err(vm.new_type_error(format!( + "%{} format: {} is required, not {}", + self.format_char, + required_type_string, + obj.lease_class() + ))); + } + Ok(self.format_number(objint::get_value(&obj))) + } + CFormatType::Float(_) => if objtype::isinstance(&obj, &vm.ctx.types.float_type) { + self.format_float(objfloat::get_value(&obj)) + } else if objtype::isinstance(&obj, &vm.ctx.types.int_type) { + self.format_float(objint::get_value(&obj).to_f64().unwrap()) + } else { + let required_type_string = "an floating point or integer"; + return Err(vm.new_type_error(format!( + "%{} format: {} is required, not {}", + self.format_char, + required_type_string, + obj.lease_class() + ))); + } + .map_err(|e| vm.new_not_implemented_error(e)), + CFormatType::Character => { + let char_string = { + if objtype::isinstance(&obj, &vm.ctx.types.int_type) { + // BigInt truncation is fine in this case because only the unicode range is relevant + match objint::get_value(&obj) + .to_u32() + .and_then(std::char::from_u32) + { + Some(value) => Ok(value.to_string()), + None => { + Err(vm + .new_overflow_error("%c arg not in range(0x110000)".to_owned())) + } + } + } else if objtype::isinstance(&obj, &vm.ctx.types.str_type) { + let s = objstr::borrow_value(&obj); + let num_chars = s.chars().count(); + if num_chars != 1 { + Err(vm.new_type_error("%c requires int or char".to_owned())) + } else { + Ok(s.chars().next().unwrap().to_string()) + } + } else { + // TODO re-arrange this block so this error is only created once + Err(vm.new_type_error("%c requires int or char".to_owned())) + } + }?; + self.precision = Some(CFormatQuantity::Amount(1)); + Ok(self.format_string(char_string)) + } + } + } } #[derive(Debug, PartialEq)] @@ -319,6 +406,146 @@ impl FromStr for CFormatString { } } +impl CFormatString { + pub fn format(&mut self, vm: &VirtualMachine, values_obj: PyObjectRef) -> PyResult { + fn call_getitem( + vm: &VirtualMachine, + container: &PyObjectRef, + key: &PyObjectRef, + ) -> PyResult { + vm.call_method(container, "__getitem__", vec![key.clone()]) + } + + fn try_update_quantity_from_tuple( + vm: &VirtualMachine, + elements: &mut dyn Iterator, + q: &mut Option, + mut tuple_index: usize, + ) -> PyResult { + match q { + Some(CFormatQuantity::FromValuesTuple) => { + match elements.next() { + Some(width_obj) => { + tuple_index += 1; + if !objtype::isinstance(&width_obj, &vm.ctx.types.int_type) { + Err(vm.new_type_error("* wants int".to_owned())) + } else { + // TODO: handle errors when truncating BigInt to usize + *q = Some(CFormatQuantity::Amount( + objint::get_value(&width_obj).to_usize().unwrap(), + )); + Ok(tuple_index) + } + } + None => Err( + vm.new_type_error("not enough arguments for format string".to_owned()) + ), + } + } + _ => Ok(tuple_index), + } + } + + let mut final_string = String::new(); + let num_specifiers = self + .format_parts + .iter() + .filter(|(_, part)| CFormatPart::is_specifier(part)) + .count(); + let mapping_required = self + .format_parts + .iter() + .any(|(_, part)| CFormatPart::has_key(part)) + && self + .format_parts + .iter() + .filter(|(_, part)| CFormatPart::is_specifier(part)) + .all(|(_, part)| CFormatPart::has_key(part)); + + let values = if mapping_required { + if !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) { + return Err(vm.new_type_error("format requires a mapping".to_owned())); + } + values_obj.clone() + } else { + // check for only literal parts, in which case only dict or empty tuple is allowed + if num_specifiers == 0 + && !(objtype::isinstance(&values_obj, &vm.ctx.types.tuple_type) + && objtuple::get_value(&values_obj).is_empty()) + && !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) + { + return Err(vm.new_type_error( + "not all arguments converted during string formatting".to_owned(), + )); + } + + // convert `values_obj` to a new tuple if it's not a tuple + if !objtype::isinstance(&values_obj, &vm.ctx.types.tuple_type) { + vm.ctx.new_tuple(vec![values_obj.clone()]) + } else { + values_obj.clone() + } + }; + + let mut tuple_index: usize = 0; + for (_, part) in &mut self.format_parts { + let result_string: String = match part { + CFormatPart::Spec(format_spec) => { + // try to get the object + let obj: PyObjectRef = match &format_spec.mapping_key { + Some(key) => { + // TODO: change the KeyError message to match the one in cpython + call_getitem(vm, &values, &vm.ctx.new_str(key.to_owned()))? + } + None => { + let mut elements = objtuple::get_value(&values) + .to_vec() + .into_iter() + .skip(tuple_index); + + tuple_index = try_update_quantity_from_tuple( + vm, + &mut elements, + &mut format_spec.min_field_width, + tuple_index, + )?; + tuple_index = try_update_quantity_from_tuple( + vm, + &mut elements, + &mut format_spec.precision, + tuple_index, + )?; + + let obj = match elements.next() { + Some(obj) => Ok(obj), + None => Err(vm.new_type_error( + "not enough arguments for format string".to_owned(), + )), + }?; + tuple_index += 1; + + obj + } + }; + format_spec.format(vm, obj) + } + CFormatPart::Literal(literal) => Ok(literal.clone()), + }?; + final_string.push_str(&result_string); + } + + // check that all arguments were converted + if (!mapping_required && objtuple::get_value(&values).get(tuple_index).is_some()) + && !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) + { + return Err(vm.new_type_error( + "not all arguments converted during string formatting".to_owned(), + )); + } + Ok(final_string) + } +} + fn parse_quantity(text: &str) -> (Option, &str) { let num_digits: usize = get_num_digits(text); if num_digits == 0 { @@ -863,7 +1090,7 @@ mod tests { fn test_format_parse() { let fmt = "Hello, my name is %s and I'm %d years old"; let expected = Ok(CFormatString { - format_parts: vec![ + parts: vec![ (0, CFormatPart::Literal("Hello, my name is ".to_owned())), ( 18, diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index aea9e6372..9ca3a775e 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -18,7 +18,6 @@ use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; use crate::cformat::CFormatString; use crate::function::{OptionalArg, OptionalOption}; -use crate::obj::objstr::do_cformat_string; use crate::pyobject::{ Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -556,22 +555,13 @@ impl PyByteArray { self.borrow_value_mut().irepeat(n) } - fn do_cformat( - &self, - vm: &VirtualMachine, - format_string: CFormatString, - values_obj: PyObjectRef, - ) -> PyResult { - let final_string = do_cformat_string(vm, format_string, values_obj)?; - Ok(final_string.as_str().as_bytes().to_owned().into()) - } - #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let format_string = + let mut format_string = CFormatString::from_str(std::str::from_utf8(&self.borrow_value().elements).unwrap()) .map_err(|err| vm.new_value_error(err.to_string()))?; - self.do_cformat(vm, format_string, values.clone()) + let final_string = format_string.format(vm, values)?; + Ok(final_string.as_str().as_bytes().to_owned().into()) } #[pymethod(name = "__rmod__")] diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index e5ed355ff..f2659a5ca 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -16,7 +16,6 @@ use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; use crate::cformat::CFormatString; use crate::function::{OptionalArg, OptionalOption}; -use crate::obj::objstr::do_cformat_string; use crate::pyhash; use crate::pyobject::{ Either, IntoPyObject, @@ -454,24 +453,15 @@ impl PyBytes { Ok(self.inner.repeat(value).into()) } - fn do_cformat( - &self, - vm: &VirtualMachine, - format_string: CFormatString, - values_obj: PyObjectRef, - ) -> PyResult { - let final_string = do_cformat_string(vm, format_string, values_obj)?; - Ok(vm - .ctx - .new_bytes(final_string.as_str().as_bytes().to_owned())) - } - #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { let format_string_text = std::str::from_utf8(&self.inner.elements).unwrap(); - let format_string = CFormatString::from_str(format_string_text) + let mut format_string = CFormatString::from_str(format_string_text) .map_err(|err| vm.new_value_error(err.to_string()))?; - self.do_cformat(vm, format_string, values.clone()) + let final_string = format_string.format(vm, values)?; + Ok(vm + .ctx + .new_bytes(final_string.as_str().as_bytes().to_owned())) } #[pymethod(name = "__rmod__")] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index cdd0579d6..35a3e3d32 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -15,20 +15,15 @@ use unicode_casing::CharExt; use super::objbytes::{PyBytes, PyBytesRef}; use super::objdict::PyDict; -use super::objfloat; -use super::objint::{self, PyInt, PyIntRef}; +use super::objint::{PyInt, PyIntRef}; use super::objiter; use super::objnone::PyNone; use super::objsequence::{PySliceableSequence, SequenceIndex}; -use super::objtuple; use super::objtype::{self, PyClassRef}; use super::pystr::{ self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, }; -use crate::cformat::{ - CFormatPart, CFormatPreconversor, CFormatQuantity, CFormatSpec, CFormatString, CFormatType, - CNumberType, -}; +use crate::cformat::CFormatString; use crate::format::{ FormatParseError, FormatPart, FormatPreconversor, FormatSpec, FormatString, FromTemplate, }; @@ -587,9 +582,9 @@ impl PyString { #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { let format_string_text = &self.value; - let format_string = CFormatString::from_str(format_string_text) + let mut format_string = CFormatString::from_str(format_string_text) .map_err(|err| vm.new_value_error(err.to_string()))?; - do_cformat(vm, format_string, values.clone()) + Ok(vm.ctx.new_str(format_string.format(vm, values)?)) } #[pymethod(name = "__rmod__")] @@ -1217,10 +1212,6 @@ pub fn borrow_value(obj: &PyObjectRef) -> &str { &obj.payload::().unwrap().value } -fn call_getitem(vm: &VirtualMachine, container: &PyObjectRef, key: &PyObjectRef) -> PyResult { - vm.call_method(container, "__getitem__", vec![key.clone()]) -} - fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { let (preconversor, new_format_spec) = FormatPreconversor::parse_and_consume(format_spec); let argument = match preconversor { @@ -1241,226 +1232,6 @@ fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: & Ok(result) } -fn do_cformat_specifier( - vm: &VirtualMachine, - format_spec: &mut CFormatSpec, - obj: PyObjectRef, -) -> PyResult { - use CNumberType::*; - // do the formatting by type - let format_type = &format_spec.format_type; - - match format_type { - CFormatType::String(preconversor) => { - let result = match preconversor { - CFormatPreconversor::Str => vm.call_method(&obj.clone(), "__str__", vec![])?, - CFormatPreconversor::Repr => vm.call_method(&obj.clone(), "__repr__", vec![])?, - CFormatPreconversor::Ascii => vm.call_method(&obj.clone(), "__repr__", vec![])?, - CFormatPreconversor::Bytes => vm.call_method(&obj.clone(), "decode", vec![])?, - }; - Ok(format_spec.format_string(clone_value(&result))) - } - CFormatType::Number(_) => { - if !objtype::isinstance(&obj, &vm.ctx.types.int_type) { - let required_type_string = match format_type { - CFormatType::Number(Decimal) => "a number", - CFormatType::Number(_) => "an integer", - _ => unreachable!(), - }; - return Err(vm.new_type_error(format!( - "%{} format: {} is required, not {}", - format_spec.format_char, - required_type_string, - obj.lease_class() - ))); - } - Ok(format_spec.format_number(objint::get_value(&obj))) - } - CFormatType::Float(_) => if objtype::isinstance(&obj, &vm.ctx.types.float_type) { - format_spec.format_float(objfloat::get_value(&obj)) - } else if objtype::isinstance(&obj, &vm.ctx.types.int_type) { - format_spec.format_float(objint::get_value(&obj).to_f64().unwrap()) - } else { - let required_type_string = "an floating point or integer"; - return Err(vm.new_type_error(format!( - "%{} format: {} is required, not {}", - format_spec.format_char, - required_type_string, - obj.lease_class() - ))); - } - .map_err(|e| vm.new_not_implemented_error(e)), - CFormatType::Character => { - let char_string = { - if objtype::isinstance(&obj, &vm.ctx.types.int_type) { - // BigInt truncation is fine in this case because only the unicode range is relevant - match objint::get_value(&obj).to_u32().and_then(char::from_u32) { - Some(value) => Ok(value.to_string()), - None => { - Err(vm.new_overflow_error("%c arg not in range(0x110000)".to_owned())) - } - } - } else if objtype::isinstance(&obj, &vm.ctx.types.str_type) { - let s = borrow_value(&obj); - let num_chars = s.chars().count(); - if num_chars != 1 { - Err(vm.new_type_error("%c requires int or char".to_owned())) - } else { - Ok(s.chars().next().unwrap().to_string()) - } - } else { - // TODO re-arrange this block so this error is only created once - Err(vm.new_type_error("%c requires int or char".to_owned())) - } - }?; - format_spec.precision = Some(CFormatQuantity::Amount(1)); - Ok(format_spec.format_string(char_string)) - } - } -} - -fn try_update_quantity_from_tuple( - vm: &VirtualMachine, - elements: &mut dyn Iterator, - q: &mut Option, - mut tuple_index: usize, -) -> PyResult { - match q { - Some(CFormatQuantity::FromValuesTuple) => { - match elements.next() { - Some(width_obj) => { - tuple_index += 1; - if !objtype::isinstance(&width_obj, &vm.ctx.types.int_type) { - Err(vm.new_type_error("* wants int".to_owned())) - } else { - // TODO: handle errors when truncating BigInt to usize - *q = Some(CFormatQuantity::Amount( - objint::get_value(&width_obj).to_usize().unwrap(), - )); - Ok(tuple_index) - } - } - None => Err(vm.new_type_error("not enough arguments for format string".to_owned())), - } - } - _ => Ok(tuple_index), - } -} - -pub fn do_cformat_string( - vm: &VirtualMachine, - mut format_string: CFormatString, - values_obj: PyObjectRef, -) -> PyResult { - let mut final_string = String::new(); - let num_specifiers = format_string - .format_parts - .iter() - .filter(|(_, part)| CFormatPart::is_specifier(part)) - .count(); - let mapping_required = format_string - .format_parts - .iter() - .any(|(_, part)| CFormatPart::has_key(part)) - && format_string - .format_parts - .iter() - .filter(|(_, part)| CFormatPart::is_specifier(part)) - .all(|(_, part)| CFormatPart::has_key(part)); - - let values = if mapping_required { - if !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) { - return Err(vm.new_type_error("format requires a mapping".to_owned())); - } - values_obj.clone() - } else { - // check for only literal parts, in which case only dict or empty tuple is allowed - if num_specifiers == 0 - && !(objtype::isinstance(&values_obj, &vm.ctx.types.tuple_type) - && objtuple::get_value(&values_obj).is_empty()) - && !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) - { - return Err(vm.new_type_error( - "not all arguments converted during string formatting".to_owned(), - )); - } - - // convert `values_obj` to a new tuple if it's not a tuple - if !objtype::isinstance(&values_obj, &vm.ctx.types.tuple_type) { - vm.ctx.new_tuple(vec![values_obj.clone()]) - } else { - values_obj.clone() - } - }; - - let mut tuple_index: usize = 0; - for (_, part) in &mut format_string.format_parts { - let result_string: String = match part { - CFormatPart::Spec(format_spec) => { - // try to get the object - let obj: PyObjectRef = match &format_spec.mapping_key { - Some(key) => { - // TODO: change the KeyError message to match the one in cpython - call_getitem(vm, &values, &vm.ctx.new_str(key.to_owned()))? - } - None => { - let mut elements = objtuple::get_value(&values) - .to_vec() - .into_iter() - .skip(tuple_index); - - tuple_index = try_update_quantity_from_tuple( - vm, - &mut elements, - &mut format_spec.min_field_width, - tuple_index, - )?; - tuple_index = try_update_quantity_from_tuple( - vm, - &mut elements, - &mut format_spec.precision, - tuple_index, - )?; - - let obj = match elements.next() { - Some(obj) => Ok(obj), - None => Err(vm.new_type_error( - "not enough arguments for format string".to_owned(), - )), - }?; - tuple_index += 1; - - obj - } - }; - do_cformat_specifier(vm, format_spec, obj) - } - CFormatPart::Literal(literal) => Ok(literal.clone()), - }?; - final_string.push_str(&result_string); - } - - // check that all arguments were converted - if (!mapping_required && objtuple::get_value(&values).get(tuple_index).is_some()) - && !objtype::isinstance(&values_obj, &vm.ctx.types.dict_type) - { - return Err( - vm.new_type_error("not all arguments converted during string formatting".to_owned()) - ); - } - Ok(final_string) -} - -fn do_cformat( - vm: &VirtualMachine, - format_string: CFormatString, - values_obj: PyObjectRef, -) -> PyResult { - Ok(vm - .ctx - .new_str(do_cformat_string(vm, format_string, values_obj)?)) -} - fn perform_format( vm: &VirtualMachine, format_string: &FormatString, From 20812c61e998f411859e377796731e51bb9f4449 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 05:25:47 +0900 Subject: [PATCH 069/209] Drop unused `pub` from cformat --- vm/src/cformat.rs | 89 +++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index b45137de2..bb2431d72 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,9 +1,8 @@ /// Implementation of Printf-Style string formatting /// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] - use crate::format::get_num_digits; use crate::obj::{objfloat, objint, objstr, objtuple, objtype}; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use num_bigint::{BigInt, Sign}; use num_traits::cast::ToPrimitive; @@ -13,22 +12,22 @@ use std::fmt; use std::str::FromStr; #[derive(Debug, PartialEq)] -pub enum CFormatErrorType { +enum CFormatErrorType { UnmatchedKeyParentheses, MissingModuloSign, UnescapedModuloSignInLiteral, UnsupportedFormatChar(char), IncompleteFormat, - Unimplemented, + // Unimplemented, } // also contains how many chars the parsing function consumed type ParsingError = (CFormatErrorType, usize); #[derive(Debug, PartialEq)] -pub struct CFormatError { - pub typ: CFormatErrorType, - pub index: usize, +pub(crate) struct CFormatError { + typ: CFormatErrorType, + index: usize, } impl fmt::Display for CFormatError { @@ -48,7 +47,7 @@ impl fmt::Display for CFormatError { } #[derive(Debug, PartialEq)] -pub enum CFormatPreconversor { +enum CFormatPreconversor { Repr, Str, Ascii, @@ -56,27 +55,27 @@ pub enum CFormatPreconversor { } #[derive(Debug, PartialEq)] -pub enum CFormatCase { +enum CFormatCase { Lowercase, Uppercase, } #[derive(Debug, PartialEq)] -pub enum CNumberType { +enum CNumberType { Decimal, Octal, Hex(CFormatCase), } #[derive(Debug, PartialEq)] -pub enum CFloatType { +enum CFloatType { Exponent(CFormatCase), PointDecimal, General(CFormatCase), } #[derive(Debug, PartialEq)] -pub enum CFormatType { +enum CFormatType { Number(CNumberType), Float(CFloatType), Character, @@ -84,7 +83,7 @@ pub enum CFormatType { } bitflags! { - pub struct CConversionFlags: u32 { + struct CConversionFlags: u32 { const ALTERNATE_FORM = 0b0000_0001; const ZERO_PAD = 0b0000_0010; const LEFT_ADJUST = 0b0000_0100; @@ -94,19 +93,19 @@ bitflags! { } #[derive(Debug, PartialEq)] -pub enum CFormatQuantity { +enum CFormatQuantity { Amount(usize), FromValuesTuple, } #[derive(Debug, PartialEq)] -pub struct CFormatSpec { - pub mapping_key: Option, - pub flags: CConversionFlags, - pub min_field_width: Option, - pub precision: Option, - pub format_type: CFormatType, - pub format_char: char, +struct CFormatSpec { + mapping_key: Option, + flags: CConversionFlags, + min_field_width: Option, + precision: Option, + format_type: CFormatType, + format_char: char, chars_consumed: usize, } @@ -147,7 +146,7 @@ impl CFormatSpec { } } - pub fn format_string(&self, string: String) -> String { + pub(crate) fn format_string(&self, string: String) -> String { let mut string = string; // truncate if needed if let Some(CFormatQuantity::Amount(precision)) = self.precision { @@ -216,7 +215,7 @@ impl CFormatSpec { } } - pub fn format_float(&self, num: f64) -> Result { + pub(crate) fn format_float(&self, num: f64) -> Result { let magnitude = num.abs(); let sign_string = if num.is_sign_positive() { @@ -277,16 +276,14 @@ impl CFormatSpec { match format_type { CFormatType::String(preconversor) => { let result = match preconversor { - CFormatPreconversor::Str => vm.call_method(&obj.clone(), "__str__", vec![])?, - CFormatPreconversor::Repr => { - vm.call_method(&obj.clone(), "__repr__", vec![])? - } - CFormatPreconversor::Ascii => { - vm.call_method(&obj.clone(), "__repr__", vec![])? - } - CFormatPreconversor::Bytes => vm.call_method(&obj.clone(), "decode", vec![])?, + CFormatPreconversor::Str => vm.to_str(&obj)?, + CFormatPreconversor::Repr | CFormatPreconversor::Ascii => vm.to_repr(&obj)?, + CFormatPreconversor::Bytes => TryFromObject::try_from_object( + vm, + vm.call_method(&obj.clone(), "decode", vec![])?, + )?, }; - Ok(self.format_string(objstr::clone_value(&result))) + Ok(self.format_string(result.as_str().to_owned())) } CFormatType::Number(_) => { if !objtype::isinstance(&obj, &vm.ctx.types.int_type) { @@ -353,20 +350,20 @@ impl CFormatSpec { } #[derive(Debug, PartialEq)] -pub enum CFormatPart { +enum CFormatPart { Literal(String), Spec(CFormatSpec), } impl CFormatPart { - pub fn is_specifier(&self) -> bool { + fn is_specifier(&self) -> bool { match self { CFormatPart::Spec(_) => true, _ => false, } } - pub fn has_key(&self) -> bool { + fn has_key(&self) -> bool { match self { CFormatPart::Spec(s) => s.mapping_key.is_some(), _ => false, @@ -375,8 +372,8 @@ impl CFormatPart { } #[derive(Debug, PartialEq)] -pub struct CFormatString { - pub format_parts: Vec<(usize, CFormatPart)>, +pub(crate) struct CFormatString { + parts: Vec<(usize, CFormatPart)>, } impl FromStr for CFormatString { @@ -400,14 +397,16 @@ impl FromStr for CFormatString { })?; } - Ok(CFormatString { - format_parts: parts, - }) + Ok(CFormatString { parts }) } } impl CFormatString { - pub fn format(&mut self, vm: &VirtualMachine, values_obj: PyObjectRef) -> PyResult { + pub(crate) fn format( + &mut self, + vm: &VirtualMachine, + values_obj: PyObjectRef, + ) -> PyResult { fn call_getitem( vm: &VirtualMachine, container: &PyObjectRef, @@ -448,16 +447,16 @@ impl CFormatString { let mut final_string = String::new(); let num_specifiers = self - .format_parts + .parts .iter() .filter(|(_, part)| CFormatPart::is_specifier(part)) .count(); let mapping_required = self - .format_parts + .parts .iter() .any(|(_, part)| CFormatPart::has_key(part)) && self - .format_parts + .parts .iter() .filter(|(_, part)| CFormatPart::is_specifier(part)) .all(|(_, part)| CFormatPart::has_key(part)); @@ -488,7 +487,7 @@ impl CFormatString { }; let mut tuple_index: usize = 0; - for (_, part) in &mut self.format_parts { + for (_, part) in &mut self.parts { let result_string: String = match part { CFormatPart::Spec(format_spec) => { // try to get the object From d09488f159a094b0939da0fc13be993f3771560f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 05:39:02 +0900 Subject: [PATCH 070/209] Use ItemProtocol over vm.call_method("__getitem__") --- vm/src/cformat.rs | 12 ++---------- vm/src/obj/objmemory.rs | 4 ++-- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index bb2431d72..2675507e3 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -2,7 +2,7 @@ /// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] use crate::format::get_num_digits; use crate::obj::{objfloat, objint, objstr, objtuple, objtype}; -use crate::pyobject::{PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use num_bigint::{BigInt, Sign}; use num_traits::cast::ToPrimitive; @@ -407,14 +407,6 @@ impl CFormatString { vm: &VirtualMachine, values_obj: PyObjectRef, ) -> PyResult { - fn call_getitem( - vm: &VirtualMachine, - container: &PyObjectRef, - key: &PyObjectRef, - ) -> PyResult { - vm.call_method(container, "__getitem__", vec![key.clone()]) - } - fn try_update_quantity_from_tuple( vm: &VirtualMachine, elements: &mut dyn Iterator, @@ -494,7 +486,7 @@ impl CFormatString { let obj: PyObjectRef = match &format_spec.mapping_key { Some(key) => { // TODO: change the KeyError message to match the one in cpython - call_getitem(vm, &values, &vm.ctx.new_str(key.to_owned()))? + values.get_item(key, vm)? } None => { let mut elements = objtuple::get_value(&values) diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 12f3c2637..afdfbb131 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,7 +1,7 @@ use super::objbyteinner::try_as_byte; use super::objtype::{issubclass, PyClassRef}; use crate::pyobject::{ - PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::stdlib::array::PyArray; use crate::vm::VirtualMachine; @@ -57,7 +57,7 @@ impl PyMemoryView { #[pymethod(name = "__getitem__")] fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_method(&self.obj_ref, "__getitem__", vec![needle]) + self.obj_ref.get_item(&needle, vm) } #[pymethod(magic)] From a0a73e5d978a547ffbe8ec2e23bfaa4aab3f953d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 06:00:25 +0900 Subject: [PATCH 071/209] format.rs owns related code --- vm/src/format.rs | 134 ++++++++++++++++++++++++++++++++++++------- vm/src/obj/objstr.rs | 104 +-------------------------------- 2 files changed, 116 insertions(+), 122 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 86f7b08db..d468f7ae6 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,6 +1,9 @@ use crate::function::PyFuncArgs; use crate::obj::objint::PyInt; use crate::obj::objstr::PyString; +use crate::obj::{objstr, objtype}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; +use crate::vm::VirtualMachine; use num_bigint::{BigInt, Sign}; use num_traits::cast::ToPrimitive; use num_traits::Signed; @@ -8,7 +11,7 @@ use std::cmp; use std::str::FromStr; #[derive(Debug, Copy, Clone, PartialEq)] -pub enum FormatPreconversor { +enum FormatPreconversor { Str, Repr, Ascii, @@ -16,7 +19,7 @@ pub enum FormatPreconversor { } impl FormatPreconversor { - pub fn from_char(c: char) -> Option { + fn from_char(c: char) -> Option { match c { 's' => Some(FormatPreconversor::Str), 'r' => Some(FormatPreconversor::Repr), @@ -26,7 +29,7 @@ impl FormatPreconversor { } } - pub fn from_string(text: &str) -> Option { + fn from_string(text: &str) -> Option { let mut chars = text.chars(); if chars.next() != Some('!') { return None; @@ -38,7 +41,7 @@ impl FormatPreconversor { } } - pub fn parse_and_consume(text: &str) -> (Option, &str) { + fn parse_and_consume(text: &str) -> (Option, &str) { let preconversor = FormatPreconversor::from_string(text); match preconversor { None => (None, text), @@ -53,7 +56,7 @@ impl FormatPreconversor { } #[derive(Debug, Copy, Clone, PartialEq)] -pub enum FormatAlign { +enum FormatAlign { Left, Right, AfterSign, @@ -73,20 +76,20 @@ impl FormatAlign { } #[derive(Debug, Copy, Clone, PartialEq)] -pub enum FormatSign { +enum FormatSign { Plus, Minus, MinusOrSpace, } #[derive(Debug, PartialEq)] -pub enum FormatGrouping { +enum FormatGrouping { Comma, Underscore, } #[derive(Debug, PartialEq)] -pub enum FormatType { +enum FormatType { String, Binary, Character, @@ -105,7 +108,7 @@ pub enum FormatType { } #[derive(Debug, PartialEq)] -pub struct FormatSpec { +pub(crate) struct FormatSpec { preconversor: Option, fill: Option, align: Option, @@ -117,7 +120,7 @@ pub struct FormatSpec { format_type: Option, } -pub fn get_num_digits(text: &str) -> usize { +pub(crate) fn get_num_digits(text: &str) -> usize { for (index, character) in text.char_indices() { if !character.is_digit(10) { return index; @@ -286,7 +289,7 @@ fn format_float_as_exponent(precision: usize, magnitude: f64, separator: &str) - } impl FormatSpec { - pub fn parse(text: &str) -> Result { + pub(crate) fn parse(text: &str) -> Result { parse_format_spec(text) } @@ -351,7 +354,7 @@ impl FormatSpec { } } - pub fn format_float(&self, num: f64) -> Result { + pub(crate) fn format_float(&self, num: f64) -> Result { let precision = self.precision.unwrap_or(6); let magnitude = num.abs(); let raw_magnitude_string_result: Result = match self.format_type { @@ -428,7 +431,7 @@ impl FormatSpec { self.format_sign_and_align(&magnitude_string, sign_str) } - pub fn format_int(&self, num: &BigInt) -> Result { + pub(crate) fn format_int(&self, num: &BigInt) -> Result { let magnitude = num.abs(); let prefix = if self.alternate_form { match self.format_type { @@ -492,7 +495,7 @@ impl FormatSpec { self.format_sign_and_align(&magnitude_string, sign_str) } - pub fn format_string(&self, s: &str) -> Result { + pub(crate) fn format_string(&self, s: &str) -> Result { match self.format_type { Some(FormatType::String) | None => self.format_sign_and_align(s, ""), _ => Err("Unknown format code for object of type 'str'"), @@ -548,7 +551,7 @@ impl FormatSpec { } #[derive(Debug, PartialEq)] -pub enum FormatParseError { +pub(crate) enum FormatParseError { UnmatchedBracket, MissingStartBracket, UnescapedStartBracketInLiteral, @@ -563,7 +566,7 @@ impl FromStr for FormatSpec { } #[derive(Debug, PartialEq)] -pub enum FormatPart { +enum FormatPart { AutoSpec(String), IndexSpec(usize, String), KeywordSpec(String, String), @@ -571,14 +574,14 @@ pub enum FormatPart { } impl FormatPart { - pub fn is_auto(&self) -> bool { + fn is_auto(&self) -> bool { match self { FormatPart::AutoSpec(_) => true, _ => false, } } - pub fn is_index(&self) -> bool { + fn is_index(&self) -> bool { match self { FormatPart::IndexSpec(_, _) => true, _ => false, @@ -587,8 +590,8 @@ impl FormatPart { } #[derive(Debug, PartialEq)] -pub struct FormatString { - pub format_parts: Vec, +pub(crate) struct FormatString { + format_parts: Vec, } impl FormatString { @@ -719,9 +722,98 @@ impl FormatString { Err(FormatParseError::UnmatchedBracket) } } + + pub(crate) fn format(&self, arguments: &PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let mut final_string = String::new(); + if self.format_parts.iter().any(FormatPart::is_auto) + && self.format_parts.iter().any(FormatPart::is_index) + { + return Err(vm.new_value_error( + "cannot switch from automatic field numbering to manual field specification" + .to_owned(), + )); + } + let mut auto_argument_index: usize = 1; + for part in &self.format_parts { + let result_string: String = match part { + FormatPart::AutoSpec(format_spec) => { + let result = match arguments.args.get(auto_argument_index) { + Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, + None => { + return Err(vm.new_index_error("tuple index out of range".to_owned())); + } + }; + auto_argument_index += 1; + objstr::clone_value(&result) + } + FormatPart::IndexSpec(index, format_spec) => { + let result = match arguments.args.get(*index + 1) { + Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, + None => { + return Err(vm.new_index_error("tuple index out of range".to_owned())); + } + }; + objstr::clone_value(&result) + } + FormatPart::KeywordSpec(keyword, format_spec) => { + let result = match arguments.get_optional_kwarg(&keyword) { + Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, + None => { + return Err(vm.new_key_error(vm.new_str(keyword.to_owned()))); + } + }; + objstr::clone_value(&result) + } + FormatPart::Literal(literal) => literal.clone(), + }; + final_string.push_str(&result_string); + } + Ok(vm.ctx.new_str(final_string)) + } + + pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut final_string = String::new(); + for part in &self.format_parts { + let result_string: String = match part { + FormatPart::AutoSpec(_) | FormatPart::IndexSpec(_, _) => { + return Err( + vm.new_value_error("Format string contains positional fields".to_owned()) + ); + } + FormatPart::KeywordSpec(keyword, format_spec) => { + let argument = dict.get_item(keyword, &vm)?; + let result = call_object_format(vm, argument.clone(), &format_spec)?; + objstr::clone_value(&result) + } + FormatPart::Literal(literal) => literal.clone(), + }; + final_string.push_str(&result_string); + } + Ok(vm.ctx.new_str(final_string)) + } } -pub trait FromTemplate<'a>: Sized { +fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { + let (preconversor, new_format_spec) = FormatPreconversor::parse_and_consume(format_spec); + let argument = match preconversor { + Some(FormatPreconversor::Str) => vm.call_method(&argument, "__str__", vec![])?, + Some(FormatPreconversor::Repr) => vm.call_method(&argument, "__repr__", vec![])?, + Some(FormatPreconversor::Ascii) => vm.call_method(&argument, "__repr__", vec![])?, + Some(FormatPreconversor::Bytes) => vm.call_method(&argument, "decode", vec![])?, + None => argument, + }; + let returned_type = vm.ctx.new_str(new_format_spec.to_owned()); + + let result = vm.call_method(&argument, "__format__", vec![returned_type])?; + if !objtype::isinstance(&result, &vm.ctx.types.str_type) { + let result_type = result.class(); + let actual_type = vm.to_pystr(&result_type)?; + return Err(vm.new_type_error(format!("__format__ must return a str, not {}", actual_type))); + } + Ok(result) +} + +pub(crate) trait FromTemplate<'a>: Sized { type Err; fn from_str(s: &'a str, arg: &'a PyFuncArgs) -> Result; } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 35a3e3d32..b11220f4c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -24,9 +24,7 @@ use super::pystr::{ self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, }; use crate::cformat::CFormatString; -use crate::format::{ - FormatParseError, FormatPart, FormatPreconversor, FormatSpec, FormatString, FromTemplate, -}; +use crate::format::{FormatParseError, FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyhash; use crate::pyobject::{ @@ -611,7 +609,7 @@ impl PyString { } let format_string_text = borrow_value(zelf); match FormatString::from_str(format_string_text, &args) { - Ok(format_string) => perform_format(vm, &format_string, &args), + Ok(format_string) => format_string.format(&args, vm), Err(err) => match err { FormatParseError::UnmatchedBracket => { Err(vm.new_value_error("expected '}' before end of string".to_owned())) @@ -637,7 +635,7 @@ impl PyString { let zelf = &args.args[0]; let format_string_text = borrow_value(zelf); match FormatString::from_str(format_string_text, &args) { - Ok(format_string) => perform_format_map(vm, &format_string, &args.args[1]), + Ok(format_string) => format_string.format_map(&args.args[1], vm), Err(err) => match err { FormatParseError::UnmatchedBracket => { Err(vm.new_value_error("expected '}' before end of string".to_owned())) @@ -1212,102 +1210,6 @@ pub fn borrow_value(obj: &PyObjectRef) -> &str { &obj.payload::().unwrap().value } -fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { - let (preconversor, new_format_spec) = FormatPreconversor::parse_and_consume(format_spec); - let argument = match preconversor { - Some(FormatPreconversor::Str) => vm.call_method(&argument, "__str__", vec![])?, - Some(FormatPreconversor::Repr) => vm.call_method(&argument, "__repr__", vec![])?, - Some(FormatPreconversor::Ascii) => vm.call_method(&argument, "__repr__", vec![])?, - Some(FormatPreconversor::Bytes) => vm.call_method(&argument, "decode", vec![])?, - None => argument, - }; - let returned_type = vm.ctx.new_str(new_format_spec.to_owned()); - - let result = vm.call_method(&argument, "__format__", vec![returned_type])?; - if !objtype::isinstance(&result, &vm.ctx.types.str_type) { - let result_type = result.class(); - let actual_type = vm.to_pystr(&result_type)?; - return Err(vm.new_type_error(format!("__format__ must return a str, not {}", actual_type))); - } - Ok(result) -} - -fn perform_format( - vm: &VirtualMachine, - format_string: &FormatString, - arguments: &PyFuncArgs, -) -> PyResult { - let mut final_string = String::new(); - if format_string.format_parts.iter().any(FormatPart::is_auto) - && format_string.format_parts.iter().any(FormatPart::is_index) - { - return Err(vm.new_value_error( - "cannot switch from automatic field numbering to manual field specification".to_owned(), - )); - } - let mut auto_argument_index: usize = 1; - for part in &format_string.format_parts { - let result_string: String = match part { - FormatPart::AutoSpec(format_spec) => { - let result = match arguments.args.get(auto_argument_index) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_index_error("tuple index out of range".to_owned())); - } - }; - auto_argument_index += 1; - clone_value(&result) - } - FormatPart::IndexSpec(index, format_spec) => { - let result = match arguments.args.get(*index + 1) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_index_error("tuple index out of range".to_owned())); - } - }; - clone_value(&result) - } - FormatPart::KeywordSpec(keyword, format_spec) => { - let result = match arguments.get_optional_kwarg(&keyword) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_key_error(vm.new_str(keyword.to_owned()))); - } - }; - clone_value(&result) - } - FormatPart::Literal(literal) => literal.clone(), - }; - final_string.push_str(&result_string); - } - Ok(vm.ctx.new_str(final_string)) -} - -fn perform_format_map( - vm: &VirtualMachine, - format_string: &FormatString, - dict: &PyObjectRef, -) -> PyResult { - let mut final_string = String::new(); - for part in &format_string.format_parts { - let result_string: String = match part { - FormatPart::AutoSpec(_) | FormatPart::IndexSpec(_, _) => { - return Err( - vm.new_value_error("Format string contains positional fields".to_owned()) - ); - } - FormatPart::KeywordSpec(keyword, format_spec) => { - let argument = dict.get_item(keyword, &vm)?; - let result = call_object_format(vm, argument.clone(), &format_spec)?; - clone_value(&result) - } - FormatPart::Literal(literal) => literal.clone(), - }; - final_string.push_str(&result_string); - } - Ok(vm.ctx.new_str(final_string)) -} - impl PySliceableSequence for str { type Sliced = String; From 1b2d179b3892daa5507c7bb507b277d782efefd7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 06:00:36 +0900 Subject: [PATCH 072/209] Add string typed formatting --- vm/src/format.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/format.rs b/vm/src/format.rs index d468f7ae6..7a8945e1a 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -227,6 +227,7 @@ fn parse_grouping_option(text: &str) -> (Option, &str) { fn parse_format_type(text: &str) -> (Option, &str) { let mut chars = text.chars(); match chars.next() { + Some('s') => (Some(FormatType::String), chars.as_str()), Some('b') => (Some(FormatType::Binary), chars.as_str()), Some('c') => (Some(FormatType::Character), chars.as_str()), Some('d') => (Some(FormatType::Decimal), chars.as_str()), From 5b20601748c1e4d7651c578cc48b77bc6c8e6055 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2020 06:39:04 +0900 Subject: [PATCH 073/209] pystr py_cformat --- vm/src/obj/objbytearray.rs | 9 ++------- vm/src/obj/objbyteinner.rs | 8 ++++++++ vm/src/obj/objbytes.rs | 11 ++--------- vm/src/obj/objstr.rs | 14 +++++++------- vm/src/obj/pystr.rs | 10 ++++++++++ 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 9ca3a775e..da7a47551 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -4,7 +4,6 @@ use crossbeam_utils::atomic::AtomicCell; use num_traits::cast::ToPrimitive; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::mem::size_of; -use std::str::FromStr; use super::objbyteinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, @@ -16,7 +15,6 @@ use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; -use crate::cformat::CFormatString; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, @@ -557,11 +555,8 @@ impl PyByteArray { #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut format_string = - CFormatString::from_str(std::str::from_utf8(&self.borrow_value().elements).unwrap()) - .map_err(|err| vm.new_value_error(err.to_string()))?; - let final_string = format_string.format(vm, values)?; - Ok(final_string.as_str().as_bytes().to_owned().into()) + let formatted = self.borrow_value().cformat(values, vm)?; + Ok(formatted.as_str().as_bytes().to_owned().into()) } #[pymethod(name = "__rmod__")] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 9893156e0..b4ef38139 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1133,6 +1133,10 @@ impl PyByteInner { res } + pub fn cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.elements.py_cformat(values, vm) + } + pub fn repeat(&self, n: isize) -> Vec { self.elements.repeat(n.to_usize().unwrap_or(0)) } @@ -1261,6 +1265,10 @@ impl<'s> PyCommonString<'s, u8> for [u8] { self } + fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { + std::str::from_utf8(self) + } + fn chars(&'s self) -> Self::CharIter { bstr::ByteSlice::chars(self) } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f2659a5ca..ffeea4c05 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -2,7 +2,6 @@ use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; use std::mem::size_of; use std::ops::Deref; -use std::str::FromStr; use super::objbyteinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, @@ -14,7 +13,6 @@ use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; -use crate::cformat::CFormatString; use crate::function::{OptionalArg, OptionalOption}; use crate::pyhash; use crate::pyobject::{ @@ -455,13 +453,8 @@ impl PyBytes { #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let format_string_text = std::str::from_utf8(&self.inner.elements).unwrap(); - let mut format_string = CFormatString::from_str(format_string_text) - .map_err(|err| vm.new_value_error(err.to_string()))?; - let final_string = format_string.format(vm, values)?; - Ok(vm - .ctx - .new_bytes(final_string.as_str().as_bytes().to_owned())) + let formatted = self.inner.cformat(values, vm)?; + Ok(vm.ctx.new_bytes(formatted.as_bytes().to_owned())) } #[pymethod(name = "__rmod__")] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index b11220f4c..02acf5d04 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -2,7 +2,6 @@ use std::char; use std::fmt; use std::mem::size_of; use std::ops::Range; -use std::str::FromStr; use std::string::ToString; use crossbeam_utils::atomic::AtomicCell; @@ -23,7 +22,6 @@ use super::objtype::{self, PyClassRef}; use super::pystr::{ self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, }; -use crate::cformat::CFormatString; use crate::format::{FormatParseError, FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyhash; @@ -578,11 +576,9 @@ impl PyString { } #[pymethod(name = "__mod__")] - fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let format_string_text = &self.value; - let mut format_string = CFormatString::from_str(format_string_text) - .map_err(|err| vm.new_value_error(err.to_string()))?; - Ok(vm.ctx.new_str(format_string.format(vm, values)?)) + fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let formatted = self.value.py_cformat(values, vm)?; + Ok(formatted) } #[pymethod(name = "__rmod__")] @@ -1393,6 +1389,10 @@ impl<'s> PyCommonString<'s, char> for str { self.as_bytes() } + fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { + Ok(self) + } + fn chars(&'s self) -> Self::CharIter { str::chars(self) } diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index c37ea0b68..3b37d4c9e 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -1,8 +1,10 @@ +use crate::cformat::CFormatString; use crate::function::{single_or_tuple_any, OptionalOption}; use crate::obj::objint::PyIntRef; use crate::pyobject::{PyIterator, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use num_traits::{cast::ToPrimitive, sign::Signed}; +use std::str::FromStr; #[derive(FromArgs)] pub struct SplitArgs<'a, T, S, E> @@ -152,6 +154,7 @@ where fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; + fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error>; fn chars(&'s self) -> Self::CharIter; fn elements(&'s self) -> Self::ElementIter; fn get_bytes<'a>(&'a self, range: std::ops::Range) -> &'a Self; @@ -429,4 +432,11 @@ where } cased } + + fn py_cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let format_string = self.as_utf8_str().unwrap(); + CFormatString::from_str(format_string) + .map_err(|err| vm.new_value_error(err.to_string()))? + .format(vm, values) + } } From 466e6fcfc3aa0e8195dafdbd2fc052a7de305454 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 24 Jul 2020 05:57:10 +0900 Subject: [PATCH 074/209] clean up unnssesary mut from cformat --- vm/src/cformat.rs | 50 ++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 2675507e3..47ea73058 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -146,17 +146,29 @@ impl CFormatSpec { } } - pub(crate) fn format_string(&self, string: String) -> String { - let mut string = string; + fn format_string_with_precision( + &self, + string: String, + precision: Option<&CFormatQuantity>, + ) -> String { // truncate if needed - if let Some(CFormatQuantity::Amount(precision)) = self.precision { - if string.chars().count() > precision { - string = string.chars().take(precision).collect::(); + let string = match precision { + Some(CFormatQuantity::Amount(precision)) if string.chars().count() > *precision => { + string.chars().take(*precision).collect::() } - } + _ => string, + }; self.fill_string(string, ' ', None) } + pub(crate) fn format_string(&self, string: String) -> String { + self.format_string_with_precision(string, self.precision.as_ref()) + } + + fn format_char(&self, ch: char) -> String { + self.format_string_with_precision(ch.to_string(), Some(&CFormatQuantity::Amount(1))) + } + fn format_number(&self, num: &BigInt) -> String { use CFormatCase::{Lowercase, Uppercase}; use CNumberType::*; @@ -267,13 +279,9 @@ impl CFormatSpec { } } - fn format(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - use CNumberType::*; - + fn format(&self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { // do the formatting by type - let format_type = &self.format_type; - - match format_type { + match &self.format_type { CFormatType::String(preconversor) => { let result = match preconversor { CFormatPreconversor::Str => vm.to_str(&obj)?, @@ -285,12 +293,11 @@ impl CFormatSpec { }; Ok(self.format_string(result.as_str().to_owned())) } - CFormatType::Number(_) => { + CFormatType::Number(number_type) => { if !objtype::isinstance(&obj, &vm.ctx.types.int_type) { - let required_type_string = match format_type { - CFormatType::Number(Decimal) => "a number", - CFormatType::Number(_) => "an integer", - _ => unreachable!(), + let required_type_string = match number_type { + CNumberType::Decimal => "a number", + _ => "an integer", }; return Err(vm.new_type_error(format!( "%{} format: {} is required, not {}", @@ -316,14 +323,14 @@ impl CFormatSpec { } .map_err(|e| vm.new_not_implemented_error(e)), CFormatType::Character => { - let char_string = { + let ch = { if objtype::isinstance(&obj, &vm.ctx.types.int_type) { // BigInt truncation is fine in this case because only the unicode range is relevant match objint::get_value(&obj) .to_u32() .and_then(std::char::from_u32) { - Some(value) => Ok(value.to_string()), + Some(value) => Ok(value), None => { Err(vm .new_overflow_error("%c arg not in range(0x110000)".to_owned())) @@ -335,15 +342,14 @@ impl CFormatSpec { if num_chars != 1 { Err(vm.new_type_error("%c requires int or char".to_owned())) } else { - Ok(s.chars().next().unwrap().to_string()) + Ok(s.chars().next().unwrap()) } } else { // TODO re-arrange this block so this error is only created once Err(vm.new_type_error("%c requires int or char".to_owned())) } }?; - self.precision = Some(CFormatQuantity::Amount(1)); - Ok(self.format_string(char_string)) + Ok(self.format_char(ch)) } } } From 6a713a5b8bb0ecba033c1ecfd2051eac1d7bac44 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Thu, 23 Jul 2020 16:11:38 -0700 Subject: [PATCH 075/209] Added _base_executable variable to sys module --- vm/src/sysmodule.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 0b11a5619..3577bd83b 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -35,6 +35,14 @@ fn executable(ctx: &PyContext) -> PyObjectRef { } } +fn _base_executable(ctx: &PyContext) -> PyObjectRef { + if let Ok(var) = env::var("__PYVENV_LAUNCHER__") { + ctx.new_str(var) + } else { + executable(ctx) + } +} + fn getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult { let offset = offset.into_option().unwrap_or(0); if offset > vm.frames.borrow().len() - 1 { @@ -340,6 +348,7 @@ prefix -- prefix used to find the Python library thread_info -- a struct sequence with information about the thread implementation. version -- the version of this interpreter as a string version_info -- version information as a named tuple +_base_executable -- __PYVENV_LAUNCHER__ enviroment variable if defined, else sys.executable. __stdin__ -- the original stdin; don't touch! __stdout__ -- the original stdout; don't touch! __stderr__ -- the original stderr; don't touch! @@ -387,6 +396,7 @@ settrace() -- set the global debug tracing function "builtin_module_names" => builtin_module_names, "byteorder" => ctx.new_str(bytorder), "copyright" => ctx.new_str(copyright.to_owned()), + "_base_executable" => _base_executable(ctx), "executable" => executable(ctx), "flags" => flags, "getrefcount" => ctx.new_function(sys_getrefcount), From 10f53935dc280c62aca5e8953298be757283a460 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 24 Jul 2020 08:07:41 +0900 Subject: [PATCH 076/209] clean up cformat a bit --- vm/src/cformat.rs | 68 +++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 47ea73058..a04160248 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -92,6 +92,18 @@ bitflags! { } } +impl CConversionFlags { + fn sign_string(&self) -> &'static str { + if self.contains(CConversionFlags::SIGN_CHAR) { + "+" + } else if self.contains(CConversionFlags::BLANK_SIGN) { + " " + } else { + "" + } + } +} + #[derive(Debug, PartialEq)] enum CFormatQuantity { Amount(usize), @@ -198,46 +210,37 @@ impl CFormatSpec { let sign_string = match num.sign() { Sign::Minus => "-", - _ => { - if self.flags.contains(CConversionFlags::SIGN_CHAR) { - "+" - } else if self.flags.contains(CConversionFlags::BLANK_SIGN) { - " " - } else { - "" - } - } + _ => self.flags.sign_string(), }; - let prefix = format!("{}{}", sign_string, prefix); - if self.flags.contains(CConversionFlags::ZERO_PAD) { let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { '0' } else { ' ' // '-' overrides the '0' conversion if both are given }; + let signed_prefix = format!("{}{}", sign_string, prefix); format!( "{}{}", - prefix, - self.fill_string(magnitude_string, fill_char, Some(prefix.chars().count())) + signed_prefix, + self.fill_string( + magnitude_string, + fill_char, + Some(signed_prefix.chars().count()) + ) ) } else { - self.fill_string(format!("{}{}", prefix, magnitude_string), ' ', None) + self.fill_string( + format!("{}{}{}", sign_string, prefix, magnitude_string), + ' ', + None, + ) } } pub(crate) fn format_float(&self, num: f64) -> Result { - let magnitude = num.abs(); - let sign_string = if num.is_sign_positive() { - if self.flags.contains(CConversionFlags::SIGN_CHAR) { - "+" - } else if self.flags.contains(CConversionFlags::BLANK_SIGN) { - " " - } else { - "" - } + self.flags.sign_string() } else { "-" }; @@ -248,6 +251,7 @@ impl CFormatSpec { Some(CFormatQuantity::Amount(p)) => p, _ => 6, }; + let magnitude = num.abs(); format!("{:.*}", precision, magnitude) } CFormatType::Float(CFloatType::Exponent(_)) => { @@ -308,10 +312,8 @@ impl CFormatSpec { } Ok(self.format_number(objint::get_value(&obj))) } - CFormatType::Float(_) => if objtype::isinstance(&obj, &vm.ctx.types.float_type) { - self.format_float(objfloat::get_value(&obj)) - } else if objtype::isinstance(&obj, &vm.ctx.types.int_type) { - self.format_float(objint::get_value(&obj).to_f64().unwrap()) + CFormatType::Float(_) => if let Some(value) = objfloat::try_float(&obj, vm)? { + self.format_float(value) } else { let required_type_string = "an floating point or integer"; return Err(vm.new_type_error(format!( @@ -326,16 +328,12 @@ impl CFormatSpec { let ch = { if objtype::isinstance(&obj, &vm.ctx.types.int_type) { // BigInt truncation is fine in this case because only the unicode range is relevant - match objint::get_value(&obj) + objint::get_value(&obj) .to_u32() .and_then(std::char::from_u32) - { - Some(value) => Ok(value), - None => { - Err(vm - .new_overflow_error("%c arg not in range(0x110000)".to_owned())) - } - } + .ok_or_else(|| { + vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) + }) } else if objtype::isinstance(&obj, &vm.ctx.types.str_type) { let s = objstr::borrow_value(&obj); let num_chars = s.chars().count(); From f65765fcab46a72688f3c5e9b464669e134ba854 Mon Sep 17 00:00:00 2001 From: minoring Date: Fri, 24 Jul 2020 19:42:58 +0900 Subject: [PATCH 077/209] Add CPython 3.83 test test_cmd.py All tests are passed. --- Lib/test/test_cmd.py | 242 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 Lib/test/test_cmd.py diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py new file mode 100644 index 000000000..96e0c30da --- /dev/null +++ b/Lib/test/test_cmd.py @@ -0,0 +1,242 @@ +""" +Test script for the 'cmd' module +Original by Michael Schneider +""" + + +import cmd +import sys +import unittest +import io +from test import support + +class samplecmdclass(cmd.Cmd): + """ + Instance the sampleclass: + >>> mycmd = samplecmdclass() + + Test for the function parseline(): + >>> mycmd.parseline("") + (None, None, '') + >>> mycmd.parseline("?") + ('help', '', 'help ') + >>> mycmd.parseline("?help") + ('help', 'help', 'help help') + >>> mycmd.parseline("!") + ('shell', '', 'shell ') + >>> mycmd.parseline("!command") + ('shell', 'command', 'shell command') + >>> mycmd.parseline("func") + ('func', '', 'func') + >>> mycmd.parseline("func arg1") + ('func', 'arg1', 'func arg1') + + + Test for the function onecmd(): + >>> mycmd.onecmd("") + >>> mycmd.onecmd("add 4 5") + 9 + >>> mycmd.onecmd("") + 9 + >>> mycmd.onecmd("test") + *** Unknown syntax: test + + Test for the function emptyline(): + >>> mycmd.emptyline() + *** Unknown syntax: test + + Test for the function default(): + >>> mycmd.default("default") + *** Unknown syntax: default + + Test for the function completedefault(): + >>> mycmd.completedefault() + This is the completedefault method + >>> mycmd.completenames("a") + ['add'] + + Test for the function completenames(): + >>> mycmd.completenames("12") + [] + >>> mycmd.completenames("help") + ['help'] + + Test for the function complete_help(): + >>> mycmd.complete_help("a") + ['add'] + >>> mycmd.complete_help("he") + ['help'] + >>> mycmd.complete_help("12") + [] + >>> sorted(mycmd.complete_help("")) + ['add', 'exit', 'help', 'shell'] + + Test for the function do_help(): + >>> mycmd.do_help("testet") + *** No help on testet + >>> mycmd.do_help("add") + help text for add + >>> mycmd.onecmd("help add") + help text for add + >>> mycmd.do_help("") + + Documented commands (type help ): + ======================================== + add help + + Undocumented commands: + ====================== + exit shell + + + Test for the function print_topics(): + >>> mycmd.print_topics("header", ["command1", "command2"], 2 ,10) + header + ====== + command1 + command2 + + + Test for the function columnize(): + >>> mycmd.columnize([str(i) for i in range(20)]) + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 + >>> mycmd.columnize([str(i) for i in range(20)], 10) + 0 7 14 + 1 8 15 + 2 9 16 + 3 10 17 + 4 11 18 + 5 12 19 + 6 13 + + This is an interactive test, put some commands in the cmdqueue attribute + and let it execute + This test includes the preloop(), postloop(), default(), emptyline(), + parseline(), do_help() functions + >>> mycmd.use_rawinput=0 + >>> mycmd.cmdqueue=["", "add", "add 4 5", "help", "help add","exit"] + >>> mycmd.cmdloop() + Hello from preloop + help text for add + *** invalid number of arguments + 9 + + Documented commands (type help ): + ======================================== + add help + + Undocumented commands: + ====================== + exit shell + + help text for add + Hello from postloop + """ + + def preloop(self): + print("Hello from preloop") + + def postloop(self): + print("Hello from postloop") + + def completedefault(self, *ignored): + print("This is the completedefault method") + + def complete_command(self): + print("complete command") + + def do_shell(self, s): + pass + + def do_add(self, s): + l = s.split() + if len(l) != 2: + print("*** invalid number of arguments") + return + try: + l = [int(i) for i in l] + except ValueError: + print("*** arguments should be numbers") + return + print(l[0]+l[1]) + + def help_add(self): + print("help text for add") + return + + def do_exit(self, arg): + return True + + +class TestAlternateInput(unittest.TestCase): + + class simplecmd(cmd.Cmd): + + def do_print(self, args): + print(args, file=self.stdout) + + def do_EOF(self, args): + return True + + + class simplecmd2(simplecmd): + + def do_EOF(self, args): + print('*** Unknown syntax: EOF', file=self.stdout) + return True + + + def test_file_with_missing_final_nl(self): + input = io.StringIO("print test\nprint test2") + output = io.StringIO() + cmd = self.simplecmd(stdin=input, stdout=output) + cmd.use_rawinput = False + cmd.cmdloop() + self.assertMultiLineEqual(output.getvalue(), + ("(Cmd) test\n" + "(Cmd) test2\n" + "(Cmd) ")) + + + def test_input_reset_at_EOF(self): + input = io.StringIO("print test\nprint test2") + output = io.StringIO() + cmd = self.simplecmd2(stdin=input, stdout=output) + cmd.use_rawinput = False + cmd.cmdloop() + self.assertMultiLineEqual(output.getvalue(), + ("(Cmd) test\n" + "(Cmd) test2\n" + "(Cmd) *** Unknown syntax: EOF\n")) + input = io.StringIO("print \n\n") + output = io.StringIO() + cmd.stdin = input + cmd.stdout = output + cmd.cmdloop() + self.assertMultiLineEqual(output.getvalue(), + ("(Cmd) \n" + "(Cmd) \n" + "(Cmd) *** Unknown syntax: EOF\n")) + + +def test_main(verbose=None): + from test import test_cmd + support.run_doctest(test_cmd, verbose) + support.run_unittest(TestAlternateInput) + +def test_coverage(coverdir): + trace = support.import_module('trace') + tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], + trace=0, count=1) + tracer.run('import importlib; importlib.reload(cmd); test_main()') + r=tracer.results() + print("Writing coverage results...") + r.write_results(show_missing=True, summary=True, coverdir=coverdir) + +if __name__ == "__main__": + if "-c" in sys.argv: + test_coverage('/tmp/cmd.cover') + elif "-i" in sys.argv: + samplecmdclass().cmdloop() + else: + test_main() From 308468f1d5b109e5faab8fd2e639e223ab98e9c6 Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Sun, 5 Jul 2020 12:47:45 +0200 Subject: [PATCH 078/209] First implementation of chown for the os module --- vm/src/stdlib/os.rs | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c9ff3c116..2f6c1246f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1045,6 +1045,47 @@ fn os_chroot(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { nix::unistd::chroot(&*path.path).map_err(|err| convert_nix_error(vm, err)) } +// As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) +#[cfg(all(unix, not(target_os = "redox")))] +fn os_chown( + path: PyPathLike, + uid: PyIntRef, + gid: PyIntRef, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult<()> { + let uid = isize::try_from_object(&vm, uid.as_object().clone())?; + let gid = isize::try_from_object(&vm, gid.as_object().clone())?; + + let uid = if uid >= 0 { + Some(nix::unistd::Uid::from_raw(uid as u32)) + } else if uid == -1 { + None + } else { + return Err(vm.new_os_error(String::from("Specified uid is not valid."))); + }; + + let gid = if gid >= 0 { + Some(nix::unistd::Gid::from_raw(gid as u32)) + } else if gid == -1 { + None + } else { + return Err(vm.new_os_error(String::from("Specified gid is not valid."))); + }; + + let flag = match follow_symlinks.follow_symlinks { + true => nix::unistd::FchownatFlags::FollowSymlink, + false => nix::unistd::FchownatFlags::NoFollowSymlink, + }; + let dir_fd: Option = match dir_fd.dir_fd { + Some(int_ref) => Some(i32::try_from_object(&vm, int_ref.as_object().clone())?), + None => None, + }; + nix::unistd::fchownat(dir_fd, path.path.as_os_str(), uid, gid, flag) + .map_err(|err| convert_nix_error(vm, err)) +} + #[cfg(unix)] fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { use nix::fcntl::fcntl; @@ -2063,7 +2104,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "access", os_access, Some(false), Some(false), None), SupportFunc::new(vm, "chdir", os_chdir, Some(false), None, None), // chflags Some, None Some - // chown Some Some Some SupportFunc::new(vm, "listdir", os_listdir, Some(false), None, None), SupportFunc::new(vm, "mkdir", os_mkdir, Some(false), Some(false), None), // mkfifo Some Some None @@ -2088,6 +2128,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "chmod", os_chmod, Some(false), Some(false), Some(false)), #[cfg(not(target_os = "redox"))] SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), + SupportFunc::new(vm, "chown", os_chown, Some(false), Some(false), Some(false)), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), ]); let supports_fd = PySet::default().into_ref(vm); @@ -2236,6 +2277,7 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { #[cfg(not(target_os = "redox"))] extend_module!(vm, module, { + "chown" => ctx.new_function(os_chown), "chroot" => ctx.new_function(os_chroot), "getsid" => ctx.new_function(os_getsid), "setsid" => ctx.new_function(os_setsid), From eaa4a3bfd4ccd76a5a4e58d721ca9c91c5648f8e Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Tue, 14 Jul 2020 19:31:38 +0200 Subject: [PATCH 079/209] fixing clippy lint --- vm/src/stdlib/os.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 2f6c1246f..570e1bee1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1074,10 +1074,12 @@ fn os_chown( return Err(vm.new_os_error(String::from("Specified gid is not valid."))); }; - let flag = match follow_symlinks.follow_symlinks { - true => nix::unistd::FchownatFlags::FollowSymlink, - false => nix::unistd::FchownatFlags::NoFollowSymlink, + let flag = if follow_symlinks.follow_symlinks { + nix::unistd::FchownatFlags::FollowSymlink + } else { + nix::unistd::FchownatFlags::NoFollowSymlink }; + let dir_fd: Option = match dir_fd.dir_fd { Some(int_ref) => Some(i32::try_from_object(&vm, int_ref.as_object().clone())?), None => None, From 6d541634c5ccfb955ce8e6b976f9906afd6ca911 Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Tue, 14 Jul 2020 19:34:34 +0200 Subject: [PATCH 080/209] setting proper values for supports_{fd,dir_fd,follow_symlinks} sets for chown function --- vm/src/stdlib/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 570e1bee1..3b894cc11 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -2130,7 +2130,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "chmod", os_chmod, Some(false), Some(false), Some(false)), #[cfg(not(target_os = "redox"))] SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), - SupportFunc::new(vm, "chown", os_chown, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "chown", os_chown, None, Some(true), Some(true)), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), ]); let supports_fd = PySet::default().into_ref(vm); From dfffca4696b5af4d1d512f42b9af8c37e88b1730 Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Thu, 16 Jul 2020 21:55:58 +0200 Subject: [PATCH 081/209] Running flake8 linter on test file --- tests/snippets/stdlib_os.py | 346 +++++++++++++++++++----------------- 1 file changed, 178 insertions(+), 168 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 0eb820170..b746c6c8b 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -10,16 +10,17 @@ assert fd > 0 os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) -assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY)) -assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY)) -assert_raises(FileNotFoundError, lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2')) +assert_raises(FileNotFoundError, + lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY)) +assert_raises(FileNotFoundError, + lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY)) +assert_raises(FileNotFoundError, + lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2')) try: - os.open('DOES_NOT_EXIST', 0) + os.open('DOES_NOT_EXIST', 0) except OSError as err: - assert err.errno == 2 - - + assert err.errno == 2 assert os.O_RDONLY == 0 assert os.O_WRONLY == 1 @@ -28,7 +29,7 @@ assert os.O_RDWR == 2 ENV_KEY = "TEST_ENV_VAR" ENV_VALUE = "value" -assert os.getenv(ENV_KEY) == None +assert os.getenv(ENV_KEY) is None assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY, 5) == 5 os.environ[ENV_KEY] = ENV_VALUE @@ -36,31 +37,32 @@ assert ENV_KEY in os.environ assert os.getenv(ENV_KEY) == ENV_VALUE del os.environ[ENV_KEY] assert ENV_KEY not in os.environ -assert os.getenv(ENV_KEY) == None +assert os.getenv(ENV_KEY) is None if os.name == "posix": - os.putenv(ENV_KEY, ENV_VALUE) - os.unsetenv(ENV_KEY) - assert os.getenv(ENV_KEY) == None + os.putenv(ENV_KEY, ENV_VALUE) + os.unsetenv(ENV_KEY) + assert os.getenv(ENV_KEY) is None assert os.curdir == "." assert os.pardir == ".." assert os.extsep == "." if os.name == "nt": - assert os.sep == "\\" - assert os.linesep == "\r\n" - assert os.altsep == "/" - assert os.pathsep == ";" + assert os.sep == "\\" + assert os.linesep == "\r\n" + assert os.altsep == "/" + assert os.pathsep == ";" else: - assert os.sep == "/" - assert os.linesep == "\n" - assert os.altsep == None - assert os.pathsep == ":" + assert os.sep == "/" + assert os.linesep == "\n" + assert os.altsep is None + assert os.pathsep == ":" assert os.fspath("Testing") == "Testing" assert os.fspath(b"Testing") == b"Testing" -assert_raises(TypeError, lambda: os.fspath([1,2,3])) +assert_raises(TypeError, lambda: os.fspath([1, 2, 3])) + class TestWithTempDir(): def __enter__(self): @@ -69,7 +71,8 @@ class TestWithTempDir(): else: base_folder = "/tmp" - name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time()))) + name = os.path.join(base_folder, + "rustpython_test_os_" + str(int(time.time()))) while os.path.isdir(name): name = name + "_" @@ -81,6 +84,7 @@ class TestWithTempDir(): def __exit__(self, exc_type, exc_val, exc_tb): pass + class TestWithTempCurrentDir(): def __enter__(self): self.prev_cwd = os.getcwd() @@ -100,174 +104,179 @@ CONTENT2 = b"rustpython" CONTENT3 = b"BOYA" with TestWithTempDir() as tmpdir: - fname = os.path.join(tmpdir, FILE_NAME) - fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL) - assert os.write(fd, CONTENT2) == len(CONTENT2) - os.close(fd) + fname = os.path.join(tmpdir, FILE_NAME) + fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + assert os.write(fd, CONTENT2) == len(CONTENT2) + os.close(fd) - fd = os.open(fname, os.O_WRONLY | os.O_APPEND) - assert os.write(fd, CONTENT3) == len(CONTENT3) - os.close(fd) + fd = os.open(fname, os.O_WRONLY | os.O_APPEND) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) - assert_raises(FileExistsError, lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + assert_raises(FileExistsError, + lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) - fd = os.open(fname, os.O_RDONLY) - assert os.read(fd, len(CONTENT2)) == CONTENT2 - assert os.read(fd, len(CONTENT3)) == CONTENT3 - os.close(fd) + fd = os.open(fname, os.O_RDONLY) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) - fname3 = os.path.join(tmpdir, FILE_NAME3) - os.rename(fname, fname3) - assert os.path.exists(fname) == False - assert os.path.exists(fname3) == True + fname3 = os.path.join(tmpdir, FILE_NAME3) + os.rename(fname, fname3) + assert os.path.exists(fname) is False + assert os.path.exists(fname3) is True - fd = os.open(fname3, 0) - assert os.read(fd, len(CONTENT2) + len(CONTENT3)) == CONTENT2 + CONTENT3 - os.close(fd) + fd = os.open(fname3, 0) + assert os.read(fd, len(CONTENT2) + len(CONTENT3)) == CONTENT2 + CONTENT3 + os.close(fd) - assert not os.isatty(fd) + assert not os.isatty(fd) - # TODO: get os.lseek working on windows - if os.name != 'nt': - fd = os.open(fname3, 0) - assert os.read(fd, len(CONTENT2)) == CONTENT2 - assert os.read(fd, len(CONTENT3)) == CONTENT3 - os.lseek(fd, len(CONTENT2), os.SEEK_SET) - assert os.read(fd, len(CONTENT3)) == CONTENT3 - os.close(fd) + # TODO: get os.lseek working on windows + if os.name != 'nt': + fd = os.open(fname3, 0) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.lseek(fd, len(CONTENT2), os.SEEK_SET) + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) - os.rename(fname3, fname) - assert os.path.exists(fname3) == False - assert os.path.exists(fname) == True + os.rename(fname3, fname) + assert os.path.exists(fname3) is False + assert os.path.exists(fname) is True - # wait a little bit to ensure that the file times aren't the same - time.sleep(0.1) + # wait a little bit to ensure that the file times aren't the same + time.sleep(0.1) - fname2 = os.path.join(tmpdir, FILE_NAME2) - with open(fname2, "wb"): - pass - folder = os.path.join(tmpdir, FOLDER) - os.mkdir(folder) + fname2 = os.path.join(tmpdir, FILE_NAME2) + with open(fname2, "wb"): + pass + folder = os.path.join(tmpdir, FOLDER) + os.mkdir(folder) - symlink_file = os.path.join(tmpdir, SYMLINK_FILE) - os.symlink(fname, symlink_file) - symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER) - os.symlink(folder, symlink_folder) + symlink_file = os.path.join(tmpdir, SYMLINK_FILE) + os.symlink(fname, symlink_file) + symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER) + os.symlink(folder, symlink_folder) - names = set() - paths = set() - dirs = set() - dirs_no_symlink = set() - files = set() - files_no_symlink = set() - symlinks = set() - for dir_entry in os.scandir(tmpdir): - names.add(dir_entry.name) - paths.add(dir_entry.path) - if dir_entry.is_dir(): - assert stat.S_ISDIR(dir_entry.stat().st_mode) == True - dirs.add(dir_entry.name) - if dir_entry.is_dir(follow_symlinks=False): - assert stat.S_ISDIR(dir_entry.stat().st_mode) == True - dirs_no_symlink.add(dir_entry.name) - if dir_entry.is_file(): - files.add(dir_entry.name) - assert stat.S_ISREG(dir_entry.stat().st_mode) == True - if dir_entry.is_file(follow_symlinks=False): - files_no_symlink.add(dir_entry.name) - assert stat.S_ISREG(dir_entry.stat().st_mode) == True - if dir_entry.is_symlink(): - symlinks.add(dir_entry.name) + names = set() + paths = set() + dirs = set() + dirs_no_symlink = set() + files = set() + files_no_symlink = set() + symlinks = set() + for dir_entry in os.scandir(tmpdir): + names.add(dir_entry.name) + paths.add(dir_entry.path) + if dir_entry.is_dir(): + assert stat.S_ISDIR(dir_entry.stat().st_mode) is True + dirs.add(dir_entry.name) + if dir_entry.is_dir(follow_symlinks=False): + assert stat.S_ISDIR(dir_entry.stat().st_mode) is True + dirs_no_symlink.add(dir_entry.name) + if dir_entry.is_file(): + files.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) is True + if dir_entry.is_file(follow_symlinks=False): + files_no_symlink.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) is True + if dir_entry.is_symlink(): + symlinks.add(dir_entry.name) - assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) - assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) - assert dirs == set([FOLDER, SYMLINK_FOLDER]) - assert dirs_no_symlink == set([FOLDER]) - assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE]) - assert files_no_symlink == set([FILE_NAME, FILE_NAME2]) - assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER]) + assert names == set( + [FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) + assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) + assert dirs == set([FOLDER, SYMLINK_FOLDER]) + assert dirs_no_symlink == set([FOLDER]) + assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE]) + assert files_no_symlink == set([FILE_NAME, FILE_NAME2]) + assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER]) - # Stat - stat_res = os.stat(fname) - print(stat_res.st_mode) - assert stat.S_ISREG(stat_res.st_mode) == True - print(stat_res.st_ino) - print(stat_res.st_dev) - print(stat_res.st_nlink) - print(stat_res.st_uid) - print(stat_res.st_gid) - print(stat_res.st_size) - assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) - print(stat_res.st_atime) - print(stat_res.st_ctime) - print(stat_res.st_mtime) - # test that it all of these times are greater than the 10 May 2019, when this test was written - assert stat_res.st_atime > 1557500000 - assert stat_res.st_ctime > 1557500000 - assert stat_res.st_mtime > 1557500000 + # Stat + stat_res = os.stat(fname) + print(stat_res.st_mode) + assert stat.S_ISREG(stat_res.st_mode) is True + print(stat_res.st_ino) + print(stat_res.st_dev) + print(stat_res.st_nlink) + print(stat_res.st_uid) + print(stat_res.st_gid) + print(stat_res.st_size) + assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + # test that it all of these times are greater than the 10 May 2019, + # when this test was written + assert stat_res.st_atime > 1557500000 + assert stat_res.st_ctime > 1557500000 + assert stat_res.st_mtime > 1557500000 - bytes_stats_res = os.stat(fname.encode()) + bytes_stats_res = os.stat(fname.encode()) - stat_file2 = os.stat(fname2) - print(stat_file2.st_ctime) - assert stat_file2.st_ctime > stat_res.st_ctime + stat_file2 = os.stat(fname2) + print(stat_file2.st_ctime) + assert stat_file2.st_ctime > stat_res.st_ctime - # wait a little bit to ensures that the access/modify time will change - time.sleep(0.1) + # wait a little bit to ensures that the access/modify time will change + time.sleep(0.1) - old_atime = stat_res.st_atime - old_mtime = stat_res.st_mtime + old_atime = stat_res.st_atime + old_mtime = stat_res.st_mtime - fd = os.open(fname, os.O_RDWR) - os.write(fd, CONTENT) - os.fsync(fd) + fd = os.open(fname, os.O_RDWR) + os.write(fd, CONTENT) + os.fsync(fd) - # wait a little bit to ensures that the access/modify time is different - time.sleep(0.1) + # wait a little bit to ensures that the access/modify time is different + time.sleep(0.1) - os.read(fd, 1) - os.fsync(fd) - os.close(fd) + os.read(fd, 1) + os.fsync(fd) + os.close(fd) - # retrieve update file stats - stat_res = os.stat(fname) - print(stat_res.st_atime) - print(stat_res.st_ctime) - print(stat_res.st_mtime) - if os.name != "nt": - # access time on windows has a resolution ranging from 1 hour to 1 day - # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime - assert stat_res.st_atime > old_atime, "Access time should be update" - assert stat_res.st_atime > stat_res.st_mtime - assert stat_res.st_mtime > old_mtime, "Modified time should be update" + # retrieve update file stats + stat_res = os.stat(fname) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + if os.name != "nt": + # access time on windows has a resolution ranging from 1 hour to 1 day + # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime + assert stat_res.st_atime > old_atime, "Access time should be update" + assert stat_res.st_atime > stat_res.st_mtime + assert stat_res.st_mtime > old_mtime, "Modified time should be update" - # stat default is follow_symlink=True - os.stat(fname).st_ino == os.stat(symlink_file).st_ino - os.stat(fname).st_mode == os.stat(symlink_file).st_mode + # stat default is follow_symlink=True + os.stat(fname).st_ino == os.stat(symlink_file).st_ino + os.stat(fname).st_mode == os.stat(symlink_file).st_mode - os.stat(fname, follow_symlinks=False).st_ino == os.stat(symlink_file, follow_symlinks=False).st_ino - os.stat(fname, follow_symlinks=False).st_mode == os.stat(symlink_file, follow_symlinks=False).st_mode + os.stat(fname, follow_symlinks=False).st_ino == os.stat( + symlink_file, follow_symlinks=False).st_ino + os.stat(fname, follow_symlinks=False).st_mode == os.stat( + symlink_file, follow_symlinks=False).st_mode - # os.chmod - if os.name != "nt": - os.chmod(fname, 0o666) - assert oct(os.stat(fname).st_mode) == '0o100666' + # os.chmod + if os.name != "nt": + os.chmod(fname, 0o666) + assert oct(os.stat(fname).st_mode) == '0o100666' - # os.path - assert os.path.exists(fname) == True - assert os.path.exists("NO_SUCH_FILE") == False - assert os.path.isfile(fname) == True - assert os.path.isdir(folder) == True - assert os.path.isfile(folder) == False - assert os.path.isdir(fname) == False + # os.path + assert os.path.exists(fname) is True + assert os.path.exists("NO_SUCH_FILE") is False + assert os.path.isfile(fname) is True + assert os.path.isdir(folder) is True + assert os.path.isfile(folder) is False + assert os.path.isdir(fname) is False - assert os.path.basename(fname) == FILE_NAME - assert os.path.dirname(fname) == tmpdir + assert os.path.basename(fname) == FILE_NAME + assert os.path.dirname(fname) == tmpdir - with TestWithTempCurrentDir(): - os.chdir(tmpdir) - assert os.path.realpath(os.getcwd()) == os.path.realpath(tmpdir) - assert os.path.exists(FILE_NAME) + with TestWithTempCurrentDir(): + os.chdir(tmpdir) + assert os.path.realpath(os.getcwd()) == os.path.realpath(tmpdir) + assert os.path.exists(FILE_NAME) # supports assert isinstance(os.supports_fd, set) @@ -367,7 +376,6 @@ if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'): os.close(rfd) os.close(wfd) - with TestWithTempDir() as tmpdir: for i in range(0, 4): file_name = os.path.join(tmpdir, 'file' + str(i)) @@ -386,10 +394,13 @@ with TestWithTempDir() as tmpdir: dir_iter.close() - expected_files_bytes = [(file.encode(), os.path.join(tmpdir, file).encode()) for file in expected_files] + expected_files_bytes = [(file.encode(), os.path.join(tmpdir, + file).encode()) + for file in expected_files] dir_iter_bytes = os.scandir(tmpdir.encode()) - collected_files_bytes = [(dir_entry.name, dir_entry.path) for dir_entry in dir_iter_bytes] + collected_files_bytes = [(dir_entry.name, dir_entry.path) + for dir_entry in dir_iter_bytes] assert set(collected_files_bytes) == set(expected_files_bytes) @@ -399,7 +410,8 @@ with TestWithTempDir() as tmpdir: assert set(collected_files) == set(expected_files) collected_files = os.listdir(tmpdir.encode()) - assert set(collected_files) == set([file.encode() for file in expected_files ]) + assert set(collected_files) == set( + [file.encode() for file in expected_files]) with TestWithTempCurrentDir(): os.chdir(tmpdir) @@ -414,5 +426,3 @@ if "win" not in sys.platform: for arg in [None, 1, 1.0, TabError]: assert_raises(TypeError, os.system, arg) - - From 12dee0d55a6816130d9199bcae7e144b4075ff68 Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Thu, 16 Jul 2020 22:07:20 +0200 Subject: [PATCH 082/209] Adding base snippets tests for os.chown --- tests/snippets/stdlib_os.py | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index b746c6c8b..ee762d92e 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -262,6 +262,71 @@ with TestWithTempDir() as tmpdir: os.chmod(fname, 0o666) assert oct(os.stat(fname).st_mode) == '0o100666' +# os.chown + if os.name != "nt": + # setup + root_in_posix = False + if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + try: + import pwd + all_users = [u.pw_uid for u in pwd.getpwall()] + except (ImportError, AttributeError): + all_users = [] + + fname1 = os.path.join(tmpdir, FILE_NAME) + fname2 = os.path.join(tmpdir, FILE_NAME2) + fd = os.open(fname2, os.O_RDONLY) + + # test chown without root permissions + if not root_in_posix and len(all_users) > 1: + uid_1, uid_2 = all_users[:2] + gid = os.stat(fname1).st_gid + assert_raises(PermissionError, + lambda: os.chown(fname1, uid_1, gid)) + assert_raises(PermissionError, + lambda: os.chown(fname1, uid_2, gid)) + + # test chown with root perm and file name + if root_in_posix and len(all_users) > 1: + uid_1, uid_2 = all_users[:2] + gid = os.stat(fname1).st_gid + os.chown(fname1, uid_1, gid) + uid = os.stat(fname1).st_uid + assert uid == uid_1 + os.chown(fname1, uid_2, gid) + uid = os.stat(fname1).st_uid + assert uid == uid_2 + + # test chown with root perm and file descriptor + if root_in_posix and len(all_users) > 1: + uid_1, uid_2 = all_users[:2] + gid = os.stat(fd).st_gid + os.chown(fd, uid_1, gid) + uid = os.stat(fd).st_uid + assert uid == uid_1 + os.chown(fd, uid_2, gid) + uid = os.stat(fd).st_uid + assert uid == uid_2 + + # test gid change + if hasattr(os, 'getgroups'): + groups = os.getgroups() + if len(groups) > 1: + gid_1, gid_2 = groups[:2] + uid = os.stat(fname1).st_uid + + os.chown(fname1, uid, gid_1) + gid = os.stat(fname1).st_gid + assert gid == gid_1 + + os.chown(fname1, uid, gid_2) + gid = os.stat(fname1).st_gid + assert gid == gid_2 + + # teardown + os.close(fd) + # os.path assert os.path.exists(fname) is True assert os.path.exists("NO_SUCH_FILE") is False From 792c40badc3eb56fdc9ceee2582dd3d6b56aee4f Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Fri, 24 Jul 2020 16:38:07 +0200 Subject: [PATCH 083/209] adding lchown function as a wrapper around chown --- vm/src/stdlib/os.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 3b894cc11..68c54d2b3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1088,6 +1088,20 @@ fn os_chown( .map_err(|err| convert_nix_error(vm, err)) } +#[cfg(all(unix, not(target_os = "redox")))] +fn os_lchown(path: PyPathLike, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + os_chown( + path, + uid, + gid, + DirFd { dir_fd: None }, + FollowSymlinks { + follow_symlinks: false, + }, + vm, + ) +} + #[cfg(unix)] fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { use nix::fcntl::fcntl; @@ -2131,6 +2145,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { #[cfg(not(target_os = "redox"))] SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), SupportFunc::new(vm, "chown", os_chown, None, Some(true), Some(true)), + SupportFunc::new(vm, "lchown", os_lchown, None, None, None), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), ]); let supports_fd = PySet::default().into_ref(vm); @@ -2280,6 +2295,7 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { #[cfg(not(target_os = "redox"))] extend_module!(vm, module, { "chown" => ctx.new_function(os_chown), + "lchown" => ctx.new_function(os_lchown), "chroot" => ctx.new_function(os_chroot), "getsid" => ctx.new_function(os_getsid), "setsid" => ctx.new_function(os_setsid), From deb5b08112ea3a1b3653c44fab5b966f4c74aac2 Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Fri, 24 Jul 2020 17:14:33 +0200 Subject: [PATCH 084/209] add support for passing fd as path argument in chown --- vm/src/stdlib/os.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 68c54d2b3..fd69209fe 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1048,7 +1048,7 @@ fn os_chroot(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { // As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) #[cfg(all(unix, not(target_os = "redox")))] fn os_chown( - path: PyPathLike, + path: Either, uid: PyIntRef, gid: PyIntRef, dir_fd: DirFd, @@ -1084,14 +1084,23 @@ fn os_chown( Some(int_ref) => Some(i32::try_from_object(&vm, int_ref.as_object().clone())?), None => None, }; - nix::unistd::fchownat(dir_fd, path.path.as_os_str(), uid, gid, flag) - .map_err(|err| convert_nix_error(vm, err)) + + match path { + Either::A(p) => nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) + .map_err(|err| convert_nix_error(vm, err)), + Either::B(fd) => { + let path = fs::read_link(format!("/proc/self/fd/{}", fd)) + .map_err(|_| vm.new_os_error(String::from("Cannot find path for specified fd")))?; + nix::unistd::fchownat(dir_fd, &path, uid, gid, flag) + .map_err(|err| convert_nix_error(vm, err)) + } + } } #[cfg(all(unix, not(target_os = "redox")))] fn os_lchown(path: PyPathLike, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { os_chown( - path, + Either::A(path), uid, gid, DirFd { dir_fd: None }, @@ -2144,7 +2153,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "chmod", os_chmod, Some(false), Some(false), Some(false)), #[cfg(not(target_os = "redox"))] SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), - SupportFunc::new(vm, "chown", os_chown, None, Some(true), Some(true)), + SupportFunc::new(vm, "chown", os_chown, Some(true), Some(true), Some(true)), SupportFunc::new(vm, "lchown", os_lchown, None, None, None), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), ]); From 428bf1e0a19b64fc3eaea2bf26191349d8f27d2f Mon Sep 17 00:00:00 2001 From: Depasse Arthur Date: Fri, 24 Jul 2020 17:18:12 +0200 Subject: [PATCH 085/209] adding fchown function as a wrapper around chown --- vm/src/stdlib/os.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index fd69209fe..6b855a53a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1111,6 +1111,20 @@ fn os_lchown(path: PyPathLike, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine ) } +#[cfg(all(unix, not(target_os = "redox")))] +fn os_fchown(fd: i64, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + os_chown( + Either::B(fd), + uid, + gid, + DirFd { dir_fd: None }, + FollowSymlinks { + follow_symlinks: true, + }, + vm, + ) +} + #[cfg(unix)] fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { use nix::fcntl::fcntl; @@ -2155,6 +2169,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), SupportFunc::new(vm, "chown", os_chown, Some(true), Some(true), Some(true)), SupportFunc::new(vm, "lchown", os_lchown, None, None, None), + SupportFunc::new(vm, "fchown", os_fchown, Some(true), None, Some(true)), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), ]); let supports_fd = PySet::default().into_ref(vm); @@ -2305,6 +2320,7 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { extend_module!(vm, module, { "chown" => ctx.new_function(os_chown), "lchown" => ctx.new_function(os_lchown), + "fchown" => ctx.new_function(os_fchown), "chroot" => ctx.new_function(os_chroot), "getsid" => ctx.new_function(os_getsid), "setsid" => ctx.new_function(os_setsid), From 447ddd1b09507ab48f3f234c15074807e4bf29a5 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Sat, 25 Jul 2020 00:25:08 +0000 Subject: [PATCH 086/209] Added newline for spacing Accidentally removed newline for spacing in documentation; re added it. --- vm/src/sysmodule.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 3577bd83b..a390f097a 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -349,6 +349,7 @@ thread_info -- a struct sequence with information about the thread implementatio version -- the version of this interpreter as a string version_info -- version information as a named tuple _base_executable -- __PYVENV_LAUNCHER__ enviroment variable if defined, else sys.executable. + __stdin__ -- the original stdin; don't touch! __stdout__ -- the original stdout; don't touch! __stderr__ -- the original stderr; don't touch! From 6735ac590285426da0fa5c4b1e618f758de40f68 Mon Sep 17 00:00:00 2001 From: hatee-hatee-hatee-ho <51845572+hatee-hatee-hatee-ho@users.noreply.github.com> Date: Sun, 26 Jul 2020 00:24:53 +0900 Subject: [PATCH 087/209] implemented __repr__ method in impl_pyclass_sequence * implemented sys.version_info.__repr__ * support 'module' optional parameter for 'pyclass sequence' attribute. * Separated module name and class name of 'pystruct_sequence' attribute parameter --- derive/src/pyclass.rs | 31 +++++++++++++++++++++++++++++-- derive/src/util.rs | 35 +++++++++++++++++++++++++++++++++++ vm/src/pyhash.rs | 2 +- vm/src/stdlib/os.rs | 6 +++--- vm/src/stdlib/pwd.rs | 2 +- vm/src/stdlib/time_module.rs | 2 +- vm/src/version.rs | 2 +- 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 11af6cdcd..bbcec3a3f 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,5 +1,8 @@ use super::Diagnostic; -use crate::util::{def_to_name, path_eq, strip_prefix, ItemIdent, ItemMeta}; +use crate::util::{ + def_to_name, module_class_name, optional_attribute_arg, path_eq, strip_prefix, ItemIdent, + ItemMeta, +}; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, quote_spanned, ToTokens}; use std::collections::{HashMap, HashSet}; @@ -478,7 +481,9 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result Result, vm: &rustpython_vm::VirtualMachine) -> ::rustpython_vm::pyobject::PyResult { + let s = if let Some(_guard) = rustpython_vm::vm::ReprGuard::enter(zelf.as_object()) { + let field_names = vec![#(stringify!(#field_names)),*]; + let mut fields = Vec::new(); + for (value, field_name) in zelf.as_slice().iter().zip(field_names.iter()) { + let s = vm.to_repr(value)?; + fields.push(format!("{}: {}", field_name, s)); + } + + if fields.len() == 1 { + format!("{}({},)", #py_class_name, fields[0]) + } else { + format!("{}({})", #py_class_name, fields.join(", ")) + } + } else { + concat!(#class_name, "(...)").to_string() + }; + Ok(s) + } } + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { fn impl_extend_class( ctx: &::rustpython_vm::pyobject::PyContext, class: &::rustpython_vm::obj::objtype::PyClassRef, ) { #(#properties)* + class.set_str_attr("__repr__", ctx.new_method(Self::repr)); } fn make_class( diff --git a/derive/src/util.rs b/derive/src/util.rs index deb2ef046..70f56abb9 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -32,6 +32,41 @@ pub fn def_to_name( Ok(name.unwrap_or_else(|| ident.to_string())) } +pub fn optional_attribute_arg( + attr_name: &'static str, + arg_name: &'static str, + attr: AttributeArgs, +) -> Result, Diagnostic> { + let mut arg_value = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if path_eq(&name_value.path, arg_name) { + if let Lit::Str(lit) = name_value.lit { + arg_value = Some(lit.value()); + } else { + bail_span!( + name_value.lit, + "#[{}({} = ...)] must be a string", + attr_name, + arg_name + ); + } + } + } + } + } + Ok(arg_value) +} + +pub fn module_class_name(mod_name: Option, class_name: &str) -> String { + if let Some(mod_name) = mod_name { + format!("{}.{}", mod_name, class_name) + } else { + class_name.into() + } +} + pub fn strip_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { if s.starts_with(prefix) { Some(&s[prefix.len()..]) diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs index 60a6d132e..598bde2a0 100644 --- a/vm/src/pyhash.rs +++ b/vm/src/pyhash.rs @@ -104,7 +104,7 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { } } -#[pystruct_sequence(name = "sys.hash_info")] +#[pystruct_sequence(module = "sys", name = "hash_info")] #[derive(Debug)] pub(crate) struct PyHashInfo { width: usize, diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6b855a53a..a372ecc75 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -801,7 +801,7 @@ fn os_scandir(path: OptionalArg, vm: &VirtualMachine) -> PyResult { } } -#[pystruct_sequence(name = "os.stat_result")] +#[pystruct_sequence(module = "os", name = "stat_result")] #[derive(Debug)] struct StatResult { st_mode: u32, @@ -1453,7 +1453,7 @@ fn os_umask(mask: ModeT, _vm: &VirtualMachine) -> PyResult { Ok(ret_mask) } -#[pystruct_sequence(name = "os.uname_result")] +#[pystruct_sequence(module = "os", name = "uname_result")] #[derive(Debug)] #[cfg(unix)] struct UnameResult { @@ -2010,7 +2010,7 @@ fn os_strerror(e: i32) -> String { .into_owned() } -#[pystruct_sequence(name = "os.terminal_size")] +#[pystruct_sequence(module = "os", name = "terminal_size")] #[allow(dead_code)] struct PyTerminalSize { columns: usize, diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index a44870363..24c4bd5a7 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -8,7 +8,7 @@ use std::ptr::NonNull; use nix::unistd::{self, User}; -#[pystruct_sequence(name = "pwd.struct_passwd")] +#[pystruct_sequence(module = "pwd", name = "struct_passwd")] struct Passwd { pw_name: String, pw_passwd: String, diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index 0bf52afa2..41bc17b91 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -170,7 +170,7 @@ fn time_strptime( Ok(PyStructTime::new(vm, instant, -1).into_obj(vm)) } -#[pystruct_sequence(name = "time.struct_time")] +#[pystruct_sequence(module = "time", name = "struct_time")] #[allow(dead_code)] struct PyStructTime { tm_year: PyObjectRef, diff --git a/vm/src/version.rs b/vm/src/version.rs index f8aa7bbda..f9dd90207 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -7,7 +7,7 @@ const MICRO: usize = 0; const RELEASELEVEL: &str = "alpha"; const SERIAL: usize = 0; -#[pystruct_sequence(name = "version_info")] +#[pystruct_sequence(module = "sys", name = "version_info")] #[derive(Default, Debug)] pub struct VersionInfo { major: usize, From 1098efa00abd513a7661b0bd2ba325b353c8ccfd Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Sat, 25 Jul 2020 11:35:19 -0700 Subject: [PATCH 088/209] Fixed bug where sys.executable would return a relative path instead of an absolute path --- vm/src/sysmodule.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a390f097a..a68354396 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -28,11 +28,12 @@ fn argv(vm: &VirtualMachine) -> PyObjectRef { } fn executable(ctx: &PyContext) -> PyObjectRef { - if let Some(arg) = env::args().next() { - ctx.new_str(arg) - } else { - ctx.none() + if let Ok(path) = env::current_exe() { + if let Some(path) = path.to_str() { + return ctx.new_str(path); + } } + ctx.none() } fn _base_executable(ctx: &PyContext) -> PyObjectRef { From 0251f3e9a2683d3780b950c7b43b5ba1a1b10b77 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 6 Jul 2020 00:46:55 -0500 Subject: [PATCH 089/209] Move pyhash and some float ops to rustpython-common --- Cargo.lock | 13 ++ Cargo.toml | 2 +- common/Cargo.toml | 15 ++ common/src/float_ops.rs | 194 +++++++++++++++++++++ vm/src/pyhash.rs => common/src/hash.rs | 68 +++----- common/src/lib.rs | 4 + parser/Cargo.toml | 4 +- vm/Cargo.toml | 1 + vm/pylib-crate/Cargo.toml | 2 +- vm/src/builtins.rs | 4 +- vm/src/dictdatatype.rs | 10 +- vm/src/lib.rs | 1 - vm/src/obj/objbytearray.rs | 4 +- vm/src/obj/objbyteinner.rs | 10 +- vm/src/obj/objbytes.rs | 8 +- vm/src/obj/objcomplex.rs | 10 +- vm/src/obj/objfloat.rs | 230 +++---------------------- vm/src/obj/objint.rs | 6 +- vm/src/obj/objobject.rs | 5 +- vm/src/obj/objrange.rs | 9 +- vm/src/obj/objset.rs | 10 +- vm/src/obj/objstr.rs | 8 +- vm/src/obj/objtuple.rs | 8 +- vm/src/obj/objweakref.rs | 2 +- vm/src/pyobject.rs | 14 ++ vm/src/stdlib/math.rs | 3 +- vm/src/sysmodule.rs | 30 +++- vm/src/vm.rs | 3 +- 28 files changed, 369 insertions(+), 309 deletions(-) create mode 100644 common/Cargo.toml create mode 100644 common/src/float_ops.rs rename vm/src/pyhash.rs => common/src/hash.rs (59%) create mode 100644 common/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 385b861fa..809e7601e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,6 +1521,18 @@ dependencies = [ "serde", ] +[[package]] +name = "rustpython-common" +version = "0.0.0" +dependencies = [ + "hexf-parse", + "lexical", + "num-bigint", + "num-traits", + "parking_lot", + "rand 0.7.3", +] + [[package]] name = "rustpython-compiler" version = "0.1.2" @@ -1631,6 +1643,7 @@ dependencies = [ "result-like", "rustc_version_runtime", "rustpython-bytecode", + "rustpython-common", "rustpython-compiler", "rustpython-derive", "rustpython-parser", diff --git a/Cargo.toml b/Cargo.toml index 78a179ade..3714ba2de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace] -members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate"] +members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate", "common"] [[bench]] name = "bench" diff --git a/common/Cargo.toml b/common/Cargo.toml new file mode 100644 index 000000000..f5f3c5658 --- /dev/null +++ b/common/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rustpython-common" +version = "0.0.0" +authors = ["RustPython Team"] +edition = "2018" + +[dependencies] +parking_lot = "0.11.0" +num-traits = "0.2" +num-bigint = "0.3" +lexical = "4" +hexf-parse = "0.1.0" + +[dev-dependencies] +rand = "0.7.3" diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs new file mode 100644 index 000000000..34f2c4e6a --- /dev/null +++ b/common/src/float_ops.rs @@ -0,0 +1,194 @@ +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{Float, Signed, ToPrimitive, Zero}; + +pub fn ufrexp(value: f64) -> (f64, i32) { + if 0.0 == value { + (0.0, 0i32) + } else { + let bits = value.to_bits(); + let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; + let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52); + (f64::from_bits(mantissa_bits), exponent) + } +} + +pub fn eq_int(value: f64, other: &BigInt) -> bool { + if let (Some(self_int), Some(other_float)) = (value.to_bigint(), other.to_f64()) { + value == other_float && self_int == *other + } else { + false + } +} + +pub fn lt_int(value: f64, other_int: &BigInt) -> bool { + match (value.to_bigint(), other_int.to_f64()) { + (Some(self_int), Some(other_float)) => value < other_float || self_int < *other_int, + // finite float, other_int too big for float, + // the result depends only on other_int’s sign + (Some(_), None) => other_int.is_positive(), + // infinite float must be bigger or lower than any int, depending on its sign + _ if value.is_infinite() => value.is_sign_negative(), + // NaN, always false + _ => false, + } +} + +pub fn gt_int(value: f64, other_int: &BigInt) -> bool { + match (value.to_bigint(), other_int.to_f64()) { + (Some(self_int), Some(other_float)) => value > other_float || self_int > *other_int, + // finite float, other_int too big for float, + // the result depends only on other_int’s sign + (Some(_), None) => other_int.is_negative(), + // infinite float must be bigger or lower than any int, depending on its sign + _ if value.is_infinite() => value.is_sign_positive(), + // NaN, always false + _ => false, + } +} + +pub fn parse_str(literal: &str) -> Option { + if literal.starts_with('_') || literal.ends_with('_') { + return None; + } + + let mut buf = String::with_capacity(literal.len()); + let mut last_tok: Option = None; + for c in literal.chars() { + if !(c.is_ascii_alphanumeric() || c == '_' || c == '+' || c == '-' || c == '.') { + return None; + } + + if !c.is_ascii_alphanumeric() { + if let Some(l) = last_tok { + if !l.is_ascii_alphanumeric() && !(c == '.' && (l == '-' || l == '+')) { + return None; + } + } + } + + if c != '_' { + buf.push(c); + } + last_tok = Some(c); + } + + if let Ok(f) = lexical::parse(buf.as_str()) { + Some(f) + } else { + None + } +} + +pub fn is_integer(v: f64) -> bool { + (v - v.round()).abs() < std::f64::EPSILON +} + +pub fn to_string(value: f64) -> String { + let lit = format!("{:e}", value); + if let Some(position) = lit.find('e') { + let significand = &lit[..position]; + let exponent = &lit[position + 1..]; + let exponent = exponent.parse::().unwrap(); + if exponent < 16 && exponent > -5 { + if is_integer(value) { + format!("{:.1?}", value) + } else { + value.to_string() + } + } else { + format!("{}e{:+#03}", significand, exponent) + } + } else { + value.to_string() + } +} + +pub fn from_hex(s: &str) -> Option { + if let Ok(f) = hexf_parse::parse_hexf64(s, false) { + return Some(f); + } + match s.to_ascii_lowercase().as_str() { + "nan" => Some(f64::NAN), + "+nan" => Some(f64::NAN), + "-nan" => Some(f64::NAN), + "inf" => Some(f64::INFINITY), + "infinity" => Some(f64::INFINITY), + "+inf" => Some(f64::INFINITY), + "+infinity" => Some(f64::INFINITY), + "-inf" => Some(f64::NEG_INFINITY), + "-infinity" => Some(f64::NEG_INFINITY), + value => { + let mut hex = String::with_capacity(value.len()); + let has_0x = value.contains("0x"); + let has_p = value.contains('p'); + let has_dot = value.contains('.'); + let mut start = 0; + + if !has_0x && value.starts_with('-') { + hex.push_str("-0x"); + start += 1; + } else if !has_0x { + hex.push_str("0x"); + if value.starts_with('+') { + start += 1; + } + } + + for (index, ch) in value.chars().enumerate() { + if ch == 'p' && has_dot { + hex.push_str("p"); + } else if ch == 'p' && !has_dot { + hex.push_str(".p"); + } else if index >= start { + hex.push(ch); + } + } + + if !has_p && has_dot { + hex.push_str("p0"); + } else if !has_p && !has_dot { + hex.push_str(".p0") + } + + hexf_parse::parse_hexf64(hex.as_str(), false).ok() + } + } +} + +pub fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt), + value if value.is_infinite() => format!("{}inf", sign_fmt), + value if value.is_nan() => "nan".to_owned(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}0x{:x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::thread_rng().gen::<[u64; 1]>(); + let f = f64::from_bits(bytes[0]); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } +} diff --git a/vm/src/pyhash.rs b/common/src/hash.rs similarity index 59% rename from vm/src/pyhash.rs rename to common/src/hash.rs index 598bde2a0..12b858e5b 100644 --- a/vm/src/pyhash.rs +++ b/common/src/hash.rs @@ -1,12 +1,8 @@ use num_bigint::BigInt; use num_traits::ToPrimitive; +use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use crate::obj::objfloat; -use crate::pyobject::PyObjectRef; -use crate::pyobject::PyResult; -use crate::vm::VirtualMachine; - pub type PyHash = i64; pub type PyUHash = u64; @@ -18,6 +14,12 @@ pub const MODULUS: PyUHash = (1 << BITS) - 1; pub const INF: PyHash = 314_159; pub const NAN: PyHash = 0; pub const IMAG: PyHash = MULTIPLIER; +pub const ALGO: &str = "siphasher13"; +pub const HASH_BITS: usize = std::mem::size_of::() * 8; +// internally DefaultHasher uses 2 u64s as the seed, but +// that's not guaranteed to be consistent across Rust releases +// TODO: use something like the siphasher crate as our hash algorithm +pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8; // pub const CUTOFF: usize = 7; @@ -35,7 +37,7 @@ pub fn hash_float(value: f64) -> PyHash { }; } - let frexp = objfloat::ufrexp(value); + let frexp = super::float_ops::ufrexp(value); // process 28 bits at a time; this should work well both for binary // and hexadecimal floating point. @@ -67,30 +69,32 @@ pub fn hash_float(value: f64) -> PyHash { } pub fn hash_value(data: &T) -> PyHash { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); + let mut hasher = DefaultHasher::new(); data.hash(&mut hasher); hasher.finish() as PyHash } -pub fn hash_iter<'a, I: std::iter::Iterator>( - iter: I, - vm: &VirtualMachine, -) -> PyResult { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); +pub fn hash_iter<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result +where + I: IntoIterator, + F: Fn(&'a T) -> Result, +{ + let mut hasher = DefaultHasher::new(); for element in iter { - let item_hash = vm._hash(&element)?; + let item_hash = hashf(element)?; item_hash.hash(&mut hasher); } Ok(hasher.finish() as PyHash) } -pub fn hash_iter_unordered<'a, I: std::iter::Iterator>( - iter: I, - vm: &VirtualMachine, -) -> PyResult { +pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result +where + I: IntoIterator, + F: Fn(&'a T) -> Result, +{ let mut hash: PyHash = 0; for element in iter { - let item_hash = vm._hash(element)?; + let item_hash = hashf(element)?; // xor is commutative and hash should be independent of order hash ^= item_hash; } @@ -103,31 +107,3 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { None => (value % MODULUS).to_i64().unwrap(), } } - -#[pystruct_sequence(module = "sys", name = "hash_info")] -#[derive(Debug)] -pub(crate) struct PyHashInfo { - width: usize, - modulus: PyUHash, - inf: PyHash, - nan: PyHash, - imag: PyHash, - algorithm: &'static str, - hash_bits: usize, - seed_bits: usize, -} -impl PyHashInfo { - pub const INFO: Self = PyHashInfo { - width: BITS, - modulus: MODULUS, - inf: INF, - nan: NAN, - imag: IMAG, - algorithm: "siphash13", - hash_bits: std::mem::size_of::() * 8, - // internally hash_map::DefaultHasher uses 2 u64s as the seed, but - // that's not guaranteed to be consistent across Rust releases - // TODO: use something like the siphasher crate as our hash algorithm - seed_bits: std::mem::size_of::() * 2 * 8, - }; -} diff --git a/common/src/lib.rs b/common/src/lib.rs new file mode 100644 index 000000000..3a961c6c1 --- /dev/null +++ b/common/src/lib.rs @@ -0,0 +1,4 @@ +//! A crate to hold types and functions common to all rustpython components. + +pub mod float_ops; +pub mod hash; diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 8d3aa4b09..da88a7586 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -12,8 +12,8 @@ edition = "2018" lalrpop="0.17" [dependencies] -lalrpop-util="0.17" -log="0.4.1" +lalrpop-util = "0.17" +log = "0.4.1" num-bigint = "0.3" num-traits = "0.2" unic-emoji-char = "0.9" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 66c08aa9b..68eef7ca0 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -91,6 +91,7 @@ unic-ucd-ident = "0.9" flame = { version = "0.2", optional = true } flamer = { version = "0.3", optional = true } +rustpython-common = { path = "../common" } [target.'cfg(unix)'.dependencies] exitcode = "1.1.2" diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index b49379f27..21e41087e 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustpython-pylib" version = "0.1.0" -authors = ["Noah <33094578+coolreader18@users.noreply.github.com>"] +authors = ["RustPython Team"] edition = "2018" [dependencies] diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index c6a073ebe..896fa8b9d 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -27,7 +27,6 @@ mod decl { use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; - use crate::pyhash; use crate::pyobject::{ Either, IdProtocol, ItemProtocol, PyCallable, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -37,6 +36,7 @@ mod decl { #[cfg(feature = "rustpython-parser")] use crate::stdlib::ast; use crate::vm::VirtualMachine; + use rustpython_common::hash::PyHash; #[pyfunction] fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -329,7 +329,7 @@ mod decl { } #[pyfunction] - fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { vm._hash(&obj) } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 7ff9cb7db..7a893cfbc 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,9 +1,9 @@ use crate::obj::objstr::{PyString, PyStringRef}; -use crate::pyhash; use crate::pyobject::{IdProtocol, IntoPyObject, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; use num_bigint::ToBigInt; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +use rustpython_common::hash; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 @@ -13,9 +13,9 @@ use std::hash::{Hash, Hasher}; use std::mem::size_of; /// hash value of an object returned by __hash__ -type HashValue = pyhash::PyHash; +type HashValue = hash::PyHash; /// index calculated by resolving collision -type HashIndex = pyhash::PyHash; +type HashIndex = hash::PyHash; /// entry index mapped in indices type EntryIndex = usize; @@ -463,8 +463,8 @@ impl DictKey for &PyStringRef { impl DictKey for &str { fn do_hash(self, _vm: &VirtualMachine) -> PyResult { // follow a similar route as the hashing of PyStringRef - let raw_hash = pyhash::hash_value(&self.to_owned()).to_bigint().unwrap(); - let raw_hash = pyhash::hash_bigint(&raw_hash); + let raw_hash = hash::hash_value(&self.to_owned()).to_bigint().unwrap(); + let raw_hash = hash::hash_bigint(&raw_hash); let mut hasher = DefaultHasher::new(); raw_hash.hash(&mut hasher); Ok(hasher.finish() as HashValue) diff --git a/vm/src/lib.rs b/vm/src/lib.rs index e775ce25d..c0f923c18 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -68,7 +68,6 @@ pub mod function; pub mod import; pub mod obj; pub mod py_serde; -mod pyhash; pub mod pyobject; pub mod readline; pub mod scope; diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index da7a47551..22e35e5da 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -98,8 +98,8 @@ impl PyByteArray { } #[pymethod(name = "__repr__")] - fn repr(&self) -> PyResult { - Ok(format!("bytearray(b'{}')", self.borrow_value().repr()?)) + fn repr(&self) -> String { + format!("bytearray(b'{}')", self.borrow_value().repr()) } #[pymethod(name = "__len__")] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b4ef38139..4e5428d29 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -14,11 +14,11 @@ use super::objslice::PySliceRef; use super::objstr::{self, PyString, PyStringRef}; use super::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; use crate::function::{OptionalArg, OptionalOption}; -use crate::pyhash; use crate::pyobject::{ Either, PyComparisonValue, PyIterable, PyObjectRef, PyResult, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::hash; #[derive(Debug, Default, Clone)] pub struct PyByteInner { @@ -253,7 +253,7 @@ impl ByteInnerSplitlinesOptions { #[allow(clippy::len_without_is_empty)] impl PyByteInner { - pub fn repr(&self) -> PyResult { + pub fn repr(&self) -> String { let mut res = String::with_capacity(self.elements.len()); for i in self.elements.iter() { match i { @@ -266,7 +266,7 @@ impl PyByteInner { _ => res.push_str(&format!("\\x{:x}", i)), } } - Ok(res) + res } pub fn len(&self) -> usize { @@ -303,8 +303,8 @@ impl PyByteInner { self.cmp(other, |a, b| a < b, vm) } - pub fn hash(&self) -> pyhash::PyHash { - pyhash::hash_value(&self.elements) + pub fn hash(&self) -> hash::PyHash { + hash::hash_value(&self.elements) } pub fn add(&self, other: PyByteInner) -> Vec { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index ffeea4c05..529ae1674 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -14,7 +14,6 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; use crate::function::{OptionalArg, OptionalOption}; -use crate::pyhash; use crate::pyobject::{ Either, IntoPyObject, PyArithmaticValue::{self, *}, @@ -22,6 +21,7 @@ use crate::pyobject::{ TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::hash::PyHash; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ @@ -102,8 +102,8 @@ impl PyBytes { } #[pymethod(name = "__repr__")] - fn repr(&self, vm: &VirtualMachine) -> PyResult { - Ok(vm.new_str(format!("b'{}'", self.inner.repr()?))) + pub(crate) fn repr(&self) -> String { + format!("b'{}'", self.inner.repr()) } #[pymethod(name = "__len__")] @@ -133,7 +133,7 @@ impl PyBytes { } #[pymethod(name = "__hash__")] - fn hash(&self) -> pyhash::PyHash { + fn hash(&self) -> PyHash { self.inner.hash() } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 4b7f04de0..367db543b 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -8,11 +8,11 @@ use super::objint::{self, PyInt}; use super::objstr::PyString; use super::objtype::PyClassRef; use crate::function::OptionalArg; -use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::hash; /// Create a complex number from a real part and an optional imaginary part. /// @@ -267,10 +267,10 @@ impl PyComplex { } #[pymethod(name = "__hash__")] - fn hash(&self) -> pyhash::PyHash { - let re_hash = pyhash::hash_float(self.value.re); - let im_hash = pyhash::hash_float(self.value.im); - let ret = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(pyhash::IMAG); + fn hash(&self) -> hash::PyHash { + let re_hash = hash::hash_float(self.value.re); + let im_hash = hash::hash_float(self.value.im); + let ret = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(hash::IMAG); ret.0 } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 20ddc97c4..4cba5dc05 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -1,20 +1,19 @@ use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; -use num_traits::{float::Float, pow, sign::Signed, ToPrimitive, Zero}; +use num_traits::{pow, ToPrimitive, Zero}; use super::objbytes::PyBytes; use super::objint::{self, PyInt, PyIntRef}; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; -use crate::exceptions::PyBaseExceptionRef; use crate::format::FormatSpec; use crate::function::{OptionalArg, OptionalOption}; -use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyArithmaticValue::*, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::{float_ops, hash}; /// Convert a string or number to a floating point number, if possible. #[pyclass(name = "float")] @@ -129,32 +128,6 @@ fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { } } -fn inner_lt_int(value: f64, other_int: &BigInt) -> bool { - match (value.to_bigint(), other_int.to_f64()) { - (Some(self_int), Some(other_float)) => value < other_float || self_int < *other_int, - // finite float, other_int too big for float, - // the result depends only on other_int’s sign - (Some(_), None) => other_int.is_positive(), - // infinite float must be bigger or lower than any int, depending on its sign - _ if value.is_infinite() => value.is_sign_negative(), - // NaN, always false - _ => false, - } -} - -fn inner_gt_int(value: f64, other_int: &BigInt) -> bool { - match (value.to_bigint(), other_int.to_f64()) { - (Some(self_int), Some(other_float)) => value > other_float || self_int > *other_int, - // finite float, other_int too big for float, - // the result depends only on other_int’s sign - (Some(_), None) => other_int.is_negative(), - // infinite float must be bigger or lower than any int, depending on its sign - _ if value.is_infinite() => value.is_sign_positive(), - // NaN, always false - _ => false, - } -} - pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v1.is_zero() { let msg = format!("{} cannot be raised to a negative power", v1); @@ -164,14 +137,6 @@ pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { } } -fn int_eq(value: f64, other: &BigInt) -> bool { - if let (Some(self_int), Some(other_float)) = (value.to_bigint(), other.to_f64()) { - value == other_float && self_int == *other - } else { - false - } -} - #[pyimpl(flags(BASETYPE))] #[allow(clippy::trivially_copy_pass_by_ref)] impl PyFloat { @@ -221,7 +186,7 @@ impl PyFloat { #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyComparisonValue { - self.cmp(other, |a, b| a == b, int_eq, vm) + self.cmp(other, |a, b| a == b, float_ops::eq_int, vm) } #[pymethod(name = "__ne__")] @@ -231,7 +196,7 @@ impl PyFloat { #[pymethod(name = "__lt__")] fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyComparisonValue { - self.cmp(other, |a, b| a < b, inner_lt_int, vm) + self.cmp(other, |a, b| a < b, float_ops::lt_int, vm) } #[pymethod(name = "__le__")] @@ -243,7 +208,7 @@ impl PyFloat { if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { a <= b_float && a_int <= *b } else { - inner_lt_int(a, b) + float_ops::lt_int(a, b) } }, vm, @@ -252,7 +217,7 @@ impl PyFloat { #[pymethod(name = "__gt__")] fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyComparisonValue { - self.cmp(other, |a, b| a > b, inner_gt_int, vm) + self.cmp(other, |a, b| a > b, float_ops::gt_int, vm) } #[pymethod(name = "__ge__")] @@ -264,7 +229,7 @@ impl PyFloat { if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { a >= b_float && a_int >= *b } else { - inner_gt_int(a, b) + float_ops::gt_int(a, b) } }, vm, @@ -380,23 +345,7 @@ impl PyFloat { #[pymethod(name = "__repr__")] fn repr(&self) -> String { - let value = format!("{:e}", self.value); - if let Some(position) = value.find('e') { - let significand = &value[..position]; - let exponent = &value[position + 1..]; - let exponent = exponent.parse::().unwrap(); - if exponent < 16 && exponent > -5 { - if self.is_integer() { - format!("{:.1?}", self.value) - } else { - self.value.to_string() - } - } else { - format!("{}e{:+#03}", significand, exponent) - } - } else { - self.value.to_string() - } + float_ops::to_string(self.value) } #[pymethod(name = "__truediv__")] @@ -493,8 +442,8 @@ impl PyFloat { } #[pymethod(name = "__hash__")] - fn hash(&self) -> pyhash::PyHash { - pyhash::hash_float(self.value) + fn hash(&self) -> hash::PyHash { + hash::hash_float(self.value) } #[pyproperty] @@ -514,8 +463,7 @@ impl PyFloat { #[pymethod(name = "is_integer")] fn is_integer(&self) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON + float_ops::is_integer(self.value) } #[pymethod(name = "as_integer_ratio")] @@ -538,116 +486,33 @@ impl PyFloat { #[pymethod] fn fromhex(repr: PyStringRef, vm: &VirtualMachine) -> PyResult { - hexf_parse::parse_hexf64(repr.as_str().trim(), false).or_else(|_| { - match repr.as_str().to_lowercase().trim() { - "nan" => Ok(std::f64::NAN), - "+nan" => Ok(std::f64::NAN), - "-nan" => Ok(std::f64::NAN), - "inf" => Ok(std::f64::INFINITY), - "infinity" => Ok(std::f64::INFINITY), - "+inf" => Ok(std::f64::INFINITY), - "+infinity" => Ok(std::f64::INFINITY), - "-inf" => Ok(std::f64::NEG_INFINITY), - "-infinity" => Ok(std::f64::NEG_INFINITY), - value => { - let mut hex = String::new(); - let has_0x = value.contains("0x"); - let has_p = value.contains('p'); - let has_dot = value.contains('.'); - let mut start = 0; - - if !has_0x && value.starts_with('-') { - hex.push_str("-0x"); - start += 1; - } else if !has_0x { - hex.push_str("0x"); - if value.starts_with('+') { - start += 1; - } - } - - for (index, ch) in value.chars().enumerate() { - if ch == 'p' && has_dot { - hex.push_str("p"); - } else if ch == 'p' && !has_dot { - hex.push_str(".p"); - } else if index >= start { - hex.push(ch); - } - } - - if !has_p && has_dot { - hex.push_str("p0"); - } else if !has_p && !has_dot { - hex.push_str(".p0") - } - - hexf_parse::parse_hexf64(hex.as_str(), false).map_err(|_| { - vm.new_value_error("invalid hexadecimal floating-point string".to_owned()) - }) - } - } + float_ops::from_hex(repr.as_str().trim()).ok_or_else(|| { + vm.new_value_error("invalid hexadecimal floating-point string".to_owned()) }) } #[pymethod] fn hex(&self) -> String { - to_hex(self.value) + float_ops::to_hex(self.value) } } -fn str_to_float(vm: &VirtualMachine, literal: &str) -> PyResult { - if literal.starts_with('_') || literal.ends_with('_') { - return Err(invalid_convert(vm, literal)); - } - - let mut buf = String::with_capacity(literal.len()); - let mut last_tok: Option = None; - for c in literal.chars() { - if !(c.is_ascii_alphanumeric() || c == '_' || c == '+' || c == '-' || c == '.') { - return Err(invalid_convert(vm, literal)); - } - - if !c.is_ascii_alphanumeric() { - if let Some(l) = last_tok { - if !l.is_ascii_alphanumeric() && !(c == '.' && (l == '-' || l == '+')) { - return Err(invalid_convert(vm, literal)); - } - } - } - - if c != '_' { - buf.push(c); - } - last_tok = Some(c); - } - - if let Ok(f) = lexical::parse(buf.as_str()) { - Ok(f) - } else { - Err(invalid_convert(vm, literal)) - } -} - -fn invalid_convert(vm: &VirtualMachine, literal: &str) -> PyBaseExceptionRef { - vm.new_value_error(format!("could not convert string to float: '{}'", literal)) -} - fn to_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { let value = if let Some(float) = obj.payload_if_subclass::(vm) { float.value } else if let Some(int) = obj.payload_if_subclass::(vm) { objint::try_float(int.as_bigint(), vm)? } else if let Some(s) = obj.payload_if_subclass::(vm) { - str_to_float(vm, s.as_str().trim())? + float_ops::parse_str(s.as_str().trim()).ok_or_else(|| { + vm.new_value_error(format!("could not convert string to float: '{}'", s)) + })? } else if let Some(bytes) = obj.payload_if_subclass::(vm) { - match lexical::parse(bytes.get_value()) { - Ok(f) => f, - Err(_) => { - let arg_repr = vm.to_pystr(obj)?; - return Err(invalid_convert(vm, arg_repr.as_str())); - } - } + lexical::parse(bytes.get_value()).map_err(|_| { + vm.new_value_error(format!( + "could not convert string to float: '{}'", + bytes.repr() + )) + })? } else { let method = vm.get_method_or_type_error(obj.clone(), "__float__", || { format!( @@ -661,55 +526,6 @@ fn to_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { Ok(value) } -fn to_hex(value: f64) -> String { - let (mantissa, exponent, sign) = value.integer_decode(); - let sign_fmt = if sign < 0 { "-" } else { "" }; - match value { - value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt), - value if value.is_infinite() => format!("{}inf", sign_fmt), - value if value.is_nan() => "nan".to_owned(), - _ => { - const BITS: i16 = 52; - const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; - format!( - "{}0x{:x}.{:013x}p{:+}", - sign_fmt, - mantissa >> BITS, - mantissa & FRACT_MASK, - exponent + BITS - ) - } - } -} - -#[test] -fn test_to_hex() { - use rand::Rng; - for _ in 0..20000 { - let bytes = rand::thread_rng().gen::<[u64; 1]>(); - let f = f64::from_bits(bytes[0]); - if !f.is_finite() { - continue; - } - let hex = to_hex(f); - // println!("{} -> {}", f, hex); - let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); - // println!(" -> {}", roundtrip); - assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); - } -} - -pub fn ufrexp(value: f64) -> (f64, i32) { - if 0.0 == value { - (0.0, 0i32) - } else { - let bits = value.to_bits(); - let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; - let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52); - (f64::from_bits(mantissa_bits), exponent) - } -} - pub type PyFloatRef = PyRef; // Retrieve inner float value: diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 780b7ebd2..e011f23c0 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -15,13 +15,13 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype::{self, PyClassRef}; use crate::format::FormatSpec; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, PyArithmaticValue, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::stdlib::array::PyArray; use crate::vm::VirtualMachine; +use rustpython_common::hash; /// int(x=0) -> integer /// int(x, base=10) -> integer @@ -419,8 +419,8 @@ impl PyInt { } #[pymethod(name = "__hash__")] - pub fn hash(&self) -> pyhash::PyHash { - pyhash::hash_bigint(&self.value) + pub fn hash(&self) -> hash::PyHash { + hash::hash_bigint(&self.value) } #[pymethod(name = "__abs__")] diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c64fa6883..4c1149681 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -5,7 +5,6 @@ use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objtype::PyClass; -use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyArithmaticValue::*, PyAttributes, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -85,8 +84,8 @@ impl PyBaseObject { } #[pymethod(magic)] - fn hash(zelf: PyObjectRef) -> pyhash::PyHash { - zelf.get_id() as pyhash::PyHash + fn hash(zelf: PyObjectRef) -> rustpython_common::hash::PyHash { + zelf.get_id() as _ } #[pymethod(magic)] diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index ef93ee230..ac8668f9c 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -10,11 +10,12 @@ use super::objtuple::PyTuple; use super::objtype::PyClassRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyhash; use crate::pyobject::{ - PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + self, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::hash::PyHash; /// range(stop) -> range object /// range(start, stop[, step]) -> range object @@ -363,7 +364,7 @@ impl PyRange { } #[pymethod(name = "__hash__")] - fn hash(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + fn hash(zelf: PyRef, vm: &VirtualMachine) -> PyResult { let length = zelf.length(); let elements = if length.is_zero() { vec![vm.ctx.new_int(length), vm.get_none(), vm.get_none()] @@ -380,7 +381,7 @@ impl PyRange { zelf.step().into_object(), ] }; - pyhash::hash_iter(elements.iter(), vm) + pyobject::hash_iter(&elements, vm) } #[pyslot] diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 7577933e6..019f00b9a 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -7,12 +7,12 @@ use super::objlist::PyListIterator; use super::objtype::{self, PyClassRef}; use crate::dictdatatype; use crate::function::{Args, OptionalArg}; -use crate::pyhash; use crate::pyobject::{ - PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + self, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; +use rustpython_common::hash::PyHash; pub type SetContentType = dictdatatype::Dict<()>; @@ -329,8 +329,8 @@ impl PySetInner { Ok(()) } - fn hash(&self, vm: &VirtualMachine) -> PyResult { - pyhash::hash_iter_unordered(self.content.keys().iter(), vm) + fn hash(&self, vm: &VirtualMachine) -> PyResult { + pyobject::hash_iter_unordered(self.content.keys().iter(), vm) } } @@ -787,7 +787,7 @@ impl PyFrozenSet { } #[pymethod(name = "__hash__")] - fn hash(&self, vm: &VirtualMachine) -> PyResult { + fn hash(&self, vm: &VirtualMachine) -> PyResult { self.inner.hash(vm) } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 02acf5d04..dd905416c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -24,12 +24,12 @@ use super::pystr::{ }; use crate::format::{FormatParseError, FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; -use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; +use rustpython_common::hash; /// str(object='') -> str /// str(bytes_or_buffer[, encoding[, errors]]) -> str @@ -45,7 +45,7 @@ use crate::vm::VirtualMachine; #[derive(Debug)] pub struct PyString { value: String, - hash: AtomicCell>, + hash: AtomicCell>, len: AtomicCell>, } @@ -280,9 +280,9 @@ impl PyString { } #[pymethod(name = "__hash__")] - pub(crate) fn hash(&self) -> pyhash::PyHash { + pub(crate) fn hash(&self) -> hash::PyHash { self.hash.load().unwrap_or_else(|| { - let hash = pyhash::hash_value(&self.value); + let hash = hash::hash_value(&self.value); self.hash.store(Some(hash)); hash }) diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 43570b386..92e591f46 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -5,14 +5,14 @@ use super::objiter; use super::objsequence::get_item; use super::objtype::PyClassRef; use crate::function::OptionalArg; -use crate::pyhash; use crate::pyobject::{ - IntoPyObject, + self, IntoPyObject, PyArithmaticValue::{self, *}, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::sequence::{self, SimpleSeq}; use crate::vm::{ReprGuard, VirtualMachine}; +use rustpython_common::hash::PyHash; /// tuple() -> empty tuple /// tuple(iterable) -> tuple initialized from iterable's items @@ -156,8 +156,8 @@ impl PyTuple { } #[pymethod(name = "__hash__")] - fn hash(&self, vm: &VirtualMachine) -> PyResult { - pyhash::hash_iter(self.elements.iter(), vm) + fn hash(&self, vm: &VirtualMachine) -> PyResult { + pyobject::hash_iter(self.elements.iter(), vm) } #[pymethod(name = "__iter__")] diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index d50fc378a..d7450f42b 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -6,8 +6,8 @@ use crate::pyobject::{ }; use crate::slots::SlotCall; use crate::vm::VirtualMachine; +use rustpython_common::hash::PyHash; -use crate::pyhash::PyHash; use crossbeam_utils::atomic::AtomicCell; use std::sync::{Arc, Weak}; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index cfd2df03e..c32f31ad0 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1452,6 +1452,20 @@ impl TryFromObject for PySequence { } } +pub fn hash_iter<'a, I: IntoIterator>( + iter: I, + vm: &VirtualMachine, +) -> PyResult { + rustpython_common::hash::hash_iter(iter, |obj| vm._hash(obj)) +} + +pub fn hash_iter_unordered<'a, I: IntoIterator>( + iter: I, + vm: &VirtualMachine, +) -> PyResult { + rustpython_common::hash::hash_iter_unordered(iter, |obj| vm._hash(obj)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index b0b957e02..4bda69436 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -15,6 +15,7 @@ use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objtype; use crate::pyobject::{Either, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use rustpython_common::float_ops; #[cfg(not(target_arch = "wasm32"))] use libc::c_double; @@ -253,7 +254,7 @@ fn math_floor(value: PyObjectRef, vm: &VirtualMachine) -> PyResult { fn math_frexp(value: IntoPyFloat) -> (f64, i32) { let value = value.to_f64(); if value.is_finite() { - let (m, e) = objfloat::ufrexp(value); + let (m, e) = float_ops::ufrexp(value); (m * value.signum(), e) } else { (value, 0) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a390f097a..ce0c0d6c5 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -5,12 +5,12 @@ use crate::builtins; use crate::frame::FrameRef; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyhash::PyHashInfo; use crate::pyobject::{ IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult, TypeProtocol, }; use crate::version; use crate::vm::{PySettings, VirtualMachine}; +use rustpython_common::hash::{PyHash, PyUHash}; /* * The magic sys module. @@ -256,6 +256,34 @@ pub fn sysconfigdata_name() -> String { format!("_sysconfigdata_{}_{}_{}", ABIFLAGS, PLATFORM, MULTIARCH) } +#[pystruct_sequence(module = "sys", name = "hash_info")] +#[derive(Debug)] +struct PyHashInfo { + width: usize, + modulus: PyUHash, + inf: PyHash, + nan: PyHash, + imag: PyHash, + algorithm: &'static str, + hash_bits: usize, + seed_bits: usize, +} +impl PyHashInfo { + pub const INFO: Self = { + use rustpython_common::hash::*; + PyHashInfo { + width: BITS, + modulus: MODULUS, + inf: INF, + nan: NAN, + imag: IMAG, + algorithm: ALGO, + hash_bits: HASH_BITS, + seed_bits: SEED_BITS, + } + }; +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index c629d8f23..5d5373acb 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -40,7 +40,6 @@ use crate::obj::objobject; use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{self, PyClassRef}; -use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, @@ -1406,7 +1405,7 @@ impl VirtualMachine { }) } - pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { + pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { let hash_obj = self.call_method(obj, "__hash__", vec![])?; if let Some(hash_value) = hash_obj.payload_if_subclass::(self) { Ok(hash_value.hash()) From 16ff02201435c5db87c9c8eaeae6ba442605540c Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 16 Jul 2020 15:57:23 -0500 Subject: [PATCH 090/209] Add the rustpython_common::{cell, rc} modules --- common/src/cell.rs | 72 ++++++++++++++++++++++++++++++++++++++ common/src/lib.rs | 2 ++ common/src/rc.rs | 6 ++++ vm/src/lib.rs | 1 + vm/src/obj/objtype.rs | 16 ++++----- vm/src/obj/objweakref.rs | 4 +-- vm/src/pyobject.rs | 40 ++++++++++----------- vm/src/stdlib/itertools.rs | 10 +++--- vm/src/stdlib/weakref.rs | 4 +-- vm/src/sysmodule.rs | 4 +-- vm/src/types.rs | 66 ++++++++++++++++++---------------- vm/src/vm.rs | 12 +++---- 12 files changed, 161 insertions(+), 76 deletions(-) create mode 100644 common/src/cell.rs create mode 100644 common/src/rc.rs diff --git a/common/src/cell.rs b/common/src/cell.rs new file mode 100644 index 000000000..c9f47f11e --- /dev/null +++ b/common/src/cell.rs @@ -0,0 +1,72 @@ +use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::ops::{Deref, DerefMut}; + +#[derive(Debug, Default)] +#[repr(transparent)] +pub struct PyMutex(Mutex); + +impl PyMutex { + pub const fn new(value: T) -> Self { + Self(parking_lot::const_mutex(value)) + } +} + +impl PyMutex { + pub fn lock(&self) -> PyMutexGuard { + PyMutexGuard(self.0.lock()) + } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct PyMutexGuard<'a, T: ?Sized>(MutexGuard<'a, T>); +impl Deref for PyMutexGuard<'_, T> { + type Target = T; + fn deref(&self) -> &T { + self.0.deref() + } +} + +#[derive(Debug, Default)] +#[repr(transparent)] +pub struct PyRwLock(RwLock); + +impl PyRwLock { + pub const fn new(value: T) -> Self { + Self(parking_lot::const_rwlock(value)) + } +} + +impl PyRwLock { + pub fn read(&self) -> PyRwLockReadGuard { + PyRwLockReadGuard(self.0.read()) + } + pub fn write(&self) -> PyRwLockWriteGuard { + PyRwLockWriteGuard(self.0.write()) + } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct PyRwLockReadGuard<'a, T: ?Sized>(RwLockReadGuard<'a, T>); +impl Deref for PyRwLockReadGuard<'_, T> { + type Target = T; + fn deref(&self) -> &T { + self.0.deref() + } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct PyRwLockWriteGuard<'a, T: ?Sized>(RwLockWriteGuard<'a, T>); +impl Deref for PyRwLockWriteGuard<'_, T> { + type Target = T; + fn deref(&self) -> &T { + self.0.deref() + } +} +impl DerefMut for PyRwLockWriteGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + self.0.deref_mut() + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 3a961c6c1..f6d704744 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,4 +1,6 @@ //! A crate to hold types and functions common to all rustpython components. +pub mod cell; pub mod float_ops; pub mod hash; +pub mod rc; diff --git a/common/src/rc.rs b/common/src/rc.rs new file mode 100644 index 000000000..1105ca1c9 --- /dev/null +++ b/common/src/rc.rs @@ -0,0 +1,6 @@ +use std::sync::{Arc, Weak}; + +// type aliases instead of newtypes because you can't do `fn method(self: PyRc)` with a +// newtype; requires the arbitrary_self_types unstable feature +pub type PyRc = Arc; +pub type PyWeak = Weak; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index c0f923c18..80724fd85 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -83,6 +83,7 @@ mod vm; // pub use self::pyobject::Executor; pub use self::vm::{InitParameter, PySettings, VirtualMachine}; pub use rustpython_bytecode::*; +pub use rustpython_common as common; #[doc(hidden)] pub mod __exports { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 1dff7f790..22d89f764 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -1,4 +1,4 @@ -use parking_lot::RwLock; +use rustpython_common::cell::PyRwLock; use std::collections::{HashMap, HashSet}; use std::fmt; @@ -29,9 +29,9 @@ pub struct PyClass { pub name: String, pub bases: Vec, pub mro: Vec, - pub subclasses: RwLock>, - pub attributes: RwLock, - pub slots: RwLock, + pub subclasses: PyRwLock>, + pub attributes: PyRwLock, + pub slots: PyRwLock, } impl fmt::Display for PyClass { @@ -595,12 +595,12 @@ pub fn new( name: String::from(name), bases, mro, - subclasses: RwLock::default(), - attributes: RwLock::new(dict), - slots: RwLock::default(), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(dict), + slots: PyRwLock::default(), }, dict: None, - typ: RwLock::new(typ.into_typed_pyobj()), + typ: PyRwLock::new(typ.into_typed_pyobj()), } .into_ref(); diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index d7450f42b..f67233eb8 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -7,9 +7,9 @@ use crate::pyobject::{ use crate::slots::SlotCall; use crate::vm::VirtualMachine; use rustpython_common::hash::PyHash; +use rustpython_common::rc::{PyRc, PyWeak as Weak}; use crossbeam_utils::atomic::AtomicCell; -use std::sync::{Arc, Weak}; #[pyclass] #[derive(Debug)] @@ -21,7 +21,7 @@ pub struct PyWeak { impl PyWeak { pub fn downgrade(obj: &PyObjectRef) -> PyWeak { PyWeak { - referent: Arc::downgrade(obj), + referent: PyRc::downgrade(obj), hash: AtomicCell::new(None), } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index c32f31ad0..ce7224092 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; use std::ops::Deref; -use std::sync::Arc; use indexmap::IndexMap; use num_bigint::BigInt; @@ -40,7 +39,8 @@ use crate::scope::Scope; use crate::slots::PyTpFlags; use crate::types::{create_type, initialize_types, TypeZoo}; use crate::vm::VirtualMachine; -use parking_lot::{RwLock, RwLockReadGuard}; +use rustpython_common::cell::{PyRwLock, PyRwLockReadGuard}; +use rustpython_common::rc::PyRc; /* Python objects and references. @@ -61,7 +61,7 @@ Basically reference counting, but then done by rust. /// this reference counting is accounted for by this type. Use the `.clone()` /// method to create a new reference and increment the amount of references /// to the python object by 1. -pub type PyObjectRef = Arc>; +pub type PyObjectRef = PyRc>; /// Use this type for functions which return a python object or an exception. /// Both the python object and the python exception are `PyObjectRef` types @@ -586,8 +586,8 @@ impl PyContext { pub fn new_base_object(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { - typ: RwLock::new(class.into_typed_pyobj()), - dict: dict.map(|d| RwLock::new(d.into_typed_pyobj())), + typ: PyRwLock::new(class.into_typed_pyobj()), + dict: dict.map(|d| PyRwLock::new(d.into_typed_pyobj())), payload: objobject::PyBaseObject, } .into_ref() @@ -643,8 +643,8 @@ pub struct PyObject where T: ?Sized + PyObjectPayload, { - pub(crate) typ: RwLock>>, // __class__ member - pub(crate) dict: Option>>>, // __dict__ member + pub(crate) typ: PyRwLock>>, // __class__ member + pub(crate) dict: Option>>>, // __dict__ member pub payload: T, } @@ -653,7 +653,7 @@ impl PyObject { /// /// If the downcast fails, the original ref is returned in as `Err` so /// another downcast can be attempted without unnecessary cloning. - pub fn downcast(self: Arc) -> Result, PyObjectRef> { + pub fn downcast(self: PyRc) -> Result, PyObjectRef> { if self.payload_is::() { Ok(PyRef { obj: self, @@ -664,15 +664,15 @@ impl PyObject { } } - /// Downcast this PyObjectRef to an `Arc>`. The [`downcast`](#method.downcast) method + /// Downcast this PyObjectRef to an `PyRc>`. The [`downcast`](#method.downcast) method /// is generally preferred, as the `PyRef` it returns implements `Deref`, and /// therefore can be used similarly to an `&T`. pub fn downcast_generic( - self: Arc, - ) -> Result>, PyObjectRef> { + self: PyRc, + ) -> Result>, PyObjectRef> { if self.payload_is::() { - let ptr = Arc::into_raw(self) as *const PyObject; - let ret = unsafe { Arc::from_raw(ptr) }; + let ptr = PyRc::into_raw(self) as *const PyObject; + let ret = unsafe { PyRc::from_raw(ptr) }; Ok(ret) } else { Err(self) @@ -738,7 +738,7 @@ impl PyRef { self.obj } - pub fn into_typed_pyobj(self) -> Arc> { + pub fn into_typed_pyobj(self) -> PyRc> { self.into_object().downcast_generic().unwrap() } } @@ -850,7 +850,7 @@ impl IdProtocol for PyObject { } } -impl IdProtocol for Arc { +impl IdProtocol for PyRc { fn get_id(&self) -> usize { (**self).get_id() } @@ -877,7 +877,7 @@ impl IdProtocol for &'_ T { /// A borrow of a reference to a Python object. This avoids having clone the `PyRef`/ /// `PyObjectRef`, which isn't that cheap as that increments the atomic reference counter. pub struct PyLease<'a, T: PyObjectPayload> { - inner: RwLockReadGuard<'a, Arc>>, + inner: PyRwLockReadGuard<'a, PyRc>>, } impl<'a, T: PyObjectPayload + PyValue> PyLease<'a, T> { @@ -1212,8 +1212,8 @@ where #[allow(clippy::new_ret_no_self)] pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { PyObject { - typ: RwLock::new(typ.into_typed_pyobj()), - dict: dict.map(|d| RwLock::new(d.into_typed_pyobj())), + typ: PyRwLock::new(typ.into_typed_pyobj()), + dict: dict.map(|d| PyRwLock::new(d.into_typed_pyobj())), payload, } .into_ref() @@ -1221,10 +1221,10 @@ where // Move this object into a reference object, transferring ownership. pub fn into_ref(self) -> PyObjectRef { - Arc::new(self) + PyRc::new(self) } - pub fn into_pyref(self: Arc) -> PyRef + pub fn into_pyref(self: PyRc) -> PyRef where T: PyValue, { diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index b5ff02074..dfebfc94f 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -7,7 +7,6 @@ mod decl { use num_traits::{One, Signed, ToPrimitive, Zero}; use parking_lot::{RwLock, RwLockWriteGuard}; use std::iter; - use std::sync::Arc; use crate::function::{Args, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; @@ -19,6 +18,7 @@ mod decl { IdProtocol, PyCallable, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; + use rustpython_common::rc::PyRc; #[pyclass(name = "chain")] #[derive(Debug)] @@ -738,8 +738,8 @@ mod decl { } impl PyItertoolsTeeData { - fn new(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - Ok(Arc::new(PyItertoolsTeeData { + fn new(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + Ok(PyRc::new(PyItertoolsTeeData { iterable: get_iter(vm, &iterable)?, values: RwLock::new(vec![]), })) @@ -757,7 +757,7 @@ mod decl { #[pyclass(name = "tee")] #[derive(Debug)] struct PyItertoolsTee { - tee_data: Arc, + tee_data: PyRc, index: AtomicCell, } @@ -812,7 +812,7 @@ mod decl { #[pymethod(name = "__copy__")] fn copy(&self, vm: &VirtualMachine) -> PyResult { Ok(PyItertoolsTee { - tee_data: Arc::clone(&self.tee_data), + tee_data: PyRc::clone(&self.tee_data), index: AtomicCell::new(self.index.load()), } .into_ref_with_type(vm, Self::class(vm))? diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 35e440af9..6f59eeb6b 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -7,10 +7,10 @@ use crate::pyobject::PyObjectRef; use crate::vm::VirtualMachine; -use std::sync::Arc; +use rustpython_common::rc::PyRc; fn weakref_getweakrefcount(obj: PyObjectRef) -> usize { - Arc::weak_count(&obj) + PyRc::weak_count(&obj) } fn weakref_getweakrefs(_obj: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index ce0c0d6c5..5db87e38e 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,4 +1,3 @@ -use std::sync::Arc; use std::{env, mem}; use crate::builtins; @@ -11,6 +10,7 @@ use crate::pyobject::{ use crate::version; use crate::vm::{PySettings, VirtualMachine}; use rustpython_common::hash::{PyHash, PyUHash}; +use rustpython_common::rc::PyRc; /* * The magic sys module. @@ -109,7 +109,7 @@ impl SysFlags { } fn sys_getrefcount(obj: PyObjectRef) -> usize { - Arc::strong_count(&obj) + PyRc::strong_count(&obj) } fn sys_getsizeof(obj: PyObjectRef) -> usize { diff --git a/vm/src/types.rs b/vm/src/types.rs index a59ccb844..54435411b 100644 --- a/vm/src/types.rs +++ b/vm/src/types.rs @@ -40,10 +40,9 @@ use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::pyobject::{PyAttributes, PyContext, PyObject}; -use parking_lot::RwLock; +use rustpython_common::{cell::PyRwLock, rc::PyRc}; use std::mem::MaybeUninit; use std::ptr; -use std::sync::Arc; /// Holder of references to builtin types. #[derive(Debug)] @@ -275,14 +274,16 @@ macro_rules! partially_init { ) => {{ // check all the fields are there but *don't* actually run it if false { - #[allow(invalid_value)] + #[allow(invalid_value, dead_code, unreachable_code)] let _ = {$ty { $($init_field: $init_value,)* - $($uninit_field: ::std::mem::MaybeUninit::uninit().assume_init(),)* + $($uninit_field: unreachable!(),)* }}; } let mut m = ::std::mem::MaybeUninit::<$ty>::uninit(); - $(::std::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* + unsafe { + $(::std::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* + } m }}; } @@ -292,61 +293,64 @@ fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { // and both `type` and `object are instances of `type`. // to produce this circular dependency, we need an unsafe block. // (and yes, this will never get dropped. TODO?) - let (type_type, object_type) = unsafe { + let (type_type, object_type) = { type PyClassObj = PyObject; - type UninitRef = RwLock>>; + type UninitRef = PyRwLock>>; - let type_type: Arc> = Arc::new(partially_init!( + let type_type: PyRc> = PyRc::new(partially_init!( PyObject:: { dict: None, payload: PyClass { name: String::from("type"), bases: vec![], mro: vec![], - subclasses: RwLock::default(), - attributes: RwLock::new(PyAttributes::new()), - slots: RwLock::default(), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(PyAttributes::new()), + slots: PyRwLock::default(), }, }, Uninit { typ } )); - let object_type: Arc> = Arc::new(partially_init!( + let object_type: PyRc> = PyRc::new(partially_init!( PyObject:: { dict: None, payload: PyClass { name: String::from("object"), bases: vec![], mro: vec![], - subclasses: RwLock::default(), - attributes: RwLock::new(PyAttributes::new()), - slots: RwLock::default(), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(PyAttributes::new()), + slots: PyRwLock::default(), }, }, Uninit { typ }, )); let object_type_ptr = - Arc::into_raw(object_type) as *mut MaybeUninit as *mut PyClassObj; + PyRc::into_raw(object_type) as *mut MaybeUninit as *mut PyClassObj; let type_type_ptr = - Arc::into_raw(type_type.clone()) as *mut MaybeUninit as *mut PyClassObj; + PyRc::into_raw(type_type.clone()) as *mut MaybeUninit as *mut PyClassObj; - ptr::write( - &mut (*object_type_ptr).typ as *mut RwLock> - as *mut UninitRef, - RwLock::new(type_type.clone()), - ); - ptr::write( - &mut (*type_type_ptr).typ as *mut RwLock> as *mut UninitRef, - RwLock::new(type_type), - ); + unsafe { + ptr::write( + &mut (*object_type_ptr).typ as *mut PyRwLock> + as *mut UninitRef, + PyRwLock::new(type_type.clone()), + ); + ptr::write( + &mut (*type_type_ptr).typ as *mut PyRwLock> + as *mut UninitRef, + PyRwLock::new(type_type), + ); - let type_type = PyClassRef::from_obj_unchecked(Arc::from_raw(type_type_ptr)); - let object_type = PyClassRef::from_obj_unchecked(Arc::from_raw(object_type_ptr)); + let type_type = PyClassRef::from_obj_unchecked(PyRc::from_raw(type_type_ptr)); + let object_type = PyClassRef::from_obj_unchecked(PyRc::from_raw(object_type_ptr)); - (*type_type_ptr).payload.mro = vec![object_type.clone()]; - (*type_type_ptr).payload.bases = vec![object_type.clone()]; + (*type_type_ptr).payload.mro = vec![object_type.clone()]; + (*type_type_ptr).payload.bases = vec![object_type.clone()]; - (type_type, object_type) + (type_type, object_type) + } }; object_type diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 5d5373acb..354acf774 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -7,7 +7,6 @@ use std::cell::{Cell, Ref, RefCell}; use std::collections::hash_map::HashMap; use std::collections::hash_set::HashSet; -use std::sync::Arc; use std::{env, fmt}; use arr_macro::arr; @@ -47,6 +46,7 @@ use crate::pyobject::{ use crate::scope::Scope; use crate::stdlib; use crate::sysmodule; +use rustpython_common::rc::PyRc; // use objects::objects; @@ -57,7 +57,7 @@ use crate::sysmodule; pub struct VirtualMachine { pub builtins: PyObjectRef, pub sys_module: PyObjectRef, - pub ctx: Arc, + pub ctx: PyRc, pub frames: RefCell>, pub wasm_id: Option, pub exceptions: RefCell>, @@ -67,7 +67,7 @@ pub struct VirtualMachine { pub use_tracing: Cell, pub recursion_limit: Cell, pub signal_handlers: Option>>, - pub state: Arc, + pub state: PyRc, pub initialized: bool, } @@ -193,7 +193,7 @@ impl VirtualMachine { let mut vm = VirtualMachine { builtins: builtins.clone(), sys_module: sysmod.clone(), - ctx: Arc::new(ctx), + ctx: PyRc::new(ctx), frames: RefCell::new(vec![]), wasm_id: None, exceptions: RefCell::new(vec![]), @@ -203,7 +203,7 @@ impl VirtualMachine { use_tracing: Cell::new(false), recursion_limit: Cell::new(if cfg!(debug_assertions) { 256 } else { 512 }), signal_handlers: Some(Box::new(signal_handlers)), - state: Arc::new(PyGlobalState { + state: PyRc::new(PyGlobalState { settings, stdlib_inits, frozen, @@ -752,7 +752,7 @@ impl VirtualMachine { pub fn isinstance(&self, obj: &PyObjectRef, cls: &PyClassRef) -> PyResult { // cpython first does an exact check on the type, although documentation doesn't state that // https://github.com/python/cpython/blob/a24107b04c1277e3c1105f98aff5bfa3a98b33a0/Objects/abstract.c#L2408 - if Arc::ptr_eq(&obj.class().into_object(), cls.as_object()) { + if PyRc::ptr_eq(&obj.class().into_object(), cls.as_object()) { Ok(true) } else { let ret = self.call_method(cls.as_object(), "__instancecheck__", vec![obj.clone()])?; From 39303f0567221fd2a83d5421bcb65aa973654d93 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 16 Jul 2020 20:45:52 -0500 Subject: [PATCH 091/209] Transition more files to use PyMutex/PyRwLock --- common/src/cell.rs | 5 +++ vm/src/dictdatatype.rs | 12 ++++---- vm/src/exceptions.rs | 18 +++++------ vm/src/frame.rs | 6 ++-- vm/src/obj/objbytearray.rs | 12 ++++---- vm/src/obj/objcoroinner.rs | 6 ++-- vm/src/obj/objenumerate.rs | 6 ++-- vm/src/obj/objlist.rs | 24 +++++++-------- vm/src/obj/objproperty.rs | 12 ++++---- vm/src/obj/objtype.rs | 2 +- vm/src/stdlib/array.rs | 10 +++--- vm/src/stdlib/collections.rs | 20 ++++++------ vm/src/stdlib/csv.rs | 7 ++--- vm/src/stdlib/hashlib.rs | 10 +++--- vm/src/stdlib/io.rs | 14 ++++----- vm/src/stdlib/itertools.rs | 60 ++++++++++++++++++------------------ vm/src/stdlib/os.rs | 6 ++-- vm/src/stdlib/random.rs | 7 ++--- vm/src/stdlib/socket.rs | 10 +++--- vm/src/stdlib/ssl.rs | 16 +++++----- vm/src/stdlib/winreg.rs | 10 +++--- vm/src/stdlib/zlib.rs | 18 +++++------ 22 files changed, 147 insertions(+), 144 deletions(-) diff --git a/common/src/cell.rs b/common/src/cell.rs index c9f47f11e..48343dc90 100644 --- a/common/src/cell.rs +++ b/common/src/cell.rs @@ -26,6 +26,11 @@ impl Deref for PyMutexGuard<'_, T> { self.0.deref() } } +impl DerefMut for PyMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + self.0.deref_mut() + } +} #[derive(Debug, Default)] #[repr(transparent)] diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 7a893cfbc..6116af321 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,8 +1,8 @@ +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::obj::objstr::{PyString, PyStringRef}; use crate::pyobject::{IdProtocol, IntoPyObject, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; use num_bigint::ToBigInt; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use rustpython_common::hash; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html @@ -20,7 +20,7 @@ type HashIndex = hash::PyHash; type EntryIndex = usize; pub struct Dict { - inner: RwLock>, + inner: PyRwLock>, } struct InnerDict { @@ -42,7 +42,7 @@ impl Clone for InnerDict { impl Clone for Dict { fn clone(&self) -> Self { Dict { - inner: RwLock::new(self.inner.read().clone()), + inner: PyRwLock::new(self.inner.read().clone()), } } } @@ -50,7 +50,7 @@ impl Clone for Dict { impl Default for Dict { fn default() -> Self { Dict { - inner: RwLock::new(InnerDict { + inner: PyRwLock::new(InnerDict { size: 0, indices: HashMap::new(), entries: Vec::new(), @@ -82,11 +82,11 @@ pub struct DictSize { } impl Dict { - fn borrow_value(&self) -> RwLockReadGuard<'_, InnerDict> { + fn borrow_value(&self) -> PyRwLockReadGuard<'_, InnerDict> { self.inner.read() } - fn borrow_value_mut(&self) -> RwLockWriteGuard<'_, InnerDict> { + fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, InnerDict> { self.inner.write() } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 5f4e6b4a9..7f5d4d242 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -1,3 +1,4 @@ +use crate::common::cell::PyRwLock; use crate::function::PyFuncArgs; use crate::obj::objnone::PyNone; use crate::obj::objstr::{PyString, PyStringRef}; @@ -14,7 +15,6 @@ use crate::types::create_type; use crate::VirtualMachine; use itertools::Itertools; -use parking_lot::RwLock; use std::fmt; use std::fs::File; use std::io::{self, BufRead, BufReader, Write}; @@ -23,11 +23,11 @@ use crossbeam_utils::atomic::AtomicCell; #[pyclass] pub struct PyBaseException { - traceback: RwLock>, - cause: RwLock>, - context: RwLock>, + traceback: PyRwLock>, + cause: PyRwLock>, + context: PyRwLock>, suppress_context: AtomicCell, - args: RwLock, + args: PyRwLock, } impl fmt::Debug for PyBaseException { @@ -51,11 +51,11 @@ impl PyValue for PyBaseException { impl PyBaseException { pub(crate) fn new(args: Vec, vm: &VirtualMachine) -> PyBaseException { PyBaseException { - traceback: RwLock::new(None), - cause: RwLock::new(None), - context: RwLock::new(None), + traceback: PyRwLock::new(None), + cause: PyRwLock::new(None), + context: PyRwLock::new(None), suppress_context: AtomicCell::new(false), - args: RwLock::new(PyTuple::from(args).into_ref(vm)), + args: PyRwLock::new(PyTuple::from(args).into_ref(vm)), } } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 35fef5315..02f036b62 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -3,10 +3,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use indexmap::IndexMap; use itertools::Itertools; -use parking_lot::Mutex; use crate::builtins::builtin_isinstance; use crate::bytecode; +use crate::common::cell::PyMutex; use crate::exceptions::{self, ExceptionCtor, PyBaseExceptionRef}; use crate::function::PyFuncArgs; use crate::obj::objasyncgenerator::PyAsyncGenWrappedValue; @@ -92,7 +92,7 @@ pub struct Frame { pub scope: Scope, /// index of last instruction ran pub lasti: AtomicUsize, - state: Mutex, + state: PyMutex, } impl PyValue for Frame { @@ -163,7 +163,7 @@ impl Frame { code, scope, lasti: AtomicUsize::new(0), - state: Mutex::new(FrameState { + state: PyMutex::new(FrameState { stack: Vec::new(), blocks: Vec::new(), }), diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 22e35e5da..1be33d24f 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,8 +1,8 @@ //! Implementation of the python bytearray object. +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; use num_traits::cast::ToPrimitive; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::mem::size_of; use super::objbyteinner::{ @@ -36,7 +36,7 @@ use crate::vm::VirtualMachine; #[pyclass(name = "bytearray")] #[derive(Debug)] pub struct PyByteArray { - inner: RwLock, + inner: PyRwLock, } pub type PyByteArrayRef = PyRef; @@ -44,21 +44,21 @@ pub type PyByteArrayRef = PyRef; impl PyByteArray { pub fn new(data: Vec) -> Self { PyByteArray { - inner: RwLock::new(PyByteInner { elements: data }), + inner: PyRwLock::new(PyByteInner { elements: data }), } } fn from_inner(inner: PyByteInner) -> Self { PyByteArray { - inner: RwLock::new(inner), + inner: PyRwLock::new(inner), } } - pub fn borrow_value(&self) -> RwLockReadGuard<'_, PyByteInner> { + pub fn borrow_value(&self) -> PyRwLockReadGuard<'_, PyByteInner> { self.inner.read() } - pub fn borrow_value_mut(&self) -> RwLockWriteGuard<'_, PyByteInner> { + pub fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, PyByteInner> { self.inner.write() } } diff --git a/vm/src/obj/objcoroinner.rs b/vm/src/obj/objcoroinner.rs index 61a43357d..11d4515ab 100644 --- a/vm/src/obj/objcoroinner.rs +++ b/vm/src/obj/objcoroinner.rs @@ -4,8 +4,8 @@ use crate::frame::{ExecutionResult, FrameRef}; use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use crate::common::cell::PyRwLock; use crossbeam_utils::atomic::AtomicCell; -use parking_lot::RwLock; #[derive(Debug, PartialEq, Clone, Copy)] pub enum Variant { @@ -37,7 +37,7 @@ pub struct Coro { frame: FrameRef, pub closed: AtomicCell, running: AtomicCell, - exceptions: RwLock>, + exceptions: PyRwLock>, started: AtomicCell, variant: Variant, } @@ -48,7 +48,7 @@ impl Coro { frame, closed: AtomicCell::new(false), running: AtomicCell::new(false), - exceptions: RwLock::new(vec![]), + exceptions: PyRwLock::new(vec![]), started: AtomicCell::new(false), variant, } diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 4adbfda78..e36e35ec7 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -1,4 +1,4 @@ -use parking_lot::RwLock; +use crate::common::cell::PyRwLock; use std::ops::AddAssign; use num_bigint::BigInt; @@ -14,7 +14,7 @@ use crate::vm::VirtualMachine; #[pyclass] #[derive(Debug)] pub struct PyEnumerate { - counter: RwLock, + counter: PyRwLock, iterator: PyObjectRef, } type PyEnumerateRef = PyRef; @@ -41,7 +41,7 @@ impl PyEnumerate { let iterator = objiter::get_iter(vm, &iterable)?; PyEnumerate { - counter: RwLock::new(counter), + counter: PyRwLock::new(counter), iterator, } .into_ref_with_type(vm, cls) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 70139ab45..6c48f3b6c 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -2,10 +2,10 @@ use std::fmt; use std::mem::size_of; use std::ops::{DerefMut, Range}; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crossbeam_utils::atomic::AtomicCell; use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use super::objbool; use super::objbyteinner; @@ -29,7 +29,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; #[pyclass] #[derive(Default)] pub struct PyList { - elements: RwLock>, + elements: PyRwLock>, } impl fmt::Debug for PyList { @@ -42,7 +42,7 @@ impl fmt::Debug for PyList { impl From> for PyList { fn from(elements: Vec) -> Self { PyList { - elements: RwLock::new(elements), + elements: PyRwLock::new(elements), } } } @@ -54,11 +54,11 @@ impl PyValue for PyList { } impl PyList { - pub fn borrow_elements(&self) -> RwLockReadGuard<'_, Vec> { + pub fn borrow_elements(&self) -> PyRwLockReadGuard<'_, Vec> { self.elements.read() } - pub fn borrow_elements_mut(&self) -> RwLockWriteGuard<'_, Vec> { + pub fn borrow_elements_mut(&self) -> PyRwLockWriteGuard<'_, Vec> { self.elements.write() } @@ -315,7 +315,7 @@ impl PyList { } fn _set_slice( - mut elements: RwLockWriteGuard<'_, Vec>, + mut elements: PyRwLockWriteGuard<'_, Vec>, range: Range, items: Vec, vm: &VirtualMachine, @@ -327,7 +327,7 @@ impl PyList { } fn _set_stepped_slice( - elements: RwLockWriteGuard<'_, Vec>, + elements: PyRwLockWriteGuard<'_, Vec>, range: Range, step: usize, items: Vec, @@ -365,7 +365,7 @@ impl PyList { } fn _set_stepped_slice_reverse( - elements: RwLockWriteGuard<'_, Vec>, + elements: PyRwLockWriteGuard<'_, Vec>, range: Range, step: usize, items: Vec, @@ -403,7 +403,7 @@ impl PyList { } fn _replace_indexes( - mut elements: RwLockWriteGuard<'_, Vec>, + mut elements: PyRwLockWriteGuard<'_, Vec>, indexes: I, items: &[PyObjectRef], ) where @@ -673,12 +673,12 @@ impl PyList { } } - fn _del_slice(mut elements: RwLockWriteGuard<'_, Vec>, range: Range) { + fn _del_slice(mut elements: PyRwLockWriteGuard<'_, Vec>, range: Range) { elements.drain(range); } fn _del_stepped_slice( - mut elements: RwLockWriteGuard<'_, Vec>, + mut elements: PyRwLockWriteGuard<'_, Vec>, range: Range, step: usize, ) { @@ -702,7 +702,7 @@ impl PyList { } fn _del_stepped_slice_reverse( - mut elements: RwLockWriteGuard<'_, Vec>, + mut elements: PyRwLockWriteGuard<'_, Vec>, range: Range, step: usize, ) { diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index 909ca4321..737f7e8cf 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -1,7 +1,7 @@ /*! Python `property` descriptor class. */ -use parking_lot::RwLock; +use crate::common::cell::PyRwLock; use super::objtype::PyClassRef; use crate::function::OptionalArg; @@ -49,7 +49,7 @@ pub struct PyProperty { getter: Option, setter: Option, deleter: Option, - doc: RwLock>, + doc: PyRwLock>, } impl PyValue for PyProperty { @@ -98,7 +98,7 @@ impl PyProperty { getter: args.fget, setter: args.fset, deleter: args.fdel, - doc: RwLock::new(args.doc), + doc: PyRwLock::new(args.doc), } .into_ref_with_type(vm, cls) } @@ -160,7 +160,7 @@ impl PyProperty { getter: getter.or_else(|| zelf.getter.clone()), setter: zelf.setter.clone(), deleter: zelf.deleter.clone(), - doc: RwLock::new(None), + doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } @@ -175,7 +175,7 @@ impl PyProperty { getter: zelf.getter.clone(), setter: setter.or_else(|| zelf.setter.clone()), deleter: zelf.deleter.clone(), - doc: RwLock::new(None), + doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } @@ -190,7 +190,7 @@ impl PyProperty { getter: zelf.getter.clone(), setter: zelf.setter.clone(), deleter: deleter.or_else(|| zelf.deleter.clone()), - doc: RwLock::new(None), + doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 22d89f764..236c76216 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -1,4 +1,4 @@ -use rustpython_common::cell::PyRwLock; +use crate::common::cell::PyRwLock; use std::collections::{HashMap, HashSet}; use std::fmt; diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 0f6dde5f2..87c2afaa2 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -10,7 +10,7 @@ use crate::pyobject::{ }; use crate::VirtualMachine; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use std::fmt; use crossbeam_utils::atomic::AtomicCell; @@ -239,7 +239,7 @@ def_array_enum!( #[pyclass(name = "array")] #[derive(Debug)] pub struct PyArray { - array: RwLock, + array: PyRwLock, } pub type PyArrayRef = PyRef; @@ -252,11 +252,11 @@ impl PyValue for PyArray { #[pyimpl(flags(BASETYPE))] impl PyArray { - fn borrow_value(&self) -> RwLockReadGuard<'_, ArrayContentType> { + fn borrow_value(&self) -> PyRwLockReadGuard<'_, ArrayContentType> { self.array.read() } - fn borrow_value_mut(&self) -> RwLockWriteGuard<'_, ArrayContentType> { + fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, ArrayContentType> { self.array.write() } @@ -278,7 +278,7 @@ impl PyArray { let array = ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err.to_string()))?; let zelf = PyArray { - array: RwLock::new(array), + array: PyRwLock::new(array), }; if let OptionalArg::Present(init) = init { zelf.extend(init, vm)?; diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index a959970b8..eabc0fce1 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -2,6 +2,7 @@ pub(crate) use _collections::make_module; #[pymodule] mod _collections { + use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::OptionalArg; use crate::obj::{objiter, objtype::PyClassRef}; use crate::pyobject::{ @@ -12,7 +13,6 @@ mod _collections { use crate::vm::ReprGuard; use crate::VirtualMachine; use itertools::Itertools; - use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::collections::VecDeque; use crossbeam_utils::atomic::AtomicCell; @@ -20,7 +20,7 @@ mod _collections { #[pyclass(name = "deque")] #[derive(Debug)] struct PyDeque { - deque: RwLock>, + deque: PyRwLock>, maxlen: AtomicCell>, } type PyDequeRef = PyRef; @@ -38,16 +38,16 @@ mod _collections { } impl PyDeque { - fn borrow_deque(&self) -> RwLockReadGuard<'_, VecDeque> { + fn borrow_deque(&self) -> PyRwLockReadGuard<'_, VecDeque> { self.deque.read() } - fn borrow_deque_mut(&self) -> RwLockWriteGuard<'_, VecDeque> { + fn borrow_deque_mut(&self) -> PyRwLockWriteGuard<'_, VecDeque> { self.deque.write() } } - struct SimpleSeqDeque<'a>(RwLockReadGuard<'a, VecDeque>); + struct SimpleSeqDeque<'a>(PyRwLockReadGuard<'a, VecDeque>); impl sequence::SimpleSeq for SimpleSeqDeque<'_> { fn len(&self) -> usize { @@ -59,8 +59,8 @@ mod _collections { } } - impl<'a> From>> for SimpleSeqDeque<'a> { - fn from(from: RwLockReadGuard<'a, VecDeque>) -> Self { + impl<'a> From>> for SimpleSeqDeque<'a> { + fn from(from: PyRwLockReadGuard<'a, VecDeque>) -> Self { Self { 0: from } } } @@ -75,7 +75,7 @@ mod _collections { vm: &VirtualMachine, ) -> PyResult> { let py_deque = PyDeque { - deque: RwLock::default(), + deque: PyRwLock::default(), maxlen: AtomicCell::new(maxlen), }; if let OptionalArg::Present(iter) = iter { @@ -110,7 +110,7 @@ mod _collections { #[pymethod] fn copy(&self) -> Self { PyDeque { - deque: RwLock::new(self.borrow_deque().clone()), + deque: PyRwLock::new(self.borrow_deque().clone()), maxlen: AtomicCell::new(self.maxlen.load()), } } @@ -383,7 +383,7 @@ mod _collections { }; let deque = mul.skip(skipped).cloned().collect(); PyDeque { - deque: RwLock::new(deque), + deque: PyRwLock::new(deque), maxlen: AtomicCell::new(self.maxlen.load()), } } diff --git a/vm/src/stdlib/csv.rs b/vm/src/stdlib/csv.rs index bef432b00..452650c75 100644 --- a/vm/src/stdlib/csv.rs +++ b/vm/src/stdlib/csv.rs @@ -1,11 +1,10 @@ -use parking_lot::RwLock; use std::fmt::{self, Debug, Formatter}; use csv as rust_csv; use itertools::join; +use crate::common::cell::PyRwLock; use crate::function::PyFuncArgs; - use crate::obj::objiter; use crate::obj::objstr::{self, PyString}; use crate::obj::objtype::PyClassRef; @@ -126,7 +125,7 @@ impl ReadState { #[pyclass(name = "Reader")] struct Reader { - state: RwLock, + state: PyRwLock, } impl Debug for Reader { @@ -143,7 +142,7 @@ impl PyValue for Reader { impl Reader { fn new(iter: PyIterable, config: ReaderOption) -> Self { - let state = RwLock::new(ReadState::new(iter, config)); + let state = PyRwLock::new(ReadState::new(iter, config)); Reader { state } } } diff --git a/vm/src/stdlib/hashlib.rs b/vm/src/stdlib/hashlib.rs index 43f8442dc..262755279 100644 --- a/vm/src/stdlib/hashlib.rs +++ b/vm/src/stdlib/hashlib.rs @@ -1,10 +1,10 @@ +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::fmt; use blake2::{Blake2b, Blake2s}; @@ -17,7 +17,7 @@ use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; // TODO: , Shake128, Shake25 #[pyclass(name = "hasher")] struct PyHasher { name: String, - buffer: RwLock, + buffer: PyRwLock, } impl fmt::Debug for PyHasher { @@ -37,15 +37,15 @@ impl PyHasher { fn new(name: &str, d: HashWrapper) -> Self { PyHasher { name: name.to_owned(), - buffer: RwLock::new(d), + buffer: PyRwLock::new(d), } } - fn borrow_value(&self) -> RwLockReadGuard<'_, HashWrapper> { + fn borrow_value(&self) -> PyRwLockReadGuard<'_, HashWrapper> { self.buffer.read() } - fn borrow_value_mut(&self) -> RwLockWriteGuard<'_, HashWrapper> { + fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, HashWrapper> { self.buffer.write() } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 26daec8d0..743e9955f 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,13 +1,13 @@ /* * I/O core tools. */ -use parking_lot::{RwLock, RwLockWriteGuard}; use std::fs; use std::io::{self, prelude::*, Cursor, SeekFrom}; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; +use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; use crate::exceptions::PyBaseExceptionRef; use crate::function::{Args, KwArgs, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; @@ -121,7 +121,7 @@ impl BufferedIO { #[derive(Debug)] struct PyStringIO { - buffer: RwLock, + buffer: PyRwLock, closed: AtomicCell, } @@ -134,7 +134,7 @@ impl PyValue for PyStringIO { } impl PyStringIORef { - fn buffer(&self, vm: &VirtualMachine) -> PyResult> { + fn buffer(&self, vm: &VirtualMachine) -> PyResult> { if !self.closed.load() { Ok(self.buffer.write()) } else { @@ -236,7 +236,7 @@ fn string_io_new( let input = flatten.map_or_else(Vec::new, |v| objstr::borrow_value(&v).as_bytes().to_vec()); PyStringIO { - buffer: RwLock::new(BufferedIO::new(Cursor::new(input))), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(input))), closed: AtomicCell::new(false), } .into_ref_with_type(vm, cls) @@ -244,7 +244,7 @@ fn string_io_new( #[derive(Debug)] struct PyBytesIO { - buffer: RwLock, + buffer: PyRwLock, closed: AtomicCell, } @@ -257,7 +257,7 @@ impl PyValue for PyBytesIO { } impl PyBytesIORef { - fn buffer(&self, vm: &VirtualMachine) -> PyResult> { + fn buffer(&self, vm: &VirtualMachine) -> PyResult> { if !self.closed.load() { Ok(self.buffer.write()) } else { @@ -341,7 +341,7 @@ fn bytes_io_new( }; PyBytesIO { - buffer: RwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), closed: AtomicCell::new(false), } .into_ref_with_type(vm, cls) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index dfebfc94f..eb9752f5d 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -5,9 +5,9 @@ mod decl { use crossbeam_utils::atomic::AtomicCell; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; - use parking_lot::{RwLock, RwLockWriteGuard}; use std::iter; + use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; use crate::function::{Args, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objint::{self, PyInt, PyIntRef}; @@ -25,7 +25,7 @@ mod decl { struct PyItertoolsChain { iterables: Vec, cur_idx: AtomicCell, - cached_iter: RwLock>, + cached_iter: PyRwLock>, } impl PyValue for PyItertoolsChain { @@ -41,7 +41,7 @@ mod decl { PyItertoolsChain { iterables: args.args, cur_idx: AtomicCell::new(0), - cached_iter: RwLock::new(None), + cached_iter: PyRwLock::new(None), } .into_ref_with_type(vm, cls) } @@ -99,7 +99,7 @@ mod decl { PyItertoolsChain { iterables, cur_idx: AtomicCell::new(0), - cached_iter: RwLock::new(None), + cached_iter: PyRwLock::new(None), } .into_ref_with_type(vm, cls) } @@ -159,7 +159,7 @@ mod decl { #[pyclass(name = "count")] #[derive(Debug)] struct PyItertoolsCount { - cur: RwLock, + cur: PyRwLock, step: BigInt, } @@ -188,7 +188,7 @@ mod decl { }; PyItertoolsCount { - cur: RwLock::new(start), + cur: PyRwLock::new(start), step, } .into_ref_with_type(vm, cls) @@ -212,7 +212,7 @@ mod decl { #[derive(Debug)] struct PyItertoolsCycle { iter: PyObjectRef, - saved: RwLock>, + saved: PyRwLock>, index: AtomicCell, } @@ -234,7 +234,7 @@ mod decl { PyItertoolsCycle { iter: iter.clone(), - saved: RwLock::new(Vec::new()), + saved: PyRwLock::new(Vec::new()), index: AtomicCell::new(0), } .into_ref_with_type(vm, cls) @@ -273,7 +273,7 @@ mod decl { #[derive(Debug)] struct PyItertoolsRepeat { object: PyObjectRef, - times: Option>, + times: Option>, } impl PyValue for PyItertoolsRepeat { @@ -292,7 +292,7 @@ mod decl { vm: &VirtualMachine, ) -> PyResult> { let times = match times.into_option() { - Some(int) => Some(RwLock::new(int.as_bigint().clone())), + Some(int) => Some(PyRwLock::new(int.as_bigint().clone())), None => None, }; @@ -675,7 +675,7 @@ mod decl { struct PyItertoolsAccumulate { iterable: PyObjectRef, binop: PyObjectRef, - acc_value: RwLock>, + acc_value: PyRwLock>, } impl PyValue for PyItertoolsAccumulate { @@ -698,7 +698,7 @@ mod decl { PyItertoolsAccumulate { iterable: iter, binop: binop.unwrap_or_else(|| vm.get_none()), - acc_value: RwLock::new(None), + acc_value: PyRwLock::new(None), } .into_ref_with_type(vm, cls) } @@ -734,14 +734,14 @@ mod decl { #[derive(Debug)] struct PyItertoolsTeeData { iterable: PyObjectRef, - values: RwLock>, + values: PyRwLock>, } impl PyItertoolsTeeData { fn new(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult> { Ok(PyRc::new(PyItertoolsTeeData { iterable: get_iter(vm, &iterable)?, - values: RwLock::new(vec![]), + values: PyRwLock::new(vec![]), })) } @@ -836,7 +836,7 @@ mod decl { #[derive(Debug)] struct PyItertoolsProduct { pools: Vec>, - idxs: RwLock>, + idxs: PyRwLock>, cur: AtomicCell, stop: AtomicCell, } @@ -883,7 +883,7 @@ mod decl { PyItertoolsProduct { pools, - idxs: RwLock::new(vec![0; l]), + idxs: PyRwLock::new(vec![0; l]), cur: AtomicCell::new(l - 1), stop: AtomicCell::new(false), } @@ -920,7 +920,7 @@ mod decl { Ok(res.into_ref(vm).into_object()) } - fn update_idxs(&self, mut idxs: RwLockWriteGuard<'_, Vec>) { + fn update_idxs(&self, mut idxs: PyRwLockWriteGuard<'_, Vec>) { let cur = self.cur.load(); let lst_idx = &self.pools[cur].len() - 1; @@ -948,7 +948,7 @@ mod decl { #[derive(Debug)] struct PyItertoolsCombinations { pool: Vec, - indices: RwLock>, + indices: PyRwLock>, r: AtomicCell, exhausted: AtomicCell, } @@ -981,7 +981,7 @@ mod decl { PyItertoolsCombinations { pool, - indices: RwLock::new((0..r).collect()), + indices: PyRwLock::new((0..r).collect()), r: AtomicCell::new(r), exhausted: AtomicCell::new(r > n), } @@ -1047,7 +1047,7 @@ mod decl { #[derive(Debug)] struct PyItertoolsCombinationsWithReplacement { pool: Vec, - indices: RwLock>, + indices: PyRwLock>, r: AtomicCell, exhausted: AtomicCell, } @@ -1080,7 +1080,7 @@ mod decl { PyItertoolsCombinationsWithReplacement { pool, - indices: RwLock::new(vec![0; r]), + indices: PyRwLock::new(vec![0; r]), r: AtomicCell::new(r), exhausted: AtomicCell::new(n == 0 && r > 0), } @@ -1140,12 +1140,12 @@ mod decl { #[pyclass(name = "permutations")] #[derive(Debug)] struct PyItertoolsPermutations { - pool: Vec, // Collected input iterable - indices: RwLock>, // One index per element in pool - cycles: RwLock>, // One rollover counter per element in the result - result: RwLock>>, // Indexes of the most recently returned result - r: AtomicCell, // Size of result tuple - exhausted: AtomicCell, // Set when the iterator is exhausted + pool: Vec, // Collected input iterable + indices: PyRwLock>, // One index per element in pool + cycles: PyRwLock>, // One rollover counter per element in the result + result: PyRwLock>>, // Indexes of the most recently returned result + r: AtomicCell, // Size of result tuple + exhausted: AtomicCell, // Set when the iterator is exhausted } impl PyValue for PyItertoolsPermutations { @@ -1186,9 +1186,9 @@ mod decl { PyItertoolsPermutations { pool, - indices: RwLock::new((0..n).collect()), - cycles: RwLock::new((0..r).map(|i| n - i).collect()), - result: RwLock::new(None), + indices: PyRwLock::new((0..n).collect()), + cycles: PyRwLock::new((0..r).map(|i| n - i).collect()), + result: PyRwLock::new(None), r: AtomicCell::new(r), exhausted: AtomicCell::new(r > n), } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a372ecc75..746e98d48 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,4 +1,3 @@ -use parking_lot::RwLock; use std::ffi; use std::fs::File; use std::fs::OpenOptions; @@ -26,6 +25,7 @@ use std::os::unix::io::RawFd; use std::os::windows::io::RawHandle; use super::errno::errors; +use crate::common::cell::PyRwLock; use crate::exceptions::PyBaseExceptionRef; use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; use crate::obj::objbyteinner::PyBytesLike; @@ -726,7 +726,7 @@ impl DirEntryRef { #[pyclass] #[derive(Debug)] struct ScandirIterator { - entries: RwLock, + entries: PyRwLock, exhausted: AtomicCell, mode: OutputMode, } @@ -791,7 +791,7 @@ fn os_scandir(path: OptionalArg, vm: &VirtualMachine) -> PyResult { match fs::read_dir(path.path) { Ok(iter) => Ok(ScandirIterator { - entries: RwLock::new(iter), + entries: PyRwLock::new(iter), exhausted: AtomicCell::new(false), mode: path.mode, } diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index bbff45667..1cdab8ec3 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,6 +4,7 @@ pub(crate) use _random::make_module; #[pymodule] mod _random { + use crate::common::cell::PyMutex; use crate::function::OptionalOption; use crate::obj::objint::PyIntRef; use crate::obj::objtype::PyClassRef; @@ -13,8 +14,6 @@ mod _random { use num_traits::Signed; use rand::{rngs::StdRng, RngCore, SeedableRng}; - use parking_lot::Mutex; - #[derive(Debug)] enum PyRng { Std(Box), @@ -57,7 +56,7 @@ mod _random { #[pyclass(name = "Random")] #[derive(Debug)] struct PyRandom { - rng: Mutex, + rng: PyMutex, } impl PyValue for PyRandom { @@ -71,7 +70,7 @@ mod _random { #[pyslot(new)] fn new(cls: PyClassRef, vm: &VirtualMachine) -> PyResult> { PyRandom { - rng: Mutex::default(), + rng: PyMutex::default(), } .into_ref_with_type(vm, cls) } diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 25f7ab9da..ad423d083 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -1,4 +1,4 @@ -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use std::io::{self, prelude::*}; use std::net::{Ipv4Addr, Shutdown, SocketAddr, ToSocketAddrs}; use std::time::Duration; @@ -62,7 +62,7 @@ pub struct PySocket { kind: AtomicCell, family: AtomicCell, proto: AtomicCell, - sock: RwLock, + sock: PyRwLock, } impl PyValue for PySocket { @@ -75,11 +75,11 @@ pub type PySocketRef = PyRef; #[pyimpl(flags(BASETYPE))] impl PySocket { - fn sock(&self) -> RwLockReadGuard<'_, Socket> { + fn sock(&self) -> PyRwLockReadGuard<'_, Socket> { self.sock.read() } - fn sock_mut(&self) -> RwLockWriteGuard<'_, Socket> { + fn sock_mut(&self) -> PyRwLockWriteGuard<'_, Socket> { self.sock.write() } @@ -89,7 +89,7 @@ impl PySocket { kind: AtomicCell::default(), family: AtomicCell::default(), proto: AtomicCell::default(), - sock: RwLock::new(invalid_sock()), + sock: PyRwLock::new(invalid_sock()), } .into_ref_with_type(vm, cls) } diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index 9601e3764..3386df8ce 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -11,7 +11,7 @@ use crate::pyobject::{ use crate::types::create_type; use crate::VirtualMachine; -use parking_lot::{RwLock, RwLockWriteGuard}; +use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; use std::convert::TryFrom; use std::ffi::{CStr, CString}; use std::fmt; @@ -230,7 +230,7 @@ fn ssl_rand_pseudo_bytes(n: i32, vm: &VirtualMachine) -> PyResult<(Vec, bool #[pyclass(name = "_SSLContext")] struct PySslContext { - ctx: RwLock, + ctx: PyRwLock, check_hostname: bool, } @@ -248,7 +248,7 @@ impl PyValue for PySslContext { #[pyimpl(flags(BASETYPE))] impl PySslContext { - fn builder(&self) -> RwLockWriteGuard<'_, SslContextBuilder> { + fn builder(&self) -> PyRwLockWriteGuard<'_, SslContextBuilder> { self.ctx.write() } fn exec_ctx(&self, func: F) -> R @@ -308,7 +308,7 @@ impl PySslContext { .map_err(|e| convert_openssl_error(vm, e))?; PySslContext { - ctx: RwLock::new(builder), + ctx: PyRwLock::new(builder), check_hostname, } .into_ref_with_type(vm, cls) @@ -471,10 +471,10 @@ impl PySslContext { Ok(PySslSocket { ctx: zelf, - stream: RwLock::new(Some(stream)), + stream: PyRwLock::new(Some(stream)), socket_type, server_hostname: args.server_hostname, - owner: RwLock::new(args.owner.as_ref().map(PyWeak::downgrade)), + owner: PyRwLock::new(args.owner.as_ref().map(PyWeak::downgrade)), }) } } @@ -507,10 +507,10 @@ struct LoadVerifyLocationsArgs { #[pyclass(name = "_SSLSocket")] struct PySslSocket { ctx: PyRef, - stream: RwLock>>, + stream: PyRwLock>>, socket_type: SslServerOrClient, server_hostname: Option, - owner: RwLock>, + owner: PyRwLock>, } impl fmt::Debug for PySslSocket { diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index c5a70a7c0..550b99d05 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -1,5 +1,5 @@ #![allow(non_snake_case)] -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use std::convert::TryInto; use std::io; @@ -16,7 +16,7 @@ use winreg::{enums::RegType, RegKey, RegValue}; #[pyclass] #[derive(Debug)] struct PyHKEY { - key: RwLock, + key: PyRwLock, } type PyHKEYRef = PyRef; @@ -33,15 +33,15 @@ impl PyValue for PyHKEY { impl PyHKEY { fn new(key: RegKey) -> Self { Self { - key: RwLock::new(key), + key: PyRwLock::new(key), } } - fn key(&self) -> RwLockReadGuard<'_, RegKey> { + fn key(&self) -> PyRwLockReadGuard<'_, RegKey> { self.key.read() } - fn key_mut(&self) -> RwLockWriteGuard<'_, RegKey> { + fn key_mut(&self) -> PyRwLockWriteGuard<'_, RegKey> { self.key.write() } diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index b7ee1bdce..43feb1cc5 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -1,3 +1,4 @@ +use crate::common::cell::PyMutex; use crate::exceptions::PyBaseExceptionRef; use crate::function::OptionalArg; use crate::obj::objbyteinner::PyBytesLike; @@ -15,7 +16,6 @@ use flate2::{ }; use libz_sys as libz; -use parking_lot::Mutex; use std::io::Write; // copied from zlibmodule.c (commit 530f506ac91338) @@ -159,19 +159,19 @@ fn zlib_decompressobj( dict.with_ref(|d| decompress.set_dictionary(d).unwrap()); } PyDecompress { - decompress: Mutex::new(decompress), + decompress: PyMutex::new(decompress), eof: AtomicCell::new(false), - unused_data: Mutex::new(PyBytes::new(vec![]).into_ref(vm)), - unconsumed_tail: Mutex::new(PyBytes::new(vec![]).into_ref(vm)), + unused_data: PyMutex::new(PyBytes::new(vec![]).into_ref(vm)), + unconsumed_tail: PyMutex::new(PyBytes::new(vec![]).into_ref(vm)), } } #[pyclass(name = "Decompress")] #[derive(Debug)] struct PyDecompress { - decompress: Mutex, + decompress: PyMutex, eof: AtomicCell, - unused_data: Mutex, - unconsumed_tail: Mutex, + unused_data: PyMutex, + unconsumed_tail: PyMutex, } impl PyValue for PyDecompress { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -359,7 +359,7 @@ fn zlib_compressobj( }; let compress = Compress::new_with_window_bits(Compression::new(level), header, wbits); Ok(PyCompress { - inner: Mutex::new(CompressInner { + inner: PyMutex::new(CompressInner { compress, unconsumed: Vec::new(), }), @@ -375,7 +375,7 @@ struct CompressInner { #[pyclass] #[derive(Debug)] struct PyCompress { - inner: Mutex, + inner: PyMutex, } impl PyValue for PyCompress { From 3647568ac5f4b4256e0881ce6a998ae84615edd1 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:13:09 -0500 Subject: [PATCH 092/209] Make threading optional --- Cargo.lock | 1 + Cargo.toml | 8 +-- Lib/importlib/_bootstrap_external.py | 5 +- common/Cargo.toml | 6 ++- common/src/cell.rs | 75 ++++++++++++++++++++++++---- common/src/rc.rs | 8 +++ vm/Cargo.toml | 3 +- vm/src/pyobject.rs | 14 +++++- vm/src/stdlib/mod.rs | 2 + vm/src/vm.rs | 1 + 10 files changed, 105 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 809e7601e..36aaa7c11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1525,6 +1525,7 @@ dependencies = [ name = "rustpython-common" version = "0.0.0" dependencies = [ + "cfg-if", "hexf-parse", "lexical", "num-bigint", diff --git a/Cargo.toml b/Cargo.toml index 3714ba2de..f76a5da34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,10 @@ name = "bench" path = "./benchmarks/bench.rs" [features] +default = ["threading"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["rustpython-vm/freeze-stdlib"] +threading = ["rustpython-vm/threading"] ssl = ["rustpython-vm/ssl"] @@ -24,9 +26,9 @@ ssl = ["rustpython-vm/ssl"] log = "0.4" env_logger = "0.7" clap = "2.33" -rustpython-compiler = {path = "compiler", version = "0.1.1"} -rustpython-parser = {path = "parser", version = "0.1.1"} -rustpython-vm = {path = "vm", version = "0.1.1"} +rustpython-compiler = { path = "compiler", version = "0.1.1" } +rustpython-parser = { path = "parser", version = "0.1.1" } +rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["rustpython-parser", "rustpython-compiler"] } dirs = { package = "dirs-next", version = "1.0" } num-traits = "0.2.8" cfg-if = "0.1" diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f8ff5f4f2..a4e6a7176 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1587,7 +1587,10 @@ def _setup(_bootstrap_module): setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators}) # Directly load the _thread module (needed during bootstrap). - thread_module = _bootstrap._builtin_from_name('_thread') + try: + thread_module = _bootstrap._builtin_from_name('_thread') + except ImportError: + thread_module = None setattr(self_module, '_thread', thread_module) # Directly load the _weakref module (needed during bootstrap). diff --git a/common/Cargo.toml b/common/Cargo.toml index f5f3c5658..b8603397a 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -4,12 +4,16 @@ version = "0.0.0" authors = ["RustPython Team"] edition = "2018" +[features] +threading = ["parking_lot"] + [dependencies] -parking_lot = "0.11.0" +parking_lot = { version = "0.11.0", optional = true } num-traits = "0.2" num-bigint = "0.3" lexical = "4" hexf-parse = "0.1.0" +cfg-if = "0.1" [dev-dependencies] rand = "0.7.3" diff --git a/common/src/cell.rs b/common/src/cell.rs index 48343dc90..4c48326eb 100644 --- a/common/src/cell.rs +++ b/common/src/cell.rs @@ -1,25 +1,50 @@ +#[cfg(feature = "threading")] use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; +#[cfg(not(feature = "threading"))] +use std::cell::{Ref, RefCell, RefMut}; use std::ops::{Deref, DerefMut}; +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + type MutexInner = Mutex; + type MutexGuardInner<'a, T> = MutexGuard<'a, T>; + const fn new_mutex(value: T) -> MutexInner { + parking_lot::const_mutex(value) + } + fn lock_mutex(m: &MutexInner) -> MutexGuardInner { + m.lock() + } + } else { + type MutexInner = RefCell; + type MutexGuardInner<'a, T> = RefMut<'a, T>; + const fn new_mutex(value: T) -> MutexInner { + RefCell::new(value) + } + fn lock_mutex(m: &MutexInner) -> MutexGuardInner { + m.borrow_mut() + } + } +} + #[derive(Debug, Default)] #[repr(transparent)] -pub struct PyMutex(Mutex); +pub struct PyMutex(MutexInner); impl PyMutex { pub const fn new(value: T) -> Self { - Self(parking_lot::const_mutex(value)) + Self(new_mutex(value)) } } impl PyMutex { pub fn lock(&self) -> PyMutexGuard { - PyMutexGuard(self.0.lock()) + PyMutexGuard(lock_mutex(&self.0)) } } #[derive(Debug)] #[repr(transparent)] -pub struct PyMutexGuard<'a, T: ?Sized>(MutexGuard<'a, T>); +pub struct PyMutexGuard<'a, T: ?Sized>(MutexGuardInner<'a, T>); impl Deref for PyMutexGuard<'_, T> { type Target = T; fn deref(&self) -> &T { @@ -32,28 +57,58 @@ impl DerefMut for PyMutexGuard<'_, T> { } } +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + type RwLockInner = RwLock; + type RwLockReadInner<'a, T> = RwLockReadGuard<'a, T>; + type RwLockWriteInner<'a, T> = RwLockWriteGuard<'a, T>; + const fn new_rwlock(value: T) -> RwLockInner { + parking_lot::const_rwlock(value) + } + fn read_rwlock(m: &RwLockInner) -> RwLockReadInner { + m.read() + } + fn write_rwlock(m: &RwLockInner) -> RwLockWriteInner { + m.write() + } + } else { + type RwLockInner = RefCell; + type RwLockReadInner<'a, T> = Ref<'a, T>; + type RwLockWriteInner<'a, T> = RefMut<'a, T>; + const fn new_rwlock(value: T) -> RwLockInner { + RefCell::new(value) + } + fn read_rwlock(m: &RwLockInner) -> RwLockReadInner { + m.borrow() + } + fn write_rwlock(m: &RwLockInner) -> RwLockWriteInner { + m.borrow_mut() + } + } +} + #[derive(Debug, Default)] #[repr(transparent)] -pub struct PyRwLock(RwLock); +pub struct PyRwLock(RwLockInner); impl PyRwLock { pub const fn new(value: T) -> Self { - Self(parking_lot::const_rwlock(value)) + Self(new_rwlock(value)) } } impl PyRwLock { pub fn read(&self) -> PyRwLockReadGuard { - PyRwLockReadGuard(self.0.read()) + PyRwLockReadGuard(read_rwlock(&self.0)) } pub fn write(&self) -> PyRwLockWriteGuard { - PyRwLockWriteGuard(self.0.write()) + PyRwLockWriteGuard(write_rwlock(&self.0)) } } #[derive(Debug)] #[repr(transparent)] -pub struct PyRwLockReadGuard<'a, T: ?Sized>(RwLockReadGuard<'a, T>); +pub struct PyRwLockReadGuard<'a, T: ?Sized>(RwLockReadInner<'a, T>); impl Deref for PyRwLockReadGuard<'_, T> { type Target = T; fn deref(&self) -> &T { @@ -63,7 +118,7 @@ impl Deref for PyRwLockReadGuard<'_, T> { #[derive(Debug)] #[repr(transparent)] -pub struct PyRwLockWriteGuard<'a, T: ?Sized>(RwLockWriteGuard<'a, T>); +pub struct PyRwLockWriteGuard<'a, T: ?Sized>(RwLockWriteInner<'a, T>); impl Deref for PyRwLockWriteGuard<'_, T> { type Target = T; fn deref(&self) -> &T { diff --git a/common/src/rc.rs b/common/src/rc.rs index 1105ca1c9..8021b6112 100644 --- a/common/src/rc.rs +++ b/common/src/rc.rs @@ -1,6 +1,14 @@ +#[cfg(not(feature = "threading"))] +use std::rc::{Rc, Weak}; +#[cfg(feature = "threading")] use std::sync::{Arc, Weak}; // type aliases instead of newtypes because you can't do `fn method(self: PyRc)` with a // newtype; requires the arbitrary_self_types unstable feature + +#[cfg(feature = "threading")] pub type PyRc = Arc; +#[cfg(not(feature = "threading"))] +pub type PyRc = Rc; + pub type PyWeak = Weak; diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 68eef7ca0..e848a22ad 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -9,10 +9,11 @@ edition = "2018" include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] [features] -default = ["rustpython-parser", "rustpython-compiler"] +default = ["rustpython-parser", "rustpython-compiler", "threading"] vm-tracing-logging = [] flame-it = ["flame", "flamer"] freeze-stdlib = ["rustpython-pylib"] +threading = ["rustpython-common/threading"] ssl = ["openssl", "openssl-sys", "openssl-probe"] diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index ce7224092..639fd85d8 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1276,7 +1276,17 @@ impl PyObject { } } -pub trait PyValue: fmt::Debug + Send + Sync + Sized + 'static { +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + pub trait PyThreadingConstraint: Send + Sync {} + impl PyThreadingConstraint for T {} + } else { + pub trait PyThreadingConstraint {} + impl PyThreadingConstraint for T {} + } +} + +pub trait PyValue: fmt::Debug + PyThreadingConstraint + Sized + 'static { const HAVE_DICT: bool = false; fn class(vm: &VirtualMachine) -> PyClassRef; @@ -1306,7 +1316,7 @@ pub trait PyValue: fmt::Debug + Send + Sync + Sized + 'static { } } -pub trait PyObjectPayload: Any + fmt::Debug + Send + Sync + 'static { +pub trait PyObjectPayload: Any + fmt::Debug + PyThreadingConstraint + 'static { fn as_any(&self) -> &dyn Any; } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 71ff0d8be..880c1f9f2 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -32,6 +32,7 @@ mod string; mod symtable; mod sysconfigdata; #[cfg(not(target_arch = "wasm32"))] +#[cfg(feature = "threading")] mod thread; mod time_module; #[cfg(feature = "rustpython-parser")] @@ -131,6 +132,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("select".to_owned(), Box::new(select::make_module)); #[cfg(feature = "ssl")] modules.insert("_ssl".to_owned(), Box::new(ssl::make_module)); + #[cfg(feature = "threading")] modules.insert("_thread".to_owned(), Box::new(thread::make_module)); #[cfg(not(target_os = "redox"))] modules.insert("zlib".to_owned(), Box::new(zlib::make_module)); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 354acf774..c58486dcc 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -287,6 +287,7 @@ impl VirtualMachine { } } + #[cfg(feature = "threading")] pub(crate) fn new_thread(&self) -> VirtualMachine { VirtualMachine { builtins: self.builtins.clone(), From 642a1ee7eee44c5b22a4fb7b99949e6d5e4df100 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:20:01 -0500 Subject: [PATCH 093/209] Disable the threading feature for rustpython-wasm --- Cargo.lock | 376 ++++++++-------- Cargo.toml | 2 +- vm/Cargo.toml | 3 +- wasm/lib/Cargo.lock | 1019 ------------------------------------------- wasm/lib/Cargo.toml | 3 +- 5 files changed, 192 insertions(+), 1211 deletions(-) delete mode 100644 wasm/lib/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 36aaa7c11..fe3dddafd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,16 +7,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" [[package]] -name = "adler32" -version = "1.0.4" +name = "adler" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "adler32" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" [[package]] name = "aho-corasick" -version = "0.7.10" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" dependencies = [ "memchr", ] @@ -47,8 +53,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0609c78bd572f4edc74310dfb63a01f5609d53fa8b4dd7c4d98aef3b3e8d72d1" dependencies = [ "proc-macro-hack", - "quote 1.0.3", - "syn 1.0.17", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] @@ -112,9 +118,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "bincode" -version = "1.2.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" dependencies = [ "byteorder 1.3.4", "serde", @@ -122,18 +128,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.5.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" +checksum = "5f0dc55f2d8a1a85650ac47858bb001b4c0dd73d79e3c455a842925e68d29cd3" [[package]] name = "bitflags" @@ -187,9 +193,9 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" +checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931" dependencies = [ "lazy_static 1.4.0", "memchr", @@ -205,9 +211,9 @@ checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" [[package]] name = "bumpalo" -version = "3.2.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "byte-tools" @@ -239,9 +245,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" [[package]] name = "cfg-if" @@ -251,9 +257,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" dependencies = [ "js-sys", "num-integer", @@ -264,9 +270,9 @@ dependencies = [ [[package]] name = "clap" -version = "2.33.0" +version = "2.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" dependencies = [ "ansi_term", "atty", @@ -379,9 +385,9 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] @@ -410,16 +416,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dirs" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4ef5a8b902d393339e2a2c7fe573af92ce7e0ee5a3ff827b4c9ad7e07e4fa1" -dependencies = [ - "cfg-if", - "dirs-sys", -] - [[package]] name = "dirs-next" version = "1.0.1" @@ -430,18 +426,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" -dependencies = [ - "cfg-if", - "libc", - "redox_users", - "winapi", -] - [[package]] name = "dirs-sys-next" version = "0.1.0" @@ -455,9 +439,9 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13988670860b076248c74e1b54444efc4f1dec70c8bb25da4b7c0024396b72bf" +checksum = "4cbea23c15f7e685a7ad01c994407f3849fa7c88556aaafe4054215b6df93f78" dependencies = [ "libc", "socket2", @@ -560,9 +544,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" dependencies = [ "cfg-if", "crc32fast", @@ -573,9 +557,9 @@ dependencies = [ [[package]] name = "fnv" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foreign-types" @@ -606,9 +590,9 @@ checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" [[package]] name = "generational-arena" -version = "0.2.3" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921c3803adaeb9f9639de5149d9f0f9f4b79f00c423915b701db2e02ed80b9ce" +checksum = "8e1d3b771574f62d0548cee0ad9057857e9fc25d7a3335f140c84f6acd0bf601" dependencies = [ "cfg-if", ] @@ -645,10 +629,19 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.1.10" +name = "hashbrown" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb" +dependencies = [ + "autocfg 1.0.0", +] + +[[package]] +name = "hermit-abi" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -676,18 +669,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.3.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7" dependencies = [ "autocfg 1.0.0", + "hashbrown", ] [[package]] name = "instant" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7777a24a1ce5de49fcdde84ec46efa487c3af49d5b6e6e0a50367cc5c1096182" +checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" [[package]] name = "is-macro" @@ -697,9 +691,9 @@ checksum = "04807f3dc9e3ea39af3f8469a5297267faf94859637afb836b33f47d9b2650ee" dependencies = [ "Inflector", "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] @@ -713,15 +707,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.37" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" +checksum = "52732a3d3ad72c58ad2dc70624f9c17b46ecd0943b9a4f1ee37c4c18c5d983e2" dependencies = [ "wasm-bindgen", ] @@ -801,9 +795,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.71" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9" [[package]] name = "libz-sys" @@ -819,18 +813,18 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ "cfg-if", ] @@ -876,11 +870,11 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "miniz_oxide" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" dependencies = [ - "adler32", + "adler", ] [[package]] @@ -985,9 +979,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", @@ -1010,16 +1004,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] name = "once_cell" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" [[package]] name = "opaque-debug" @@ -1029,9 +1023,9 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "openssl" -version = "0.10.29" +version = "0.10.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", "cfg-if", @@ -1049,18 +1043,18 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.8.1+1.1.1f" +version = "111.10.2+1.1.1g" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04f0299a91de598dde58d2e99101895498dcf3d58896a3297798f28b27c8b72" +checksum = "a287fdb22e32b5b60624d4a5a7a02dbe82777f730ec0dbc42a0554326fef5a70" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.55" +version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ "autocfg 1.0.0", "cc", @@ -1104,9 +1098,9 @@ dependencies = [ [[package]] name = "paste" -version = "0.1.10" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4fb1930692d1b6a9cfabdde3d06ea0a7d186518e2f4d67660d8970e2fa647a" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" dependencies = [ "paste-impl", "proc-macro-hack", @@ -1114,14 +1108,11 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.10" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", ] [[package]] @@ -1155,9 +1146,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" [[package]] name = "pmutil" @@ -1165,16 +1156,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "precomputed-hash" @@ -1184,18 +1175,18 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro-crate" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ "toml", ] [[package]] name = "proc-macro-hack" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" [[package]] name = "proc-macro2" @@ -1208,11 +1199,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.10" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ - "unicode-xid 0.2.0", + "unicode-xid 0.2.1", ] [[package]] @@ -1242,11 +1233,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.3" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.10", + "proc-macro2 1.0.19", ] [[package]] @@ -1407,9 +1398,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_users" @@ -1424,9 +1415,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.6" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "aho-corasick", "memchr", @@ -1445,9 +1436,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "result-like" @@ -1553,11 +1544,11 @@ version = "0.1.2" dependencies = [ "maplit", "once_cell", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.19", + "quote 1.0.7", "rustpython-bytecode", "rustpython-compiler", - "syn 1.0.17", + "syn 1.0.35", ] [[package]] @@ -1702,16 +1693,17 @@ dependencies = [ [[package]] name = "rustyline" -version = "6.1.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6155a1169c281a8b75ee943a83eef0e2bb63aff057688c2111be319e51fc9f63" +checksum = "3358c21cbbc1a751892528db4e1de4b7a2b6a73f001e215aaba97d712cfa9777" dependencies = [ "cfg-if", - "dirs 2.0.1", + "dirs-next", "libc", "log", "memchr", "nix", + "scopeguard", "unicode-segmentation", "unicode-width", "utf8parse", @@ -1720,15 +1712,15 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "schannel" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static 1.4.0", "winapi", @@ -1757,9 +1749,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.106" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" dependencies = [ "serde_derive", ] @@ -1778,20 +1770,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.106" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", ] [[package]] name = "serde_json" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" dependencies = [ "itoa", "ryu", @@ -1812,9 +1804,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ "block-buffer", "digest", @@ -1849,9 +1841,9 @@ checksum = "aa0eec6425219a55511a35afac416a2eecc62f5fa1bec14f9b97de0caa7daebe" [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" [[package]] name = "socket2" @@ -1903,8 +1895,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator", "phf_shared", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.19", + "quote 1.0.7", "string_cache_shared", ] @@ -1945,13 +1937,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.17" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" +checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "unicode-xid 0.2.0", + "proc-macro2 1.0.19", + "quote 1.0.7", + "unicode-xid 0.2.1", ] [[package]] @@ -1961,7 +1953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" dependencies = [ "byteorder 1.3.4", - "dirs 1.0.5", + "dirs", "winapi", ] @@ -2013,6 +2005,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" + [[package]] name = "toml" version = "0.5.6" @@ -2024,9 +2022,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" [[package]] name = "uname" @@ -2161,11 +2159,11 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" [[package]] name = "unicode-normalization" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" dependencies = [ - "smallvec", + "tinyvec", ] [[package]] @@ -2176,9 +2174,9 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" @@ -2188,9 +2186,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "unicode_names2" @@ -2216,15 +2214,15 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "void" @@ -2246,9 +2244,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.60" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" +checksum = "f3edbcc9536ab7eababcc6d2374a0b7bfe13a2b6d562c5e07f370456b1a8f33d" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2256,16 +2254,16 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.60" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" +checksum = "89ed2fb8c84bfad20ea66b26a3743f3e7ba8735a69fe7d95118c33ec8fc1244d" dependencies = [ "bumpalo", "lazy_static 1.4.0", "log", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", "wasm-bindgen-shared", ] @@ -2284,38 +2282,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.60" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" +checksum = "eb071268b031a64d92fc6cf691715ca5a40950694d8f683c5bb43db7c730929e" dependencies = [ - "quote 1.0.3", + "quote 1.0.7", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.60" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" +checksum = "cf592c807080719d1ff2f245a687cbadb3ed28b2077ed7084b47aba8b691f2c6" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.35", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.60" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" +checksum = "72b6c0220ded549d63860c78c38f3bcc558d1ca3f4efa74942c536ddbbb55e87" [[package]] name = "web-sys" -version = "0.3.37" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" +checksum = "8be2398f326b7ba09815d0b403095f34dd708579220d099caae89be0b32137b2" dependencies = [ "js-sys", "wasm-bindgen", @@ -2323,9 +2321,9 @@ dependencies = [ [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -2339,9 +2337,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index f76a5da34..8491aea3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ env_logger = "0.7" clap = "2.33" rustpython-compiler = { path = "compiler", version = "0.1.1" } rustpython-parser = { path = "parser", version = "0.1.1" } -rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["rustpython-parser", "rustpython-compiler"] } +rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] } dirs = { package = "dirs-next", version = "1.0" } num-traits = "0.2.8" cfg-if = "0.1" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e848a22ad..9279eadae 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -9,11 +9,12 @@ edition = "2018" include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] [features] -default = ["rustpython-parser", "rustpython-compiler", "threading"] +default = ["compile-parse", "threading"] vm-tracing-logging = [] flame-it = ["flame", "flamer"] freeze-stdlib = ["rustpython-pylib"] threading = ["rustpython-common/threading"] +compile-parse = ["rustpython-parser", "rustpython-compiler"] ssl = ["openssl", "openssl-sys", "openssl-probe"] diff --git a/wasm/lib/Cargo.lock b/wasm/lib/Cargo.lock deleted file mode 100644 index 9bad7eeba..000000000 --- a/wasm/lib/Cargo.lock +++ /dev/null @@ -1,1019 +0,0 @@ -[[package]] -name = "aho-corasick" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "arrayref" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ascii-canvas" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "atty" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "atty" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bit-set" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bit-vec" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "bitflags" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "block-buffer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "byte-tools" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "byteorder" -version = "1.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cc" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cfg-if" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "diff" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "digest" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "docopt" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "either" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ena" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "env_logger" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure_derive" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "fixedbitset" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "generic-array" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "heck" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "humantime" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "itertools" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "itoa" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "js-sys" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lalrpop" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lalrpop-snap" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lalrpop-util" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "lazy_static" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.43" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "new_debug_unreachable" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom" -version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-bigint" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-complex" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-integer" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-traits" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ordermap" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "petgraph" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_generator" -version = "0.7.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_shared" -version = "0.7.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "proc-macro2" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "quick-error" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand_core" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand_core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_syscall" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "redox_syscall 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "regex-syntax" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rustpython_parser" -version = "0.0.1" -dependencies = [ - "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustpython_vm" -version = "0.1.0" -dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustpython_parser 0.0.1", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", - "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustpython_wasm" -version = "0.1.0" -dependencies = [ - "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustpython_parser 0.0.1", - "rustpython_vm 0.1.0", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ryu" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde" -version = "1.0.80" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde_derive" -version = "1.0.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serde_json" -version = "1.0.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sha2" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", - "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "siphasher" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "sourcefile" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "statrs" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "string_cache" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)", - "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "string_cache_codegen" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "string_cache_shared" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "strsim" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "syn" -version = "0.15.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synstructure" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "term" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termcolor" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termion" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "thread_local" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "typenum" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ucd-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-segmentation" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "wasm-bindgen" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "wasm-bindgen-webidl" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "web-sys" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "weedle" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "wincolor" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" -"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" -"checksum ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b385d69402821a1c254533a011a312531cbcc0e3e24f19bbb4747a5a2daf37e2" -"checksum atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0fd4c0631f06448cc45a6bbb3b710ebb7ff8ccb96a0800c994afe23a70d5df2" -"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" -"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" -"checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" -"checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" -"checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" -"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" -"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" -"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" -"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" -"checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" -"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" -"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" -"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" -"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" -"checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5" -"checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e" -"checksum failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6dd377bcc1b1b7ce911967e3ec24fa19c3224394ec05b54aa7b083d498341ac7" -"checksum failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "64c2d913fe8ed3b6c6518eedf4538255b989945c14c2a7d5cbff62a5e2120596" -"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -"checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" -"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" -"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" -"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" -"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" -"checksum js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58cfec35fd4a94f3cf357d5cb7da71c71cd52720c2f2a7320090a8db5f06f655" -"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf" -"checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f" -"checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c" -"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" -"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" -"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -"checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f" -"checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b" -"checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" -"checksum nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9c349f68f25f596b9f44cf0e7c69752a5c633b0550c3ff849518bfba0233774a" -"checksum num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "10b8423ea72ec64751198856a853e07b37087cfc9b53a87ecb19bff67b6d1320" -"checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" -"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" -"checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" -"checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" -"checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" -"checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b" -"checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93" -"checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -"checksum proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "3d7b7eaaa90b4a90a932a9ea6666c95a389e424eff347f0f793979289429feee" -"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" -"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" -"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" -"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" -"checksum redox_syscall 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "679da7508e9a6390aeaf7fbd02a800fdc64b73fe2204dd2c8ae66d22d9d5ad5d" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -"checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341" -"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" -"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" -"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" -"checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" -"checksum serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "15c141fc7027dd265a47c090bf864cf62b42c4d228bbcf4e51a0c9e2b0d3f7ef" -"checksum serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "225de307c6302bec3898c51ca302fc94a7a1697ef0845fcee6448f33c032249c" -"checksum serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "43344e7ce05d0d8280c5940cabb4964bea626aa58b1ec0e8c73fa2a8512a38ce" -"checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" -"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" -"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" -"checksum statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8" -"checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423" -"checksum string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eea1eee654ef80933142157fdad9dd8bc43cf7c74e999e369263496f04ff4da" -"checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" -"checksum syn 0.15.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4439ee8325b4e4b57e59309c3724c9a4478eaeb4eb094b6f3fac180a3b2876" -"checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" -"checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" -"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" -"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" -"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" -"checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" -"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -"checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" -"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" -"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "91f95b8f30407b9ca0c2de157281d3828bbed1fc1f55bea6eb54f40c52ec75ec" -"checksum wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "ab7c242ebcb45bae45340986c48d1853eb2c1c52ff551f7724951b62a2c51429" -"checksum wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "6e353f83716dec9a3597b5719ef88cb6c9e461ec16528f38aa023d3224b4e569" -"checksum wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "3cc90b65fe69c3dd5a09684517dc79f42b847baa2d479c234d125e0a629d9b0a" -"checksum wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "a71a37df4f5845025f96f279d20bbe5b19cbcb77f5410a3a90c6c544d889a162" -"checksum wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b064b8b2336b5a6bf5f31bc95fc1310842395df29877d910cb6f8f791070f319" -"checksum web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d7c588c2e5a091bc4532c5a87032955f9133b644e868b54d08ead0185dcc5b9" -"checksum weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26a4c67f132386d965390b8a734d5d10adbcd30eb5cc74bd9229af8b83f10044" -"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" -"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index e9be7798d..d5f104f57 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -17,7 +17,8 @@ freeze-stdlib = ["rustpython-vm/freeze-stdlib"] [dependencies] rustpython-compiler = { path = "../../compiler" } rustpython-parser = { path = "../../parser" } -rustpython-vm = { path = "../../vm" } +# no threading feature for rustpython-vm -- doesn't much matter anyway, but it might be more optimized +rustpython-vm = { path = "../../vm", default-features = false, features = ["compile-parse"] } cfg-if = "0.1.2" wasm-bindgen = "0.2" wasm-bindgen-futures = "0.3" From 76df8773e7f727447dd43dd9ec394fa849840847 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 10:41:40 -0500 Subject: [PATCH 094/209] Use PyRc in rustpython_wasm --- Cargo.lock | 6 ++++++ wasm/lib/Cargo.toml | 2 ++ wasm/lib/src/browser_module.rs | 4 ++-- wasm/lib/src/js_module.rs | 4 ++-- wasm/lib/src/vm_class.rs | 8 ++++---- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe3dddafd..fbb71e5d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,6 +682,11 @@ name = "instant" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +dependencies = [ + "js-sys", + "wasm-bindgen", + "web-sys", +] [[package]] name = "is-macro" @@ -1681,6 +1686,7 @@ dependencies = [ "futures", "generational-arena", "js-sys", + "parking_lot", "rustpython-compiler", "rustpython-parser", "rustpython-vm", diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index d5f104f57..9f74c5a98 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -27,6 +27,8 @@ serde = "1.0" js-sys = "0.3" futures = "0.1" generational-arena = "0.2" +# make parking_lot use wasm-bingden for instant +parking_lot = { version = "0.11", features = ["wasm-bindgen"] } [dependencies.web-sys] version = "0.3" diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index a58d3f11b..d3b14fa20 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -1,10 +1,10 @@ use futures::Future; use js_sys::Promise; -use std::sync::Arc; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::{future_to_promise, JsFuture}; +use rustpython_vm::common::rc::PyRc; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::import::import_file; use rustpython_vm::obj::{objdict::PyDictRef, objstr::PyStringRef, objtype::PyClassRef}; @@ -377,7 +377,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { } pub fn setup_browser_module(vm: &mut VirtualMachine) { - let state = Arc::get_mut(&mut vm.state).unwrap(); + let state = PyRc::get_mut(&mut vm.state).unwrap(); state .stdlib_inits .insert("_browser".to_owned(), Box::new(make_module)); diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index 9e39e7296..d84a8cadd 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -1,11 +1,11 @@ use js_sys::{Array, Object, Reflect}; +use rustpython_vm::common::rc::PyRc; use rustpython_vm::exceptions::PyBaseExceptionRef; use rustpython_vm::function::Args; use rustpython_vm::obj::{objfloat::PyFloatRef, objstr::PyStringRef, objtype::PyClassRef}; use rustpython_vm::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use rustpython_vm::types::create_type; use rustpython_vm::VirtualMachine; -use std::sync::Arc; use wasm_bindgen::{prelude::*, JsCast}; #[wasm_bindgen(inline_js = " @@ -259,7 +259,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { } pub fn setup_js_module(vm: &mut VirtualMachine) { - let state = Arc::get_mut(&mut vm.state).unwrap(); + let state = PyRc::get_mut(&mut vm.state).unwrap(); state .stdlib_inits .insert("_js".to_owned(), Box::new(make_module)); diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index afc6bb1a8..7e20a2bff 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -1,12 +1,12 @@ use std::cell::RefCell; use std::collections::HashMap; use std::rc::{Rc, Weak}; -use std::sync::Arc; use js_sys::{Object, TypeError}; use wasm_bindgen::prelude::*; use rustpython_compiler::compile; +use rustpython_vm::common::rc::{PyRc, PyWeak}; use rustpython_vm::pyobject::{ItemProtocol, PyObject, PyObjectPayload, PyObjectRef, PyValue}; use rustpython_vm::scope::{NameProtocol, Scope}; use rustpython_vm::{InitParameter, PySettings, VirtualMachine}; @@ -39,7 +39,7 @@ impl StoredVirtualMachine { js_module::setup_js_module(&mut vm); if inject_browser_module { - Arc::get_mut(&mut vm.state).unwrap().stdlib_inits.insert( + PyRc::get_mut(&mut vm.state).unwrap().stdlib_inits.insert( "_window".to_owned(), Box::new(|vm| { py_module!(vm, "_window", { @@ -177,9 +177,9 @@ impl WASMVirtualMachine { pub(crate) fn push_held_rc( &self, obj: PyObjectRef, - ) -> Result>, JsValue> { + ) -> Result>, JsValue> { self.with(|stored_vm| { - let weak = std::sync::Arc::downgrade(&obj); + let weak = PyRc::downgrade(&obj); stored_vm.held_objects.borrow_mut().push(obj); weak }) From 126beb75e002b9ded3d63da16658954a052df2ff Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 25 Jul 2020 15:29:48 -0400 Subject: [PATCH 095/209] Add no-threading compilation check to rust_tests --- .github/workflows/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d01a9b17..d30eb909f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,6 +35,11 @@ jobs: with: command: test args: --verbose ${{ env.CARGO_ARGS }} + - name: check compilation without threading + uses: actions-rs/cargo@v1 + with: + command: check + args: ${{ env.CARGO_ARGS }} --no-default-features snippets_cpython: name: Run snippets and cpython tests From 84aa0e094af61d2b8e1d85e6ef8b05bfb4a363f6 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Sat, 25 Jul 2020 14:33:41 -0700 Subject: [PATCH 096/209] avoid unnecessary cloning --- vm/src/sysmodule.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a68354396..ec3018e9e 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -29,7 +29,7 @@ fn argv(vm: &VirtualMachine) -> PyObjectRef { fn executable(ctx: &PyContext) -> PyObjectRef { if let Ok(path) = env::current_exe() { - if let Some(path) = path.to_str() { + if let Ok(path) = path.into_os_string().into_string() { return ctx.new_str(path); } } From ab926d17f4c7075d9ab6a99bd37a101dd85def2c Mon Sep 17 00:00:00 2001 From: Basix Date: Wed, 15 Jul 2020 10:36:40 +0900 Subject: [PATCH 097/209] Implement sys.getwindowsversion --- vm/src/sysmodule.rs | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index ec3018e9e..f84ca6323 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -5,6 +5,7 @@ use crate::builtins; use crate::frame::FrameRef; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; use crate::pyhash::PyHashInfo; use crate::pyobject::{ IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult, TypeProtocol, @@ -231,6 +232,63 @@ fn sys_displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } +#[pystruct_sequence(name = "sys.getwindowsversion")] +#[derive(Default, Debug)] +#[cfg(windows)] +struct WindowsVersion { + major: u32, + minor: u32, + build: u32, + platform: u32, + service_pack: String, + service_pack_major: u16, + service_pack_minor: u16, + suite_mask: u16, + product_type: u8, + platform_version: (u32, u32, u32), +} + +#[cfg(windows)] +fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { + use winapi::um::{sysinfoapi::GetVersionExW, winnt::{OSVERSIONINFOEXW, LPOSVERSIONINFOW, LPOSVERSIONINFOEXW}}; + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + + let mut version: OSVERSIONINFOEXW = Default::default(); + version.dwOSVersionInfoSize = std::mem::size_of::() as u32; + let result = unsafe { + let osvi = &mut version as LPOSVERSIONINFOEXW as LPOSVERSIONINFOW; + // SAFE: GetVersionExW accepts a pointer of OSVERSIONINFOW, but winapi crate's type currently doesn't allow to do so. + // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getversionexw#parameters + GetVersionExW(osvi) + }; + + if result == 0 { + Err(vm.new_os_error("failed to get windows version".to_owned())) + } else { + let service_pack = { + let sp = OsString::from_wide(&version.szCSDVersion); + if let Ok(string) = sp.into_string() { + string + } else { + Err(vm.new_os_error("service pack is not ASCII".to_owned()))? + } + }; + WindowsVersion { + major: version.dwMajorVersion, + minor: version.dwMinorVersion, + build: version.dwBuildNumber, + platform: version.dwPlatformId, + service_pack, + service_pack_major: version.wServicePackMajor, + service_pack_minor: version.wServicePackMinor, + suite_mask: version.wSuiteMask, + product_type: version.wProductType, + platform_version: (version.dwMajorVersion, version.dwMinorVersion, version.dwBuildNumber) // TODO Provide accurate version, like CPython impl + }.into_struct_sequence(vm, vm.try_class("sys", "windows_version")?) + } +} + const PLATFORM: &str = { cfg_if::cfg_if! { if #[cfg(any(target_os = "linux", target_os = "android"))] { From 9bafa29969edb2b32d8179bd75b65315cde9b6fe Mon Sep 17 00:00:00 2001 From: Basix Date: Wed, 15 Jul 2020 15:54:33 +0900 Subject: [PATCH 098/209] Fix handling of null-terminated string --- vm/src/sysmodule.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index f84ca6323..598510d11 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -267,7 +267,8 @@ fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { Err(vm.new_os_error("failed to get windows version".to_owned())) } else { let service_pack = { - let sp = OsString::from_wide(&version.szCSDVersion); + let (last, _) = version.szCSDVersion.iter().take_while(|&x| x != &0).enumerate().last().unwrap_or((0, &0)); + let sp = OsString::from_wide(&version.szCSDVersion[..last]); if let Ok(string) = sp.into_string() { string } else { From d4d4f7a398f9ff1a81e366ceb13879e7d0b177e1 Mon Sep 17 00:00:00 2001 From: Basix Date: Thu, 16 Jul 2020 21:13:49 +0900 Subject: [PATCH 099/209] Add getwindowsversion in sys module Compatibility note: The return type of this function has a different name from CPython. --- vm/src/sysmodule.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 598510d11..588afd52e 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -232,7 +232,7 @@ fn sys_displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } -#[pystruct_sequence(name = "sys.getwindowsversion")] +#[pystruct_sequence(name = "sys.getwindowsversion_type")] #[derive(Default, Debug)] #[cfg(windows)] struct WindowsVersion { @@ -286,7 +286,7 @@ fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { suite_mask: version.wSuiteMask, product_type: version.wProductType, platform_version: (version.dwMajorVersion, version.dwMinorVersion, version.dwBuildNumber) // TODO Provide accurate version, like CPython impl - }.into_struct_sequence(vm, vm.try_class("sys", "windows_version")?) + }.into_struct_sequence(vm, vm.try_class("sys", "getwindowsversion_type")?) } } @@ -505,6 +505,15 @@ settrace() -- set the global debug tracing function "__displayhook__" => ctx.new_function(sys_displayhook), }); + #[cfg(windows)] + { + let getwindowsversion = WindowsVersion::make_class(ctx); + extend_module!(vm, module, { + "getwindowsversion" => ctx.new_function(sys_getwindowsversion), + "getwindowsversion_type" => getwindowsversion, + }) + } + modules.set_item("sys", module.clone(), vm).unwrap(); modules.set_item("builtins", builtins.clone(), vm).unwrap(); } From 0cc085b3a5059222111b162accc296f64180070f Mon Sep 17 00:00:00 2001 From: Basix Date: Thu, 16 Jul 2020 22:01:53 +0900 Subject: [PATCH 100/209] Fix code style --- vm/src/sysmodule.rs | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 588afd52e..4b3146fa1 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -5,7 +5,6 @@ use crate::builtins; use crate::frame::FrameRef; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::obj::objtuple::PyTupleRef; use crate::pyhash::PyHashInfo; use crate::pyobject::{ IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult, TypeProtocol, @@ -249,30 +248,39 @@ struct WindowsVersion { } #[cfg(windows)] -fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { - use winapi::um::{sysinfoapi::GetVersionExW, winnt::{OSVERSIONINFOEXW, LPOSVERSIONINFOW, LPOSVERSIONINFOEXW}}; +fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { use std::ffi::OsString; use std::os::windows::ffi::OsStringExt; + use winapi::um::{ + sysinfoapi::GetVersionExW, + winnt::{LPOSVERSIONINFOEXW, LPOSVERSIONINFOW, OSVERSIONINFOEXW}, + }; - let mut version: OSVERSIONINFOEXW = Default::default(); + let mut version = OSVERSIONINFOEXW::default(); version.dwOSVersionInfoSize = std::mem::size_of::() as u32; let result = unsafe { - let osvi = &mut version as LPOSVERSIONINFOEXW as LPOSVERSIONINFOW; + let osvi = &mut version as LPOSVERSIONINFOEXW as LPOSVERSIONINFOW; // SAFE: GetVersionExW accepts a pointer of OSVERSIONINFOW, but winapi crate's type currently doesn't allow to do so. // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getversionexw#parameters - GetVersionExW(osvi) + GetVersionExW(osvi) }; if result == 0 { Err(vm.new_os_error("failed to get windows version".to_owned())) } else { let service_pack = { - let (last, _) = version.szCSDVersion.iter().take_while(|&x| x != &0).enumerate().last().unwrap_or((0, &0)); + let (last, _) = version + .szCSDVersion + .iter() + .take_while(|&x| x != &0) + .enumerate() + .last() + .unwrap_or((0, &0)); let sp = OsString::from_wide(&version.szCSDVersion[..last]); if let Ok(string) = sp.into_string() { string } else { - Err(vm.new_os_error("service pack is not ASCII".to_owned()))? + return Err(vm.new_os_error("service pack is not ASCII".to_owned())); } }; WindowsVersion { @@ -285,8 +293,13 @@ fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult { service_pack_minor: version.wServicePackMinor, suite_mask: version.wSuiteMask, product_type: version.wProductType, - platform_version: (version.dwMajorVersion, version.dwMinorVersion, version.dwBuildNumber) // TODO Provide accurate version, like CPython impl - }.into_struct_sequence(vm, vm.try_class("sys", "getwindowsversion_type")?) + platform_version: ( + version.dwMajorVersion, + version.dwMinorVersion, + version.dwBuildNumber, + ), // TODO Provide accurate version, like CPython impl + } + .into_struct_sequence(vm, vm.try_class("sys", "getwindowsversion_type")?) } } @@ -510,7 +523,7 @@ settrace() -- set the global debug tracing function let getwindowsversion = WindowsVersion::make_class(ctx); extend_module!(vm, module, { "getwindowsversion" => ctx.new_function(sys_getwindowsversion), - "getwindowsversion_type" => getwindowsversion, + "getwindowsversion_type" => getwindowsversion, }) } From ac57b5225d785dc99af6f51fd72955f9a06fab30 Mon Sep 17 00:00:00 2001 From: Basix Date: Sun, 19 Jul 2020 14:03:50 +0900 Subject: [PATCH 101/209] Rename getwindowsversion_type --- vm/src/sysmodule.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 4b3146fa1..6e51b144c 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -231,7 +231,7 @@ fn sys_displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } -#[pystruct_sequence(name = "sys.getwindowsversion_type")] +#[pystruct_sequence(name = "sys.getwindowsversion")] #[derive(Default, Debug)] #[cfg(windows)] struct WindowsVersion { @@ -299,7 +299,7 @@ fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult ctx.new_function(sys_getwindowsversion), - "getwindowsversion_type" => getwindowsversion, + "_getwindowsversion_type" => getwindowsversion, // XXX: This is not a python spec but required by current RustPython implementation }) } From 1505c9a88e3ac7577c716415be0c48d579cd3b79 Mon Sep 17 00:00:00 2001 From: Basix Date: Sat, 25 Jul 2020 00:03:48 +0900 Subject: [PATCH 102/209] Add tests --- tests/snippets/stdlib_os.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ee762d92e..5f312efb8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -491,3 +491,26 @@ if "win" not in sys.platform: for arg in [None, 1, 1.0, TabError]: assert_raises(TypeError, os.system, arg) + +if sys.platform.startswith("win"): + winver = sys.getwindowsversion() + + # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members). + all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \ + 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 + + # We really can't test if the results are correct, so it just checks for meaningful value + assert winver.major > 0 + assert winver.minor >= 0 + assert winver.build > 0 + assert winver.platform == 2 + assert isinstance(winver.service_pack, str) + assert 0 <= winver.suite_mask <= all_masks + assert 1 <= winver.product_type <= 3 + + # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest + assert winver.major == winver.platform_version[0] + assert winver.minor == winver.platform_version[1] + assert winver.build == winver.platform_version[2] + + From 240a31e112618b8862cee98ae9a027e57d2ce36b Mon Sep 17 00:00:00 2001 From: Basix Date: Sun, 26 Jul 2020 17:11:20 +0900 Subject: [PATCH 103/209] Use module parameter for WindowsVersion --- vm/src/sysmodule.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 6e51b144c..3db27aefa 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -231,7 +231,7 @@ fn sys_displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } -#[pystruct_sequence(name = "sys.getwindowsversion")] +#[pystruct_sequence(module = "sys", name = "getwindowsversion")] #[derive(Default, Debug)] #[cfg(windows)] struct WindowsVersion { From 19326d8411ff88908be9674b68bf86c5f0367717 Mon Sep 17 00:00:00 2001 From: hyperbora Date: Sun, 26 Jul 2020 21:37:49 +0900 Subject: [PATCH 104/209] fix a divmod bug (#2025) --- Lib/test/test_builtin.py | 2 -- vm/src/obj/objfloat.rs | 8 +++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 54834fa1a..546c03371 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -446,8 +446,6 @@ class BuiltinTest(unittest.TestCase): # test that object has a __dir__() self.assertEqual(sorted([].__dir__()), dir([])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) self.assertEqual(divmod(-12, 7), (-2, 2)) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 20ddc97c4..8b30ade4d 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -123,7 +123,13 @@ fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { if v2 != 0.0 { - Ok(((v1 / v2).floor(), v1 % v2)) + let mut m = v1 % v2; + let mut d = (v1 - m) / v2; + if v2.is_sign_negative() != m.is_sign_negative() { + m += v2; + d -= 1.0; + } + Ok((d, m)) } else { Err(vm.new_zero_division_error("float divmod()".to_owned())) } From a1de3a5483b0f61cc172a0ca63778617f156ac6c Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:23:49 -0500 Subject: [PATCH 105/209] Update flamer + flame some more functions --- Cargo.lock | 101 ++++++++++++++-------------------------- vm/Cargo.toml | 3 +- vm/src/obj/objobject.rs | 1 + 3 files changed, 39 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbb71e5d6..c08e5002c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,8 +53,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0609c78bd572f4edc74310dfb63a01f5609d53fa8b4dd7c4d98aef3b3e8d72d1" dependencies = [ "proc-macro-hack", - "quote 1.0.7", - "syn 1.0.35", + "quote", + "syn", ] [[package]] @@ -385,9 +385,9 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -521,13 +521,13 @@ dependencies = [ [[package]] name = "flamer" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2add1a5e84b1ed7b5d00cdc21789a28e0a8f4e427b677313c773880ba3c4dac" +checksum = "36b732da54fd4ea34452f2431cf464ac7be94ca4b339c9cd3d3d12eb06fe7aab" dependencies = [ "flame", - "quote 0.6.13", - "syn 0.15.44", + "quote", + "syn", ] [[package]] @@ -696,9 +696,9 @@ checksum = "04807f3dc9e3ea39af3f8469a5297267faf94859637afb836b33f47d9b2650ee" dependencies = [ "Inflector", "pmutil", - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1009,9 +1009,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1161,9 +1161,9 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1193,15 +1193,6 @@ version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - [[package]] name = "proc-macro2" version = "1.0.19" @@ -1227,22 +1218,13 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.19", + "proc-macro2", ] [[package]] @@ -1549,11 +1531,11 @@ version = "0.1.2" dependencies = [ "maplit", "once_cell", - "proc-macro2 1.0.19", - "quote 1.0.7", + "proc-macro2", + "quote", "rustpython-bytecode", "rustpython-compiler", - "syn 1.0.35", + "syn", ] [[package]] @@ -1780,9 +1762,9 @@ version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1901,8 +1883,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator", "phf_shared", - "proc-macro2 1.0.19", - "quote 1.0.7", + "proc-macro2", + "quote", "string_cache_shared", ] @@ -1930,25 +1912,14 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", -] - [[package]] name = "syn" version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", + "proc-macro2", + "quote", "unicode-xid 0.2.1", ] @@ -2267,9 +2238,9 @@ dependencies = [ "bumpalo", "lazy_static 1.4.0", "log", - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-shared", ] @@ -2292,7 +2263,7 @@ version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb071268b031a64d92fc6cf691715ca5a40950694d8f683c5bb43db7c730929e" dependencies = [ - "quote 1.0.7", + "quote", "wasm-bindgen-macro-support", ] @@ -2302,9 +2273,9 @@ version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf592c807080719d1ff2f245a687cbadb3ed28b2077ed7084b47aba8b691f2c6" dependencies = [ - "proc-macro2 1.0.19", - "quote 1.0.7", - "syn 1.0.35", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 9279eadae..ccd5bbdf0 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -92,7 +92,8 @@ unic-ucd-age = "0.9" unic-ucd-ident = "0.9" flame = { version = "0.2", optional = true } -flamer = { version = "0.3", optional = true } +flamer = { version = "0.4", optional = true } + rustpython-common = { path = "../common" } [target.'cfg(unix)'.dependencies] diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 4c1149681..8428fa116 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -253,6 +253,7 @@ impl PyBaseObject { } } +#[cfg_attr(feature = "flame-it", flame)] pub(crate) fn setattr( obj: PyObjectRef, attr_name: PyStringRef, From 861532990dc157c9bbf77a511d3e85d19fdcd992 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:27:48 -0500 Subject: [PATCH 106/209] Update lalrpop --- Cargo.lock | 280 +++++++++------------------------------------- parser/Cargo.toml | 4 +- 2 files changed, 57 insertions(+), 227 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c08e5002c..235183ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,12 +98,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.0" @@ -283,15 +277,6 @@ dependencies = [ "vec_map", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "cloudabi" version = "0.1.0" @@ -342,7 +327,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg 1.0.0", + "autocfg", "cfg-if", "lazy_static 1.4.0", ] @@ -468,9 +453,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" [[package]] name = "ena" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36" +checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3" dependencies = [ "log", ] @@ -502,9 +487,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fixedbitset" -version = "0.1.9" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "flame" @@ -576,12 +561,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures" version = "0.1.29" @@ -634,7 +613,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb" dependencies = [ - "autocfg 1.0.0", + "autocfg", ] [[package]] @@ -673,7 +652,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7" dependencies = [ - "autocfg 1.0.0", + "autocfg", "hashbrown", ] @@ -710,6 +689,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -733,9 +721,9 @@ checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" [[package]] name = "lalrpop" -version = "0.17.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64dc3698e75d452867d9bd86f4a723f452ce9d01fe1d55990b79f0c790aa67db" +checksum = "d6f55673d283313791404be21209bb433f128f7e5c451986df107eb5fdbd68d2" dependencies = [ "ascii-canvas", "atty", @@ -743,7 +731,7 @@ dependencies = [ "diff", "docopt", "ena", - "itertools", + "itertools 0.9.0", "lalrpop-util", "petgraph", "regex", @@ -753,14 +741,14 @@ dependencies = [ "sha2", "string_cache", "term", - "unicode-xid 0.1.0", + "unicode-xid", ] [[package]] name = "lalrpop-util" -version = "0.17.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c277d18683b36349ab5cd030158b54856fca6bb2d5dc5263b06288f486958b7c" +checksum = "f7e88f15a7d31dfa8fb607986819039127f0161058a3b248a146142d276cbd28" [[package]] name = "lazy_static" @@ -888,8 +876,8 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c674293daac706360a8fa633c802ca15d27ee4a52394f12ecec2f6d2aa5508bf" dependencies = [ - "rand 0.7.3", - "rand_core 0.5.1", + "rand", + "rand_core", ] [[package]] @@ -923,7 +911,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f3fc75e3697059fb1bc465e3d8cca6cf92f56854f201158b3f9c77d5a3cfa0" dependencies = [ - "autocfg 1.0.0", + "autocfg", "num-integer", "num-traits", "serde", @@ -935,7 +923,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ - "autocfg 1.0.0", + "autocfg", "num-traits", "serde", ] @@ -946,7 +934,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ - "autocfg 1.0.0", + "autocfg", "num-traits", ] @@ -956,7 +944,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f" dependencies = [ - "autocfg 1.0.0", + "autocfg", "num-integer", "num-traits", ] @@ -967,7 +955,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138" dependencies = [ - "autocfg 1.0.0", + "autocfg", "num-bigint", "num-integer", "num-traits", @@ -979,7 +967,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ - "autocfg 1.0.0", + "autocfg", ] [[package]] @@ -1061,7 +1049,7 @@ version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ - "autocfg 1.0.0", + "autocfg", "cc", "libc", "openssl-src", @@ -1069,12 +1057,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "ordermap" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" - [[package]] name = "parking_lot" version = "0.11.0" @@ -1093,7 +1075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ "cfg-if", - "cloudabi 0.1.0", + "cloudabi", "instant", "libc", "redox_syscall", @@ -1122,29 +1104,19 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.4.13" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" dependencies = [ "fixedbitset", - "ordermap", -] - -[[package]] -name = "phf_generator" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" -dependencies = [ - "phf_shared", - "rand 0.6.5", + "indexmap", ] [[package]] name = "phf_shared" -version = "0.7.24" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" dependencies = [ "siphasher", ] @@ -1199,7 +1171,7 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid", ] [[package]] @@ -1227,25 +1199,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.7", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -1254,19 +1207,9 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.3.1", + "rand_chacha", + "rand_core", + "rand_hc", ] [[package]] @@ -1276,24 +1219,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core 0.5.1", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -1303,84 +1231,13 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi 0.0.3", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", + "rand_core", ] [[package]] @@ -1492,7 +1349,7 @@ version = "0.1.2" dependencies = [ "bincode", "bitflags", - "itertools", + "itertools 0.8.2", "lz4-compress", "num-bigint", "num-complex", @@ -1509,7 +1366,7 @@ dependencies = [ "num-bigint", "num-traits", "parking_lot", - "rand 0.7.3", + "rand", ] [[package]] @@ -1518,7 +1375,7 @@ version = "0.1.2" dependencies = [ "arrayvec 0.5.1", "indexmap", - "itertools", + "itertools 0.8.2", "log", "num-complex", "rustpython-bytecode", @@ -1593,7 +1450,7 @@ dependencies = [ "hexf-parse", "indexmap", "is-macro", - "itertools", + "itertools 0.8.2", "lexical", "libc", "libz-sys", @@ -1616,8 +1473,8 @@ dependencies = [ "openssl-sys", "parking_lot", "paste", - "rand 0.7.3", - "rand_core 0.5.1", + "rand", + "rand_core", "regex", "result-like", "rustc_version_runtime", @@ -1817,9 +1674,9 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.2.3" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" [[package]] name = "smallbox" @@ -1857,43 +1714,22 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cce16f6de653e88beca7bd13780d08e09d4489dbca1f9210e041bc4852481382" dependencies = [ - "rand 0.7.3", + "rand", ] [[package]] name = "string_cache" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" +checksum = "2940c75beb4e3bf3a494cef919a747a2cb81e52571e212bfbd185074add7208a" dependencies = [ "lazy_static 1.4.0", "new_debug_unreachable", "phf_shared", "precomputed-hash", "serde", - "string_cache_codegen", - "string_cache_shared", ] -[[package]] -name = "string_cache_codegen" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "string_cache_shared", -] - -[[package]] -name = "string_cache_shared" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" - [[package]] name = "strsim" version = "0.8.0" @@ -1920,7 +1756,7 @@ checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" dependencies = [ "proc-macro2", "quote", - "unicode-xid 0.2.1", + "unicode-xid", ] [[package]] @@ -2155,12 +1991,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode-xid" version = "0.2.1" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index da88a7586..76a62c680 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -9,10 +9,10 @@ license = "MIT" edition = "2018" [build-dependencies] -lalrpop="0.17" +lalrpop="0.19" [dependencies] -lalrpop-util = "0.17" +lalrpop-util = "0.19" log = "0.4.1" num-bigint = "0.3" num-traits = "0.2" From ab232e6cd8a9623b5ff8b395ec05892bd5dbfe06 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:33:10 -0500 Subject: [PATCH 107/209] Use lexical-core instead of lexical --- Cargo.lock | 46 +++++++++-------------------------------- common/Cargo.toml | 2 +- common/src/float_ops.rs | 2 +- vm/Cargo.toml | 2 +- vm/src/lib.rs | 1 - vm/src/obj/objfloat.rs | 2 +- 6 files changed, 14 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 235183ac1..df9bfd9e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,15 +63,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" -[[package]] -name = "arrayvec" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" -dependencies = [ - "nodrop", -] - [[package]] name = "arrayvec" version = "0.5.1" @@ -160,7 +151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" dependencies = [ "arrayref", - "arrayvec 0.5.1", + "arrayvec", "constant_time_eq", ] @@ -762,26 +753,15 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lexical" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afaeae1c07c575338ef6809875bfea8daa9ea8b2ee381ef1f93ba0c6e32f003" -dependencies = [ - "cfg-if", - "lexical-core", - "rustc_version", -] - [[package]] name = "lexical-core" -version = "0.6.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7043aa5c05dd34fb73b47acb8c3708eac428de4545ea3682ed2f11293ebd890" +checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" dependencies = [ - "arrayvec 0.4.12", + "arrayvec", + "bitflags", "cfg-if", - "rustc_version", "ryu", "static_assertions", ] @@ -899,12 +879,6 @@ dependencies = [ "void", ] -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "num-bigint" version = "0.3.0" @@ -1362,7 +1336,7 @@ version = "0.0.0" dependencies = [ "cfg-if", "hexf-parse", - "lexical", + "lexical-core", "num-bigint", "num-traits", "parking_lot", @@ -1373,7 +1347,7 @@ dependencies = [ name = "rustpython-compiler" version = "0.1.2" dependencies = [ - "arrayvec 0.5.1", + "arrayvec", "indexmap", "itertools 0.8.2", "log", @@ -1451,7 +1425,7 @@ dependencies = [ "indexmap", "is-macro", "itertools 0.8.2", - "lexical", + "lexical-core", "libc", "libz-sys", "log", @@ -1704,9 +1678,9 @@ dependencies = [ [[package]] name = "static_assertions" -version = "0.3.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "statrs" diff --git a/common/Cargo.toml b/common/Cargo.toml index b8603397a..951660bfb 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -11,7 +11,7 @@ threading = ["parking_lot"] parking_lot = { version = "0.11.0", optional = true } num-traits = "0.2" num-bigint = "0.3" -lexical = "4" +lexical-core = "0.7" hexf-parse = "0.1.0" cfg-if = "0.1" diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs index 34f2c4e6a..d7ebadee2 100644 --- a/common/src/float_ops.rs +++ b/common/src/float_ops.rs @@ -72,7 +72,7 @@ pub fn parse_str(literal: &str) -> Option { last_tok = Some(c); } - if let Ok(f) = lexical::parse(buf.as_str()) { + if let Ok(f) = lexical_core::parse(buf.as_bytes()) { Some(f) } else { None diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ccd5bbdf0..05c21cd8e 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -52,7 +52,7 @@ statrs = "0.12.0" caseless = "0.2.1" chrono = { version = "0.4", features = ["wasmbind"] } once_cell = "1.3.1" -lexical = "4" +lexical-core = "0.7" itertools = "0.8" hex = "0.4.0" hexf-parse = "0.1.0" diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 80724fd85..014515018 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -20,7 +20,6 @@ extern crate flamer; #[macro_use] extern crate bitflags; -extern crate lexical; #[macro_use] extern crate log; #[macro_use] diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index dab3fe265..d3c162f5a 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -513,7 +513,7 @@ fn to_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { vm.new_value_error(format!("could not convert string to float: '{}'", s)) })? } else if let Some(bytes) = obj.payload_if_subclass::(vm) { - lexical::parse(bytes.get_value()).map_err(|_| { + lexical_core::parse(bytes.get_value()).map_err(|_| { vm.new_value_error(format!( "could not convert string to float: '{}'", bytes.repr() From fcad7e00911139a55dcbdb0d3483c29ea4e46fac Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:46:11 -0500 Subject: [PATCH 108/209] Use lz4-compression instead of lz4-compress --- Cargo.lock | 28 +++++++++------------------- bytecode/Cargo.toml | 2 +- bytecode/src/bytecode.rs | 5 +++-- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df9bfd9e7..edd76527f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,7 +107,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" dependencies = [ - "byteorder 1.3.4", + "byteorder", "serde", ] @@ -163,7 +163,7 @@ checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ "block-padding", "byte-tools", - "byteorder 1.3.4", + "byteorder", "generic-array", ] @@ -206,12 +206,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -[[package]] -name = "byteorder" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" - [[package]] name = "byteorder" version = "1.3.4" @@ -803,14 +797,10 @@ dependencies = [ ] [[package]] -name = "lz4-compress" -version = "0.1.1" +name = "lz4-compression" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f966533a922a9bba9e95e594c1fdb3b9bf5fdcdb11e37e51ad84cd76e468b91" -dependencies = [ - "byteorder 0.5.3", - "quick-error", -] +checksum = "761104bf97f13a3caf47d822498a0760a10d00d220148bac2669f63fc3bb8270" [[package]] name = "maplit" @@ -1249,7 +1239,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" dependencies = [ - "byteorder 1.3.4", + "byteorder", ] [[package]] @@ -1324,7 +1314,7 @@ dependencies = [ "bincode", "bitflags", "itertools 0.8.2", - "lz4-compress", + "lz4-compression", "num-bigint", "num-complex", "serde", @@ -1402,7 +1392,7 @@ dependencies = [ "bitflags", "blake2", "bstr", - "byteorder 1.3.4", + "byteorder", "caseless", "cfg-if", "chrono", @@ -1739,7 +1729,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" dependencies = [ - "byteorder 1.3.4", + "byteorder", "dirs", "winapi", ] diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml index f46fd4e79..79a9a73f8 100644 --- a/bytecode/Cargo.toml +++ b/bytecode/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" [dependencies] bincode = "1.1" bitflags = "1.1" -lz4-compress = "0.1.1" +lz4-compression = "0.7" num-bigint = { version = "0.3", features = ["serde"] } num-complex = { version = "0.2", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 09d0e03d5..a931e888c 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -409,14 +409,15 @@ impl CodeObject { /// Load a code object from bytes pub fn from_bytes(data: &[u8]) -> Result> { - let data = lz4_compress::decompress(data)?; + let data = lz4_compression::decompress::decompress(data) + .map_err(|e| format!("lz4 error: {:?}", e))?; bincode::deserialize::(&data).map_err(|e| e.into()) } /// Serialize this bytecode to bytes. pub fn to_bytes(&self) -> Vec { let data = bincode::serialize(&self).expect("Code object must be serializable"); - lz4_compress::compress(&data) + lz4_compression::compress::compress(&data) } pub fn get_constants(&self) -> impl Iterator { From 2bc2d1721e42ca1dabea61e9f8eeb7899d2c6e47 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 21 Jul 2020 00:47:22 -0500 Subject: [PATCH 109/209] Update itertools --- Cargo.lock | 17 ++++------------- bytecode/Cargo.toml | 2 +- compiler/Cargo.toml | 2 +- vm/Cargo.toml | 4 ++-- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edd76527f..37bf69972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,15 +665,6 @@ dependencies = [ "syn", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.9.0" @@ -716,7 +707,7 @@ dependencies = [ "diff", "docopt", "ena", - "itertools 0.9.0", + "itertools", "lalrpop-util", "petgraph", "regex", @@ -1313,7 +1304,7 @@ version = "0.1.2" dependencies = [ "bincode", "bitflags", - "itertools 0.8.2", + "itertools", "lz4-compression", "num-bigint", "num-complex", @@ -1339,7 +1330,7 @@ version = "0.1.2" dependencies = [ "arrayvec", "indexmap", - "itertools 0.8.2", + "itertools", "log", "num-complex", "rustpython-bytecode", @@ -1414,7 +1405,7 @@ dependencies = [ "hexf-parse", "indexmap", "is-macro", - "itertools 0.8.2", + "itertools", "lexical-core", "libc", "libz-sys", diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml index 79a9a73f8..95206a3bf 100644 --- a/bytecode/Cargo.toml +++ b/bytecode/Cargo.toml @@ -15,4 +15,4 @@ lz4-compression = "0.7" num-bigint = { version = "0.3", features = ["serde"] } num-complex = { version = "0.2", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } -itertools = "0.8" +itertools = "0.9" diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index d968cb119..6846c4386 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] indexmap = "1.0" -itertools = "0.8.0" +itertools = "0.9" rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } rustpython-parser = { path = "../parser", version = "0.1.1" } num-complex = { version = "0.2", features = ["serde"] } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 05c21cd8e..f7a9ac5a8 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -53,7 +53,7 @@ caseless = "0.2.1" chrono = { version = "0.4", features = ["wasmbind"] } once_cell = "1.3.1" lexical-core = "0.7" -itertools = "0.8" +itertools = "0.9" hex = "0.4.0" hexf-parse = "0.1.0" indexmap = "1.0.2" @@ -135,4 +135,4 @@ features = [ wasm-bindgen = "0.2" [build-dependencies] -itertools = "0.8" +itertools = "0.9" From 2d01cffbf95190ff84944f00742a03ec35d240fd Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 25 Jul 2020 04:35:42 +0900 Subject: [PATCH 110/209] common::str::get_chars with test --- common/src/lib.rs | 1 + common/src/str.rs | 30 ++++++++++++++++++++++++++++++ vm/src/obj/objstr.rs | 11 +---------- 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 common/src/str.rs diff --git a/common/src/lib.rs b/common/src/lib.rs index f6d704744..7519f9d94 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -4,3 +4,4 @@ pub mod cell; pub mod float_ops; pub mod hash; pub mod rc; +pub mod str; diff --git a/common/src/str.rs b/common/src/str.rs new file mode 100644 index 000000000..7dbc870a3 --- /dev/null +++ b/common/src/str.rs @@ -0,0 +1,30 @@ +pub fn get_chars(s: &str, range: std::ops::Range) -> &str { + let mut chars = s.chars(); + for _ in 0..range.start { + let _ = chars.next(); + } + let start = chars.as_str(); + for _ in range { + let _ = chars.next(); + } + let end = chars.as_str(); + &start[..start.len() - end.len()] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_chars() { + let s = "0123456789"; + assert_eq!(get_chars(s, 3..7), "3456"); + assert_eq!(get_chars(s, 3..7), &s[3..7]); + + let s = "0유니코드 문자열9"; + assert_eq!(get_chars(s, 3..7), "코드 문"); + + let s = "0😀😃😄😁😆😅😂🤣9"; + assert_eq!(get_chars(s, 3..7), "😄😁😆😅"); + } +} diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index dd905416c..357da1bf6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1406,16 +1406,7 @@ impl<'s> PyCommonString<'s, char> for str { } fn get_chars<'a>(&'a self, range: std::ops::Range) -> &'a Self { - let mut chars = self.chars(); - for _ in 0..range.start { - let _ = chars.next(); - } - let start = chars.as_str(); - for _ in range { - let _ = chars.next(); - } - let end = chars.as_str(); - &start[..start.len() - end.len()] + rustpython_common::str::get_chars(self, range) } fn is_empty(&self) -> bool { From 8ea113608b965e532db5436786d3a8d3d84fb27d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 07:32:48 +0900 Subject: [PATCH 111/209] zfill to common --- common/src/str.rs | 18 ++++++++++++++++++ vm/src/obj/pystr.rs | 17 +---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/common/src/str.rs b/common/src/str.rs index 7dbc870a3..a89c25f49 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -11,6 +11,24 @@ pub fn get_chars(s: &str, range: std::ops::Range) -> &str { &start[..start.len() - end.len()] } +pub fn zfill(bytes: &[u8], width: usize) -> Vec { + if width <= bytes.len() { + bytes.to_vec() + } else { + let (sign, s) = match bytes.first() { + Some(_sign @ b'+') | Some(_sign @ b'-') => { + (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) + } + _ => (&b""[..], bytes), + }; + let mut filled = Vec::new(); + filled.extend_from_slice(sign); + filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); + filled.extend_from_slice(s); + filled + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 3b37d4c9e..93ecf4de6 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -393,23 +393,8 @@ where } fn py_zfill(&self, width: isize) -> Vec { - let bytes = self.as_bytes(); let width = width.to_usize().unwrap_or(0); - if width <= bytes.len() { - bytes.to_vec() - } else { - let (sign, s) = match bytes.first() { - Some(_sign @ b'+') | Some(_sign @ b'-') => { - (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) - } - _ => (&b""[..], bytes), - }; - let mut filled = Vec::new(); - filled.extend_from_slice(sign); - filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); - filled.extend_from_slice(s); - filled - } + rustpython_common::str::zfill(self.as_bytes(), width) } fn py_iscase(&'s self, is_case: F, is_opposite: G) -> bool From e420db65fe73eeb12387de8ae45251ec2884f983 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 25 Jul 2020 04:42:36 +0900 Subject: [PATCH 112/209] PySliceableSequence for PyString reuse len cache --- vm/src/obj/objstr.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 357da1bf6..33a1f7f49 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -253,7 +253,7 @@ impl PyString { } } SequenceIndex::Slice(slice) => { - let string = self.value.get_slice_items(vm, &slice)?; + let string = self.get_slice_items(vm, &slice)?; Ok(vm.new_str(string)) } } @@ -1206,20 +1206,21 @@ pub fn borrow_value(obj: &PyObjectRef) -> &str { &obj.payload::().unwrap().value } -impl PySliceableSequence for str { +impl PySliceableSequence for PyString { type Sliced = String; fn do_slice(&self, range: Range) -> Self::Sliced { - self.chars() + self.value + .chars() .skip(range.start) .take(range.end - range.start) .collect() } fn do_slice_reverse(&self, range: Range) -> Self::Sliced { - let count = self.chars().count(); - - self.chars() + let count = self.len(); + self.value + .chars() .rev() .skip(count - range.end) .take(range.end - range.start) @@ -1227,7 +1228,8 @@ impl PySliceableSequence for str { } fn do_stepped_slice(&self, range: Range, step: usize) -> Self::Sliced { - self.chars() + self.value + .chars() .skip(range.start) .take(range.end - range.start) .step_by(step) @@ -1235,9 +1237,9 @@ impl PySliceableSequence for str { } fn do_stepped_slice_reverse(&self, range: Range, step: usize) -> Self::Sliced { - let count = self.chars().count(); - - self.chars() + let count = self.len(); + self.value + .chars() .rev() .skip(count - range.end) .take(range.end - range.start) @@ -1246,15 +1248,15 @@ impl PySliceableSequence for str { } fn empty() -> Self::Sliced { - String::default() + String::new() } fn len(&self) -> usize { - self.chars().count() + self.len() } fn is_empty(&self) -> bool { - self.is_empty() + self.value.is_empty() } } From 6d26cfa20a379513279ef2c4eba498153ec5366f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 25 Jul 2020 04:50:32 +0900 Subject: [PATCH 113/209] Clean up str/bytes pad to be prefixed with _ and embed get_fillchar --- vm/src/obj/objbyteinner.rs | 8 ++++---- vm/src/obj/objstr.rs | 28 ++++++++++++---------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4e5428d29..aded4afef 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -671,7 +671,7 @@ impl PyByteInner { } #[inline] - fn pad( + fn _pad( &self, options: ByteInnerPaddingOptions, pad: fn(&[u8], usize, u8, usize) -> Vec, @@ -690,7 +690,7 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - self.pad(options, PyCommonString::::py_center, vm) + self._pad(options, PyCommonString::::py_center, vm) } pub fn ljust( @@ -698,7 +698,7 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - self.pad(options, PyCommonString::::py_ljust, vm) + self._pad(options, PyCommonString::::py_ljust, vm) } pub fn rjust( @@ -706,7 +706,7 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - self.pad(options, PyCommonString::::py_rjust, vm) + self._pad(options, PyCommonString::::py_rjust, vm) } pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 33a1f7f49..51ecd136d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -895,26 +895,22 @@ impl PyString { unsafe { String::from_utf8_unchecked(self.value.py_zfill(width)) } } - fn get_fill_char(fillchar: OptionalArg, vm: &VirtualMachine) -> PyResult { - match fillchar { - OptionalArg::Present(ref s) => s.value.chars().exactly_one().map_err(|_| { - vm.new_type_error( - "The fill character must be exactly one character long".to_owned(), - ) - }), - OptionalArg::Missing => Ok(' '), - } - } - #[inline] - fn pad( + fn _pad( &self, width: isize, fillchar: OptionalArg, pad: fn(&str, usize, char, usize) -> String, vm: &VirtualMachine, ) -> PyResult { - let fillchar = Self::get_fill_char(fillchar, vm)?; + let fillchar = match fillchar { + OptionalArg::Present(ref s) => s.value.chars().exactly_one().map_err(|_| { + vm.new_type_error( + "The fill character must be exactly one character long".to_owned(), + ) + }), + OptionalArg::Missing => Ok(' '), + }?; Ok(if self.len() as isize >= width { String::from(&self.value) } else { @@ -929,7 +925,7 @@ impl PyString { fillchar: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.pad(width, fillchar, PyCommonString::::py_center, vm) + self._pad(width, fillchar, PyCommonString::::py_center, vm) } #[pymethod] @@ -939,7 +935,7 @@ impl PyString { fillchar: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.pad(width, fillchar, PyCommonString::::py_ljust, vm) + self._pad(width, fillchar, PyCommonString::::py_ljust, vm) } #[pymethod] @@ -949,7 +945,7 @@ impl PyString { fillchar: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.pad(width, fillchar, PyCommonString::::py_rjust, vm) + self._pad(width, fillchar, PyCommonString::::py_rjust, vm) } #[pymethod] From 58bfd1d2216f6d3979454141497864268f8d1bcf Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 25 Jul 2020 05:58:32 +0900 Subject: [PATCH 114/209] {str,bytes}.partition into pystr --- vm/src/obj/objbytearray.rs | 2 +- vm/src/obj/objbyteinner.rs | 34 +++++++-------------- vm/src/obj/objbytes.rs | 2 +- vm/src/obj/objstr.rs | 60 +++++++++++++++++--------------------- vm/src/obj/pystr.rs | 24 +++++++++++++++ 5 files changed, 62 insertions(+), 60 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 1be33d24f..b6c176893 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -432,7 +432,7 @@ impl PyByteArray { #[pymethod(name = "rpartition")] fn rpartition(&self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { let value = self.borrow_value(); - let (front, has_mid, back) = value.rpartition(&sep, vm)?; + let (back, has_mid, front) = value.rpartition(&sep, vm)?; Ok(vm.ctx.new_tuple(vec![ vm.ctx.new_bytearray(front.to_vec()), vm.ctx diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index aded4afef..8076801f2 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -869,18 +869,11 @@ impl PyByteInner { sub: &PyByteInner, vm: &VirtualMachine, ) -> PyResult<(Vec, bool, Vec)> { - if sub.elements.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - - let mut sp = self.elements.splitn_str(2, &sub.elements); - let front = sp.next().unwrap().to_vec(); - let (has_mid, back) = if let Some(back) = sp.next() { - (true, back.to_vec()) - } else { - (false, Vec::new()) - }; - Ok((front, has_mid, back)) + self.elements.py_partition( + &sub.elements, + || self.elements.splitn_str(2, &sub.elements), + vm, + ) } pub fn rpartition( @@ -888,18 +881,11 @@ impl PyByteInner { sub: &PyByteInner, vm: &VirtualMachine, ) -> PyResult<(Vec, bool, Vec)> { - if sub.elements.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - - let mut sp = self.elements.rsplitn_str(2, &sub.elements); - let back = sp.next().unwrap().to_vec(); - let (has_mid, front) = if let Some(front) = sp.next() { - (true, front.to_vec()) - } else { - (false, Vec::new()) - }; - Ok((front, has_mid, back)) + self.elements.py_partition( + &sub.elements, + || self.elements.rsplitn_str(2, &sub.elements), + vm, + ) } pub fn expandtabs(&self, options: pystr::ExpandTabsArgs) -> Vec { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 529ae1674..0bdba5868 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -396,7 +396,7 @@ impl PyBytes { #[pymethod(name = "rpartition")] fn rpartition(&self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { let sub = PyByteInner::try_from_object(vm, sep.clone())?; - let (front, has_mid, back) = self.inner.rpartition(&sub, vm)?; + let (back, has_mid, front) = self.inner.rpartition(&sub, vm)?; Ok(vm.ctx.new_tuple(vec![ vm.ctx.new_bytes(front), if has_mid { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 51ecd136d..58d1bdb9c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -813,43 +813,35 @@ impl PyString { } #[pymethod] - fn partition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyResult { - if sub.value.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - let mut sp = self.value.splitn(2, &sub.value); - let front = sp.next().unwrap(); - let elems = if let Some(back) = sp.next() { - [front, &sub.value, back] - } else { - [front, "", ""] - }; - Ok(vm.ctx.new_tuple( - elems - .iter() - .map(|&s| vm.ctx.new_str(s.to_owned())) - .collect(), - )) + fn partition(&self, sep: PyStringRef, vm: &VirtualMachine) -> PyResult { + let (front, has_mid, back) = + self.value + .py_partition(&sep.value, || self.value.splitn(2, &sep.value), vm)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_str(front), + if has_mid { + sep.into_object() + } else { + vm.ctx.new_str("") + }, + vm.ctx.new_str(back), + ])) } #[pymethod] - fn rpartition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyResult { - if sub.value.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - let mut sp = self.value.rsplitn(2, &sub.value); - let back = sp.next().unwrap(); - let elems = if let Some(front) = sp.next() { - [front, &sub.value, back] - } else { - ["", "", back] - }; - Ok(vm.ctx.new_tuple( - elems - .iter() - .map(|&s| vm.ctx.new_str(s.to_owned())) - .collect(), - )) + fn rpartition(&self, sep: PyStringRef, vm: &VirtualMachine) -> PyResult { + let (back, has_mid, front) = + self.value + .py_partition(&sep.value, || self.value.rsplitn(2, &sep.value), vm)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_str(front), + if has_mid { + sep.into_object() + } else { + vm.ctx.new_str("") + }, + vm.ctx.new_str(back), + ])) } /// Return `true` if the sequence is ASCII titlecase and the sequence is not diff --git a/vm/src/obj/pystr.rs b/vm/src/obj/pystr.rs index 93ecf4de6..874934888 100644 --- a/vm/src/obj/pystr.rs +++ b/vm/src/obj/pystr.rs @@ -325,6 +325,30 @@ where Ok(joined) } + fn py_partition<'a, F, S>( + &'a self, + sub: &Self, + split: F, + vm: &VirtualMachine, + ) -> PyResult<(Self::Container, bool, Self::Container)> + where + F: Fn() -> S, + S: std::iter::Iterator, + { + if sub.is_empty() { + return Err(vm.new_value_error("empty separator".to_owned())); + } + + let mut sp = split(); + let front = sp.next().unwrap().to_container(); + let (has_mid, back) = if let Some(back) = sp.next() { + (true, back.to_container()) + } else { + (false, Self::Container::new()) + }; + Ok((front, has_mid, back)) + } + fn py_removeprefix( &self, prefix: &Self::Container, From aaf95bb7fb22a299400d1971ef8d10d7ac9270a1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 07:49:37 +0900 Subject: [PATCH 115/209] Remove unused ByteInnerSplitlinesOptions --- vm/src/obj/objbyteinner.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8076801f2..10c13620e 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -231,26 +231,6 @@ impl ByteInnerTranslateOptions { pub type ByteInnerSplitOptions<'a> = pystr::SplitArgs<'a, PyByteInner, [u8], u8>; -#[derive(FromArgs)] -pub struct ByteInnerSplitlinesOptions { - #[pyarg(positional_or_keyword, optional = true)] - keepends: OptionalArg, -} - -impl ByteInnerSplitlinesOptions { - pub fn get_value(self) -> bool { - match self.keepends.into_option() { - Some(x) => x, - None => false, - } - // if let OptionalArg::Present(value) = self.keepends { - // Ok(bool::try_from_object(vm, value)?) - // } else { - // Ok(false) - // } - } -} - #[allow(clippy::len_without_is_empty)] impl PyByteInner { pub fn repr(&self) -> String { From 50757499e552bb84dcca41537e4600a2452b9939 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 07:52:51 +0900 Subject: [PATCH 116/209] InnerDict -> DictInner for naming convention --- vm/src/dictdatatype.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6116af321..d18863884 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -20,18 +20,18 @@ type HashIndex = hash::PyHash; type EntryIndex = usize; pub struct Dict { - inner: PyRwLock>, + inner: PyRwLock>, } -struct InnerDict { +struct DictInner { size: usize, indices: HashMap, entries: Vec>>, } -impl Clone for InnerDict { +impl Clone for DictInner { fn clone(&self) -> Self { - InnerDict { + DictInner { size: self.size, indices: self.indices.clone(), entries: self.entries.clone(), @@ -50,7 +50,7 @@ impl Clone for Dict { impl Default for Dict { fn default() -> Self { Dict { - inner: PyRwLock::new(InnerDict { + inner: PyRwLock::new(DictInner { size: 0, indices: HashMap::new(), entries: Vec::new(), @@ -82,11 +82,11 @@ pub struct DictSize { } impl Dict { - fn borrow_value(&self) -> PyRwLockReadGuard<'_, InnerDict> { + fn borrow_value(&self) -> PyRwLockReadGuard<'_, DictInner> { self.inner.read() } - fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, InnerDict> { + fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, DictInner> { self.inner.write() } From 84b5475d4c2642882f1bd38d0acbaa6690ad5e7f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 08:04:47 +0900 Subject: [PATCH 117/209] obj::objbyteinner::PyByteInner -> bytesinner::PyBytesInner --- vm/src/builtins.rs | 4 +- vm/src/{obj/objbyteinner.rs => bytesinner.rs} | 130 +++++++----------- vm/src/byteslike.rs | 40 ++++++ vm/src/lib.rs | 2 + vm/src/obj/mod.rs | 3 +- vm/src/obj/objbytearray.rs | 52 +++---- vm/src/obj/objbytes.rs | 44 +++--- vm/src/obj/objint.rs | 4 +- vm/src/obj/objlist.rs | 11 +- vm/src/obj/objmemory.rs | 2 +- vm/src/stdlib/binascii.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/multiprocessing.rs | 2 +- vm/src/stdlib/operator.rs | 2 +- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/socket.rs | 2 +- vm/src/stdlib/ssl.rs | 2 +- vm/src/stdlib/zlib.rs | 2 +- wasm/lib/src/convert.rs | 3 +- 19 files changed, 158 insertions(+), 153 deletions(-) rename vm/src/{obj/objbyteinner.rs => bytesinner.rs} (92%) create mode 100644 vm/src/byteslike.rs diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 896fa8b9d..d27f39cb0 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -14,10 +14,10 @@ mod decl { use rustpython_parser::parser; use super::to_ascii; + use crate::bytesinner::PyBytesInner; use crate::exceptions::PyBaseExceptionRef; use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::obj::objbool::{self, IntoPyBool}; - use crate::obj::objbyteinner::PyByteInner; use crate::obj::objbytes::PyBytesRef; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; @@ -554,7 +554,7 @@ mod decl { } #[pyfunction] - fn ord(string: Either, vm: &VirtualMachine) -> PyResult { + fn ord(string: Either, vm: &VirtualMachine) -> PyResult { match string { Either::A(bytes) => { let bytes_len = bytes.elements.len(); diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/bytesinner.rs similarity index 92% rename from vm/src/obj/objbyteinner.rs rename to vm/src/bytesinner.rs index 10c13620e..4ac55adb1 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/bytesinner.rs @@ -3,17 +3,18 @@ use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use std::ops::Range; -use super::objbytearray::{PyByteArray, PyByteArrayRef}; -use super::objbytes::{PyBytes, PyBytesRef}; -use super::objint::{self, PyInt, PyIntRef}; -use super::objlist::PyList; -use super::objmemory::PyMemoryView; -use super::objnone::PyNoneRef; -use super::objsequence::{PySliceableSequence, SequenceIndex}; -use super::objslice::PySliceRef; -use super::objstr::{self, PyString, PyStringRef}; -use super::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; +use crate::byteslike::PyBytesLike; use crate::function::{OptionalArg, OptionalOption}; +use crate::obj::objbytearray::PyByteArray; +use crate::obj::objbytes::PyBytes; +use crate::obj::objint::{self, PyInt, PyIntRef}; +use crate::obj::objlist::PyList; +use crate::obj::objmemory::PyMemoryView; +use crate::obj::objnone::PyNoneRef; +use crate::obj::objsequence::{PySliceableSequence, SequenceIndex}; +use crate::obj::objslice::PySliceRef; +use crate::obj::objstr::{self, PyString, PyStringRef}; +use crate::obj::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; use crate::pyobject::{ Either, PyComparisonValue, PyIterable, PyObjectRef, PyResult, TryFromObject, TypeProtocol, }; @@ -21,26 +22,26 @@ use crate::vm::VirtualMachine; use rustpython_common::hash; #[derive(Debug, Default, Clone)] -pub struct PyByteInner { +pub struct PyBytesInner { pub elements: Vec, } -impl From> for PyByteInner { - fn from(elements: Vec) -> PyByteInner { +impl From> for PyBytesInner { + fn from(elements: Vec) -> PyBytesInner { Self { elements } } } -impl TryFromObject for PyByteInner { +impl TryFromObject for PyBytesInner { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { match_class!(match obj { - i @ PyBytes => Ok(PyByteInner { + i @ PyBytes => Ok(PyBytesInner { elements: i.get_value().to_vec() }), - j @ PyByteArray => Ok(PyByteInner { + j @ PyByteArray => Ok(PyBytesInner { elements: j.borrow_value().elements.to_vec() }), - k @ PyMemoryView => Ok(PyByteInner { + k @ PyMemoryView => Ok(PyBytesInner { elements: k.try_value().unwrap() }), l @ PyList => l.get_byte_inner(vm), @@ -49,7 +50,7 @@ impl TryFromObject for PyByteInner { format!("a bytes-like object is required, not {}", obj.class()) })?; let iter = PyIterable::from_method(iter); - Ok(PyByteInner { + Ok(PyBytesInner { elements: iter.iter(vm)?.collect::>()?, }) } @@ -66,13 +67,13 @@ pub struct ByteInnerNewOptions { } impl ByteInnerNewOptions { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = self.encoding { if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { let bytes = objstr::encode_string(input, Some(enc), None, vm)?; - Ok(PyByteInner { + Ok(PyBytesInner { elements: bytes.get_value().to_vec(), }) } else { @@ -112,7 +113,7 @@ impl ByteInnerNewOptions { // TODO: only support this method in the bytes() constructor if let Some(bytes_method) = vm.get_method(obj.clone(), "__bytes__") { let bytes = vm.invoke(&bytes_method?, vec![])?; - return PyByteInner::try_from_object(vm, bytes); + return PyBytesInner::try_from_object(vm, bytes); } let elements = vm.extract_elements(&obj).map_err(|_| { vm.new_type_error(format!( @@ -139,7 +140,7 @@ impl ByteInnerNewOptions { Ok(vec![]) }; match value { - Ok(val) => Ok(PyByteInner { elements: val }), + Ok(val) => Ok(PyBytesInner { elements: val }), Err(err) => Err(err), } } @@ -149,7 +150,7 @@ impl ByteInnerNewOptions { #[derive(FromArgs)] pub struct ByteInnerFindOptions { #[pyarg(positional_only, optional = false)] - sub: Either, + sub: Either, #[pyarg(positional_only, default = "None")] start: Option, #[pyarg(positional_only, default = "None")] @@ -202,9 +203,9 @@ impl ByteInnerPaddingOptions { #[derive(FromArgs)] pub struct ByteInnerTranslateOptions { #[pyarg(positional_only, optional = false)] - table: Either, + table: Either, #[pyarg(positional_or_keyword, optional = true)] - delete: OptionalArg, + delete: OptionalArg, } impl ByteInnerTranslateOptions { @@ -229,10 +230,10 @@ impl ByteInnerTranslateOptions { } } -pub type ByteInnerSplitOptions<'a> = pystr::SplitArgs<'a, PyByteInner, [u8], u8>; +pub type ByteInnerSplitOptions<'a> = pystr::SplitArgs<'a, PyBytesInner, [u8], u8>; #[allow(clippy::len_without_is_empty)] -impl PyByteInner { +impl PyBytesInner { pub fn repr(&self) -> String { let mut res = String::with_capacity(self.elements.len()); for i in self.elements.iter() { @@ -287,13 +288,13 @@ impl PyByteInner { hash::hash_value(&self.elements) } - pub fn add(&self, other: PyByteInner) -> Vec { + pub fn add(&self, other: PyBytesInner) -> Vec { self.elements.py_add(&other.elements) } pub fn contains( &self, - needle: Either, + needle: Either, vm: &VirtualMachine, ) -> PyResult { Ok(match needle { @@ -698,7 +699,7 @@ impl PyByteInner { pub fn join( &self, - iterable: PyIterable, + iterable: PyIterable, vm: &VirtualMachine, ) -> PyResult> { let iter = iterable.iter(vm)?; @@ -719,7 +720,7 @@ impl PyByteInner { Ok(self.elements.py_find(&needle, range, find)) } - pub fn maketrans(from: PyByteInner, to: PyByteInner, vm: &VirtualMachine) -> PyResult { + pub fn maketrans(from: PyBytesInner, to: PyBytesInner, vm: &VirtualMachine) -> PyResult { let mut res = vec![]; for i in 0..=255 { @@ -757,7 +758,7 @@ impl PyByteInner { Ok(res) } - pub fn strip(&self, chars: OptionalOption) -> Vec { + pub fn strip(&self, chars: OptionalOption) -> Vec { self.elements .py_strip( chars, @@ -767,7 +768,7 @@ impl PyByteInner { .to_vec() } - pub fn lstrip(&self, chars: OptionalOption) -> Vec { + pub fn lstrip(&self, chars: OptionalOption) -> Vec { self.elements .py_strip( chars, @@ -777,7 +778,7 @@ impl PyByteInner { .to_vec() } - pub fn rstrip(&self, chars: OptionalOption) -> Vec { + pub fn rstrip(&self, chars: OptionalOption) -> Vec { self.elements .py_strip( chars, @@ -788,7 +789,7 @@ impl PyByteInner { } // new in Python 3.9 - pub fn removeprefix(&self, prefix: PyByteInner) -> Vec { + pub fn removeprefix(&self, prefix: PyBytesInner) -> Vec { self.elements .py_removeprefix(&prefix.elements, prefix.elements.len(), |s, p| { s.starts_with(p) @@ -797,7 +798,7 @@ impl PyByteInner { } // new in Python 3.9 - pub fn removesuffix(&self, suffix: PyByteInner) -> Vec { + pub fn removesuffix(&self, suffix: PyBytesInner) -> Vec { self.elements .py_removesuffix(&suffix.elements, suffix.elements.len(), |s, p| { s.ends_with(p) @@ -846,7 +847,7 @@ impl PyByteInner { pub fn partition( &self, - sub: &PyByteInner, + sub: &PyBytesInner, vm: &VirtualMachine, ) -> PyResult<(Vec, bool, Vec)> { self.elements.py_partition( @@ -858,7 +859,7 @@ impl PyByteInner { pub fn rpartition( &self, - sub: &PyByteInner, + sub: &PyBytesInner, vm: &VirtualMachine, ) -> PyResult<(Vec, bool, Vec)> { self.elements.py_partition( @@ -912,7 +913,7 @@ impl PyByteInner { } // len(self)>=1, from="", len(to)>=1, maxcount>=1 - fn replace_interleave(&self, to: PyByteInner, maxcount: Option) -> Vec { + fn replace_interleave(&self, to: PyBytesInner, maxcount: Option) -> Vec { let place_count = self.elements.len() + 1; let count = maxcount.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; let capacity = self.elements.len() + count * to.len(); @@ -927,7 +928,7 @@ impl PyByteInner { result } - fn replace_delete(&self, from: PyByteInner, maxcount: Option) -> Vec { + fn replace_delete(&self, from: PyBytesInner, maxcount: Option) -> Vec { let count = count_substring(self.elements.as_slice(), from.elements.as_slice(), maxcount); if count == 0 { // no matches @@ -954,8 +955,8 @@ impl PyByteInner { pub fn replace_in_place( &self, - from: PyByteInner, - to: PyByteInner, + from: PyBytesInner, + to: PyBytesInner, maxcount: Option, ) -> Vec { let len = from.len(); @@ -986,8 +987,8 @@ impl PyByteInner { fn replace_general( &self, - from: PyByteInner, - to: PyByteInner, + from: PyBytesInner, + to: PyBytesInner, maxcount: Option, vm: &VirtualMachine, ) -> PyResult> { @@ -1027,8 +1028,8 @@ impl PyByteInner { pub fn replace( &self, - from: PyByteInner, - to: PyByteInner, + from: PyBytesInner, + to: PyBytesInner, maxcount: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { @@ -1157,42 +1158,7 @@ pub trait ByteOr: ToPrimitive { impl ByteOr for BigInt {} -pub enum PyBytesLike { - Bytes(PyBytesRef), - Bytearray(PyByteArrayRef), -} - -impl TryFromObject for PyBytesLike { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - match_class!(match obj { - b @ PyBytes => Ok(PyBytesLike::Bytes(b)), - b @ PyByteArray => Ok(PyBytesLike::Bytearray(b)), - obj => Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - obj.class() - ))), - }) - } -} - -impl PyBytesLike { - pub fn to_cow(&self) -> std::borrow::Cow<[u8]> { - match self { - PyBytesLike::Bytes(b) => b.get_value().into(), - PyBytesLike::Bytearray(b) => b.borrow_value().elements.clone().into(), - } - } - - #[inline] - pub fn with_ref(&self, f: impl FnOnce(&[u8]) -> R) -> R { - match self { - PyBytesLike::Bytes(b) => f(b.get_value()), - PyBytesLike::Bytearray(b) => f(&b.borrow_value().elements), - } - } -} - -impl PyCommonStringWrapper<[u8]> for PyByteInner { +impl PyCommonStringWrapper<[u8]> for PyBytesInner { fn as_ref(&self) -> &[u8] { &self.elements } diff --git a/vm/src/byteslike.rs b/vm/src/byteslike.rs new file mode 100644 index 000000000..1c1a7c252 --- /dev/null +++ b/vm/src/byteslike.rs @@ -0,0 +1,40 @@ +use crate::obj::objbytearray::{PyByteArray, PyByteArrayRef}; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::pyobject::PyObjectRef; +use crate::pyobject::{PyResult, TryFromObject, TypeProtocol}; +use crate::vm::VirtualMachine; + +pub enum PyBytesLike { + Bytes(PyBytesRef), + Bytearray(PyByteArrayRef), +} + +impl TryFromObject for PyBytesLike { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(match obj { + b @ PyBytes => Ok(PyBytesLike::Bytes(b)), + b @ PyByteArray => Ok(PyBytesLike::Bytearray(b)), + obj => Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj.class() + ))), + }) + } +} + +impl PyBytesLike { + pub fn to_cow(&self) -> std::borrow::Cow<[u8]> { + match self { + PyBytesLike::Bytes(b) => b.get_value().into(), + PyBytesLike::Bytearray(b) => b.borrow_value().elements.clone().into(), + } + } + + #[inline] + pub fn with_ref(&self, f: impl FnOnce(&[u8]) -> R) -> R { + match self { + PyBytesLike::Bytes(b) => f(b.get_value()), + PyBytesLike::Bytearray(b) => f(&b.borrow_value().elements), + } + } +} diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 014515018..de79665f8 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -55,6 +55,8 @@ macro_rules! py_compile_bytecode { pub mod macros; mod builtins; +mod bytesinner; +pub mod byteslike; pub mod cformat; mod dictdatatype; #[cfg(feature = "rustpython-compiler")] diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index f11d0eccf..e9d172325 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -4,7 +4,6 @@ pub mod objasyncgenerator; pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; -pub mod objbyteinner; pub mod objbytes; pub mod objclassmethod; pub mod objcode; @@ -44,4 +43,4 @@ pub mod objtype; pub mod objweakproxy; pub mod objweakref; pub mod objzip; -mod pystr; +pub(super) mod pystr; diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index b6c176893..c74cff8c3 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -5,16 +5,16 @@ use crossbeam_utils::atomic::AtomicCell; use num_traits::cast::ToPrimitive; use std::mem::size_of; -use super::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, ByteOr, PyByteInner, -}; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; +use crate::bytesinner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, ByteOr, PyBytesInner, +}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, @@ -36,7 +36,7 @@ use crate::vm::VirtualMachine; #[pyclass(name = "bytearray")] #[derive(Debug)] pub struct PyByteArray { - inner: PyRwLock, + inner: PyRwLock, } pub type PyByteArrayRef = PyRef; @@ -44,21 +44,21 @@ pub type PyByteArrayRef = PyRef; impl PyByteArray { pub fn new(data: Vec) -> Self { PyByteArray { - inner: PyRwLock::new(PyByteInner { elements: data }), + inner: PyRwLock::new(PyBytesInner { elements: data }), } } - fn from_inner(inner: PyByteInner) -> Self { + fn from_inner(inner: PyBytesInner) -> Self { PyByteArray { inner: PyRwLock::new(inner), } } - pub fn borrow_value(&self) -> PyRwLockReadGuard<'_, PyByteInner> { + pub fn borrow_value(&self) -> PyRwLockReadGuard<'_, PyBytesInner> { self.inner.read() } - pub fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, PyByteInner> { + pub fn borrow_value_mut(&self) -> PyRwLockWriteGuard<'_, PyBytesInner> { self.inner.write() } } @@ -80,7 +80,7 @@ pub(crate) fn init(context: &PyContext) { PyByteArray::extend_class(context, &context.types.bytearray_type); let bytearray_type = &context.types.bytearray_type; extend_class!(context, bytearray_type, { - "maketrans" => context.new_method(PyByteInner::maketrans), + "maketrans" => context.new_method(PyBytesInner::maketrans), }); PyByteArrayIterator::extend_class(context, &context.types.bytearrayiterator_type); @@ -152,7 +152,7 @@ impl PyByteArray { #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Ok(other) = PyByteInner::try_from_object(vm, other) { + if let Ok(other) = PyBytesInner::try_from_object(vm, other) { Ok(vm.ctx.new_bytearray(self.borrow_value().add(other))) } else { Ok(vm.ctx.not_implemented()) @@ -162,7 +162,7 @@ impl PyByteArray { #[pymethod(name = "__contains__")] fn contains( &self, - needle: Either, + needle: Either, vm: &VirtualMachine, ) -> PyResult { self.borrow_value().contains(needle, vm) @@ -250,7 +250,7 @@ impl PyByteArray { #[pymethod] fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { - Ok(PyByteInner::fromhex(string.as_str(), vm)?.into()) + Ok(PyBytesInner::fromhex(string.as_str(), vm)?.into()) } #[pymethod(name = "center")] @@ -286,7 +286,7 @@ impl PyByteArray { } #[pymethod(name = "join")] - fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(self.borrow_value().join(iter, vm)?.into()) } @@ -296,7 +296,7 @@ impl PyByteArray { options, "endswith", "bytes", - |s, x: &PyByteInner| s.ends_with(&x.elements[..]), + |s, x: &PyBytesInner| s.ends_with(&x.elements[..]), vm, ) } @@ -311,7 +311,7 @@ impl PyByteArray { options, "startswith", "bytes", - |s, x: &PyByteInner| s.starts_with(&x.elements[..]), + |s, x: &PyBytesInner| s.starts_with(&x.elements[..]), vm, ) } @@ -365,17 +365,17 @@ impl PyByteArray { } #[pymethod(name = "strip")] - fn strip(&self, chars: OptionalOption) -> PyByteArray { + fn strip(&self, chars: OptionalOption) -> PyByteArray { self.borrow_value().strip(chars).into() } #[pymethod(name = "lstrip")] - fn lstrip(&self, chars: OptionalOption) -> PyByteArray { + fn lstrip(&self, chars: OptionalOption) -> PyByteArray { self.borrow_value().lstrip(chars).into() } #[pymethod(name = "rstrip")] - fn rstrip(&self, chars: OptionalOption) -> PyByteArray { + fn rstrip(&self, chars: OptionalOption) -> PyByteArray { self.borrow_value().rstrip(chars).into() } @@ -387,7 +387,7 @@ impl PyByteArray { /// If the bytearray starts with the prefix string, return string[len(prefix):] /// Otherwise, return a copy of the original bytearray. #[pymethod(name = "removeprefix")] - fn removeprefix(&self, prefix: PyByteInner) -> PyByteArray { + fn removeprefix(&self, prefix: PyBytesInner) -> PyByteArray { self.borrow_value().removeprefix(prefix).into() } @@ -399,7 +399,7 @@ impl PyByteArray { /// If the bytearray ends with the suffix string, return string[:len(suffix)] /// Otherwise, return a copy of the original bytearray. #[pymethod(name = "removesuffix")] - fn removesuffix(&self, suffix: PyByteInner) -> PyByteArray { + fn removesuffix(&self, suffix: PyBytesInner) -> PyByteArray { self.borrow_value().removesuffix(suffix).to_vec().into() } @@ -416,9 +416,9 @@ impl PyByteArray { } #[pymethod(name = "partition")] - fn partition(&self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + fn partition(&self, sep: PyBytesInner, vm: &VirtualMachine) -> PyResult { // sep ALWAYS converted to bytearray even it's bytes or memoryview - // so its ok to accept PyByteInner + // so its ok to accept PyBytesInner let value = self.borrow_value(); let (front, has_mid, back) = value.partition(&sep, vm)?; Ok(vm.ctx.new_tuple(vec![ @@ -430,7 +430,7 @@ impl PyByteArray { } #[pymethod(name = "rpartition")] - fn rpartition(&self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + fn rpartition(&self, sep: PyBytesInner, vm: &VirtualMachine) -> PyResult { let value = self.borrow_value(); let (back, has_mid, front) = value.rpartition(&sep, vm)?; Ok(vm.ctx.new_tuple(vec![ @@ -462,8 +462,8 @@ impl PyByteArray { #[pymethod(name = "replace")] fn replace( &self, - old: PyByteInner, - new: PyByteInner, + old: PyBytesInner, + new: PyBytesInner, count: OptionalArg, vm: &VirtualMachine, ) -> PyResult { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0bdba5868..9d013d6d9 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -3,16 +3,16 @@ use crossbeam_utils::atomic::AtomicCell; use std::mem::size_of; use std::ops::Deref; -use super::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, PyByteInner, -}; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; use super::pystr::{self, PyCommonString}; +use crate::bytesinner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, PyBytesInner, +}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ Either, IntoPyObject, @@ -35,7 +35,7 @@ use rustpython_common::hash::PyHash; #[pyclass(name = "bytes")] #[derive(Clone, Debug)] pub struct PyBytes { - inner: PyByteInner, + inner: PyBytesInner, } pub type PyBytesRef = PyRef; @@ -43,7 +43,7 @@ pub type PyBytesRef = PyRef; impl PyBytes { pub fn new(elements: Vec) -> Self { PyBytes { - inner: PyByteInner { elements }, + inner: PyBytesInner { elements }, } } @@ -82,7 +82,7 @@ pub(crate) fn init(context: &PyContext) { PyBytes::extend_class(context, &context.types.bytes_type); let bytes_type = &context.types.bytes_type; extend_class!(context, bytes_type, { - "maketrans" => context.new_method(PyByteInner::maketrans), + "maketrans" => context.new_method(PyBytesInner::maketrans), }); PyBytesIterator::extend_class(context, &context.types.bytesiterator_type); } @@ -152,7 +152,7 @@ impl PyBytes { #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmaticValue { - if let Ok(other) = PyByteInner::try_from_object(vm, other) { + if let Ok(other) = PyBytesInner::try_from_object(vm, other) { Implemented(self.inner.add(other).into()) } else { NotImplemented @@ -162,7 +162,7 @@ impl PyBytes { #[pymethod(name = "__contains__")] fn contains( &self, - needle: Either, + needle: Either, vm: &VirtualMachine, ) -> PyResult { self.inner.contains(needle, vm) @@ -240,7 +240,7 @@ impl PyBytes { #[pymethod] fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { - Ok(PyByteInner::fromhex(string.as_str(), vm)?.into()) + Ok(PyBytesInner::fromhex(string.as_str(), vm)?.into()) } #[pymethod(name = "center")] @@ -264,7 +264,7 @@ impl PyBytes { } #[pymethod(name = "join")] - fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(self.inner.join(iter, vm)?.into()) } @@ -274,7 +274,7 @@ impl PyBytes { options, "endswith", "bytes", - |s, x: &PyByteInner| s.ends_with(&x.elements[..]), + |s, x: &PyBytesInner| s.ends_with(&x.elements[..]), vm, ) } @@ -289,7 +289,7 @@ impl PyBytes { options, "startswith", "bytes", - |s, x: &PyByteInner| s.starts_with(&x.elements[..]), + |s, x: &PyBytesInner| s.starts_with(&x.elements[..]), vm, ) } @@ -328,17 +328,17 @@ impl PyBytes { } #[pymethod(name = "strip")] - fn strip(&self, chars: OptionalOption) -> PyBytes { + fn strip(&self, chars: OptionalOption) -> PyBytes { self.inner.strip(chars).into() } #[pymethod(name = "lstrip")] - fn lstrip(&self, chars: OptionalOption) -> PyBytes { + fn lstrip(&self, chars: OptionalOption) -> PyBytes { self.inner.lstrip(chars).into() } #[pymethod(name = "rstrip")] - fn rstrip(&self, chars: OptionalOption) -> PyBytes { + fn rstrip(&self, chars: OptionalOption) -> PyBytes { self.inner.rstrip(chars).into() } @@ -350,7 +350,7 @@ impl PyBytes { /// If the bytes starts with the prefix string, return string[len(prefix):] /// Otherwise, return a copy of the original bytes. #[pymethod(name = "removeprefix")] - fn removeprefix(&self, prefix: PyByteInner) -> PyBytes { + fn removeprefix(&self, prefix: PyBytesInner) -> PyBytes { self.inner.removeprefix(prefix).into() } @@ -362,7 +362,7 @@ impl PyBytes { /// If the bytes ends with the suffix string, return string[:len(suffix)] /// Otherwise, return a copy of the original bytes. #[pymethod(name = "removesuffix")] - fn removesuffix(&self, suffix: PyByteInner) -> PyBytes { + fn removesuffix(&self, suffix: PyBytesInner) -> PyBytes { self.inner.removesuffix(suffix).into() } @@ -380,7 +380,7 @@ impl PyBytes { #[pymethod(name = "partition")] fn partition(&self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let sub = PyByteInner::try_from_object(vm, sep.clone())?; + let sub = PyBytesInner::try_from_object(vm, sep.clone())?; let (front, has_mid, back) = self.inner.partition(&sub, vm)?; Ok(vm.ctx.new_tuple(vec![ vm.ctx.new_bytes(front), @@ -395,7 +395,7 @@ impl PyBytes { #[pymethod(name = "rpartition")] fn rpartition(&self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let sub = PyByteInner::try_from_object(vm, sep.clone())?; + let sub = PyBytesInner::try_from_object(vm, sep.clone())?; let (back, has_mid, front) = self.inner.rpartition(&sub, vm)?; Ok(vm.ctx.new_tuple(vec![ vm.ctx.new_bytes(front), @@ -429,8 +429,8 @@ impl PyBytes { #[pymethod(name = "replace")] fn replace( &self, - old: PyByteInner, - new: PyByteInner, + old: PyBytesInner, + new: PyBytesInner, count: OptionalArg, vm: &VirtualMachine, ) -> PyResult { diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index e011f23c0..a63c23f27 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -7,12 +7,12 @@ use num_traits::{Num, One, Pow, Signed, ToPrimitive, Zero}; use super::objbool::IntoPyBool; use super::objbytearray::PyByteArray; -use super::objbyteinner::PyByteInner; use super::objbytes::PyBytes; use super::objfloat; use super::objmemory::PyMemoryView; use super::objstr::{PyString, PyStringRef}; use super::objtype::{self, PyClassRef}; +use crate::bytesinner::PyBytesInner; use crate::format::FormatSpec; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ @@ -714,7 +714,7 @@ impl IntOptions { #[derive(FromArgs)] struct IntFromByteArgs { #[pyarg(positional_or_keyword)] - bytes: PyByteInner, + bytes: PyBytesInner, #[pyarg(positional_or_keyword)] byteorder: PyStringRef, #[pyarg(keyword_only, optional = true)] diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 6c48f3b6c..5ee728ed7 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -2,18 +2,18 @@ use std::fmt; use std::mem::size_of; use std::ops::{DerefMut, Range}; -use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crossbeam_utils::atomic::AtomicCell; use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use super::objbool; -use super::objbyteinner; use super::objint::PyIntRef; use super::objiter; use super::objsequence::{get_item, get_pos, get_slice_range, SequenceIndex}; use super::objslice::PySliceRef; use super::objtype::PyClassRef; +use crate::bytesinner; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::OptionalArg; use crate::pyobject::{ IdProtocol, PyArithmaticValue::*, PyClassImpl, PyComparisonValue, PyContext, PyIterable, @@ -62,10 +62,7 @@ impl PyList { self.elements.write() } - pub(crate) fn get_byte_inner( - &self, - vm: &VirtualMachine, - ) -> PyResult { + pub(crate) fn get_byte_inner(&self, vm: &VirtualMachine) -> PyResult { let mut elements = Vec::::with_capacity(self.borrow_elements().len()); for elem in self.borrow_elements().iter() { match PyIntRef::try_from_object(vm, elem.clone()) { @@ -83,7 +80,7 @@ impl PyList { } } } - Ok(objbyteinner::PyByteInner { elements }) + Ok(bytesinner::PyBytesInner { elements }) } } diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index afdfbb131..4c1b9fdea 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,5 +1,5 @@ -use super::objbyteinner::try_as_byte; use super::objtype::{issubclass, PyClassRef}; +use crate::bytesinner::try_as_byte; use crate::pyobject::{ ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 9803463e8..c8b173321 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -2,9 +2,9 @@ pub(crate) use decl::make_module; #[pymodule(name = "binascii")] mod decl { + use crate::byteslike::PyBytesLike; use crate::function::OptionalArg; use crate::obj::objbytearray::{PyByteArray, PyByteArrayRef}; - use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objstr::{PyString, PyStringRef}; use crate::pyobject::{PyObjectRef, PyResult, TryFromObject, TypeProtocol}; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 743e9955f..226011e32 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -7,12 +7,12 @@ use std::io::{self, prelude::*, Cursor, SeekFrom}; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; +use crate::byteslike::PyBytesLike; use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; use crate::exceptions::PyBaseExceptionRef; use crate::function::{Args, KwArgs, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objbytearray::PyByteArray; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; use crate::obj::objiter; diff --git a/vm/src/stdlib/multiprocessing.rs b/vm/src/stdlib/multiprocessing.rs index 64580dc31..f948030f0 100644 --- a/vm/src/stdlib/multiprocessing.rs +++ b/vm/src/stdlib/multiprocessing.rs @@ -1,5 +1,5 @@ #[allow(unused_imports)] -use crate::obj::objbyteinner::PyBytesLike; +use crate::byteslike::PyBytesLike; #[allow(unused_imports)] use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index 11606cfe8..b386892bf 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -1,5 +1,5 @@ +use crate::byteslike::PyBytesLike; use crate::function::OptionalArg; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objstr::PyStringRef; use crate::obj::{objiter, objtype}; use crate::pyobject::{Either, PyObjectRef, PyResult, TypeProtocol}; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 746e98d48..81af1ae60 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -25,10 +25,10 @@ use std::os::unix::io::RawFd; use std::os::windows::io::RawHandle; use super::errno::errors; +use crate::byteslike::PyBytesLike; use crate::common::cell::PyRwLock; use crate::exceptions::PyBaseExceptionRef; use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objdict::PyDictRef; use crate::obj::objint::{PyInt, PyIntRef}; diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index ad423d083..8238c4d64 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -13,10 +13,10 @@ use socket2::{Domain, Protocol, Socket, Type as SocketType}; use super::os::convert_io_error; #[cfg(unix)] use super::os::convert_nix_error; +use crate::byteslike::PyBytesLike; use crate::exceptions::PyBaseExceptionRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytearray::PyByteArrayRef; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::PyBytesRef; use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtuple::PyTupleRef; diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index 3386df8ce..0263f748c 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -1,8 +1,8 @@ use super::socket::PySocketRef; +use crate::byteslike::PyBytesLike; use crate::exceptions::PyBaseExceptionRef; use crate::function::OptionalArg; use crate::obj::objbytearray::PyByteArrayRef; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objstr::PyStringRef; use crate::obj::{objtype::PyClassRef, objweakref::PyWeak}; use crate::pyobject::{ diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index 43feb1cc5..a95cc81b4 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -1,7 +1,7 @@ +use crate::byteslike::PyBytesLike; use crate::common::cell::PyMutex; use crate::exceptions::PyBaseExceptionRef; use crate::function::OptionalArg; -use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue}; diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index d48aab27c..8d114b9ac 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -6,9 +6,10 @@ use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_compiler::error::{CompileError, CompileErrorType}; use rustpython_parser::error::ParseErrorType; +use rustpython_vm::byteslike::PyBytesLike; use rustpython_vm::exceptions::PyBaseExceptionRef; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::obj::{objbyteinner::PyBytesLike, objtype}; +use rustpython_vm::obj::objtype; use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue, TryFromObject}; use rustpython_vm::VirtualMachine; use rustpython_vm::{exceptions, py_serde}; From c07d3f09cd06ec5e956f9d61e535ac30064ab794 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 08:16:03 +0900 Subject: [PATCH 118/209] obj::pystr -> pystr --- vm/src/bytesinner.rs | 4 ++-- vm/src/lib.rs | 1 + vm/src/obj/mod.rs | 1 - vm/src/obj/objbytearray.rs | 4 ++-- vm/src/obj/objbytes.rs | 2 +- vm/src/obj/objstr.rs | 6 +++--- vm/src/{obj => }/pystr.rs | 0 7 files changed, 9 insertions(+), 9 deletions(-) rename vm/src/{obj => }/pystr.rs (100%) diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 4ac55adb1..4d57c8423 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -14,16 +14,16 @@ use crate::obj::objnone::PyNoneRef; use crate::obj::objsequence::{PySliceableSequence, SequenceIndex}; use crate::obj::objslice::PySliceRef; use crate::obj::objstr::{self, PyString, PyStringRef}; -use crate::obj::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; use crate::pyobject::{ Either, PyComparisonValue, PyIterable, PyObjectRef, PyResult, TryFromObject, TypeProtocol, }; +use crate::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper}; use crate::vm::VirtualMachine; use rustpython_common::hash; #[derive(Debug, Default, Clone)] pub struct PyBytesInner { - pub elements: Vec, + pub(crate) elements: Vec, } impl From> for PyBytesInner { diff --git a/vm/src/lib.rs b/vm/src/lib.rs index de79665f8..246e6f80e 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -70,6 +70,7 @@ pub mod import; pub mod obj; pub mod py_serde; pub mod pyobject; +mod pystr; pub mod readline; pub mod scope; mod sequence; diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index e9d172325..b06f4d47f 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -43,4 +43,3 @@ pub mod objtype; pub mod objweakproxy; pub mod objweakref; pub mod objzip; -pub(super) mod pystr; diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index c74cff8c3..49fad9223 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,5 +1,4 @@ //! Implementation of the python bytearray object. -use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; use num_traits::cast::ToPrimitive; @@ -10,16 +9,17 @@ use super::objiter; use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; -use super::pystr::{self, PyCommonString}; use crate::bytesinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, ByteOr, PyBytesInner, }; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; +use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; /// "bytearray(iterable_of_ints) -> bytearray\n\ diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 9d013d6d9..d1c0d87ab 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -8,7 +8,6 @@ use super::objiter; use super::objsequence::SequenceIndex; use super::objstr::{PyString, PyStringRef}; use super::objtype::PyClassRef; -use super::pystr::{self, PyCommonString}; use crate::bytesinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, PyBytesInner, @@ -20,6 +19,7 @@ use crate::pyobject::{ PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; +use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; use rustpython_common::hash::PyHash; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 58d1bdb9c..21b57976e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -19,15 +19,15 @@ use super::objiter; use super::objnone::PyNone; use super::objsequence::{PySliceableSequence, SequenceIndex}; use super::objtype::{self, PyClassRef}; -use super::pystr::{ - self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, -}; use crate::format::{FormatParseError, FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; +use crate::pystr::{ + self, adjust_indices, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper, +}; use crate::vm::VirtualMachine; use rustpython_common::hash; diff --git a/vm/src/obj/pystr.rs b/vm/src/pystr.rs similarity index 100% rename from vm/src/obj/pystr.rs rename to vm/src/pystr.rs From 8e2a268c357779440e4d065e68d461bcef956df1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 08:51:25 +0900 Subject: [PATCH 119/209] Use PyBytesLike rather than PyBytesInner --- vm/src/builtins.rs | 12 ++++++------ vm/src/bytesinner.rs | 2 +- vm/src/obj/objlist.rs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index d27f39cb0..95fb403b7 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -14,7 +14,7 @@ mod decl { use rustpython_parser::parser; use super::to_ascii; - use crate::bytesinner::PyBytesInner; + use crate::byteslike::PyBytesLike; use crate::exceptions::PyBaseExceptionRef; use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::obj::objbool::{self, IntoPyBool}; @@ -554,18 +554,18 @@ mod decl { } #[pyfunction] - fn ord(string: Either, vm: &VirtualMachine) -> PyResult { + fn ord(string: Either, vm: &VirtualMachine) -> PyResult { match string { - Either::A(bytes) => { - let bytes_len = bytes.elements.len(); + Either::A(bytes) => bytes.with_ref(|bytes| { + let bytes_len = bytes.len(); if bytes_len != 1 { return Err(vm.new_type_error(format!( "ord() expected a character, but string of length {} found", bytes_len ))); } - Ok(u32::from(bytes.elements[0])) - } + Ok(u32::from(bytes[0])) + }), Either::B(string) => { let string = string.as_str(); let string_len = string.chars().count(); diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 4d57c8423..e55d37587 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -44,7 +44,7 @@ impl TryFromObject for PyBytesInner { k @ PyMemoryView => Ok(PyBytesInner { elements: k.try_value().unwrap() }), - l @ PyList => l.get_byte_inner(vm), + l @ PyList => l.to_byte_inner(vm), obj => { let iter = vm.get_method_or_type_error(obj.clone(), "__iter__", || { format!("a bytes-like object is required, not {}", obj.class()) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 5ee728ed7..90e9629e6 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -62,7 +62,7 @@ impl PyList { self.elements.write() } - pub(crate) fn get_byte_inner(&self, vm: &VirtualMachine) -> PyResult { + pub(crate) fn to_byte_inner(&self, vm: &VirtualMachine) -> PyResult { let mut elements = Vec::::with_capacity(self.borrow_elements().len()); for elem in self.borrow_elements().iter() { match PyIntRef::try_from_object(vm, elem.clone()) { From 28216a89355043d737737e3f5172e9083c278105 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 05:40:25 +0900 Subject: [PATCH 120/209] try_as_bytes take closure instead of returning vec --- vm/src/bytesinner.rs | 30 +++++++++++++++++++----------- vm/src/obj/objmemory.rs | 4 ++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index e55d37587..193585720 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -183,15 +183,20 @@ pub struct ByteInnerPaddingOptions { impl ByteInnerPaddingOptions { fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> { let fillchar = if let OptionalArg::Present(v) = self.fillchar { - match try_as_byte(v.clone()) { - Some(x) if x.len() == 1 => x[0], - _ => { - return Err(vm.new_type_error(format!( - "{}() argument 2 must be a byte string of length 1, not {}", - fn_name, &v - ))); + try_as_bytes(v.clone(), |bytes| { + if bytes.len() == 1 { + Some(bytes[0]) + } else { + None } - } + }) + .flatten() + .ok_or_else(|| { + vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, &v + )) + })? } else { b' ' // default is space }; @@ -1129,10 +1134,13 @@ impl PyBytesInner { } } -pub fn try_as_byte(obj: PyObjectRef) -> Option> { +pub fn try_as_bytes(obj: PyObjectRef, f: F) -> Option +where + F: Fn(&[u8]) -> R, +{ match_class!(match obj { - i @ PyBytes => Some(i.get_value().to_vec()), - j @ PyByteArray => Some(j.borrow_value().elements.to_vec()), + i @ PyBytes => Some(f(i.get_value())), + j @ PyByteArray => Some(f(&j.borrow_value().elements)), _ => None, }) } diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 4c1b9fdea..19986a6af 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,5 +1,5 @@ use super::objtype::{issubclass, PyClassRef}; -use crate::bytesinner::try_as_byte; +use crate::bytesinner::try_as_bytes; use crate::pyobject::{ ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; @@ -17,7 +17,7 @@ pub type PyMemoryViewRef = PyRef; #[pyimpl] impl PyMemoryView { pub fn try_value(&self) -> Option> { - try_as_byte(self.obj_ref.clone()) + try_as_bytes(self.obj_ref.clone(), |bytes| bytes.to_vec()) } #[pyslot] From 81d4bc7f9fbd345673363a5de9809b1130f5b89b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 27 Jul 2020 05:45:34 +0900 Subject: [PATCH 121/209] PyMemoryView::try_bytes take closure instead of returning vec --- vm/src/bytesinner.rs | 4 ++-- vm/src/obj/objint.rs | 3 +-- vm/src/obj/objmemory.rs | 7 +++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 193585720..c344ab9b6 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -42,7 +42,7 @@ impl TryFromObject for PyBytesInner { elements: j.borrow_value().elements.to_vec() }), k @ PyMemoryView => Ok(PyBytesInner { - elements: k.try_value().unwrap() + elements: k.try_bytes(|v| v.to_vec()).unwrap() }), l @ PyList => l.to_byte_inner(vm), obj => { @@ -358,7 +358,7 @@ impl PyBytesInner { .collect::>>()?) } _ => match_class!(match object { - i @ PyMemoryView => Ok(i.try_value().unwrap()), + i @ PyMemoryView => Ok(i.try_bytes(|v| v.to_vec()).unwrap()), _ => Err(vm.new_index_error( "can assign only bytes, buffers, or iterables of ints in range(0, 256)" .to_owned() diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index a63c23f27..f846c9f76 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -759,8 +759,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: &BigInt) -> PyResult } memoryview @ PyMemoryView => { // TODO: proper error handling instead of `unwrap()` - let bytes = memoryview.try_value().unwrap(); - bytes_to_int(&bytes) + memoryview.try_bytes(|bytes| bytes_to_int(&bytes)).unwrap() } array @ PyArray => { let bytes = array.tobytes(); diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 19986a6af..acdf09ad3 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -16,8 +16,11 @@ pub type PyMemoryViewRef = PyRef; #[pyimpl] impl PyMemoryView { - pub fn try_value(&self) -> Option> { - try_as_bytes(self.obj_ref.clone(), |bytes| bytes.to_vec()) + pub fn try_bytes(&self, f: F) -> Option + where + F: Fn(&[u8]) -> R, + { + try_as_bytes(self.obj_ref.clone(), f) } #[pyslot] From c74eb6c9a2e2c59d6aa8b844d74d8161f66631b5 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 12 Jul 2020 17:21:49 +1200 Subject: [PATCH 122/209] Implement itertools.groupby --- vm/src/stdlib/itertools.rs | 180 ++++++++++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index eb9752f5d..5cf834f11 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -5,9 +5,11 @@ mod decl { use crossbeam_utils::atomic::AtomicCell; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; + use std::fmt; use std::iter; - use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; + use crate::common::cell::{PyMutex, PyRwLock, PyRwLockWriteGuard}; + use crate::common::rc::{PyRc, PyWeak}; use crate::function::{Args, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objint::{self, PyInt, PyIntRef}; @@ -15,10 +17,10 @@ mod decl { use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{self, PyClassRef}; use crate::pyobject::{ - IdProtocol, PyCallable, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, PyCallable, PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, }; use crate::vm::VirtualMachine; - use rustpython_common::rc::PyRc; #[pyclass(name = "chain")] #[derive(Debug)] @@ -488,6 +490,178 @@ mod decl { } } + struct GroupByState { + current_value: Option, + current_key: Option, + grouper: Option>>, + } + + impl fmt::Debug for GroupByState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyItertoolsGroupBy") + .field("current_value", &self.current_value) + .field("current_key", &self.current_key) + .finish() + } + } + + impl GroupByState { + fn is_current(&self, grouper: &PyItertoolsGrouperRef) -> bool { + if let Some(ref current_weak_grouper) = self.grouper { + if let Some(current_grouper) = current_weak_grouper.upgrade() { + grouper.is(¤t_grouper) + } else { + false + } + } else { + false + } + } + } + + #[pyclass(name = "groupby")] + struct PyItertoolsGroupBy { + iterable: PyObjectRef, + key_func: Option, + state: PyMutex, + } + + impl PyValue for PyItertoolsGroupBy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "groupby") + } + } + + impl fmt::Debug for PyItertoolsGroupBy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyItertoolsGroupBy") + .field("iterable", &self.iterable) + .field("key_func", &self.key_func) + .field("state", &self.state.lock()) + .finish() + } + } + + #[pyimpl] + impl PyItertoolsGroupBy { + #[pyslot] + fn tp_new( + cls: PyClassRef, + iterable: PyObjectRef, + key: OptionalOption, + vm: &VirtualMachine, + ) -> PyResult> { + PyItertoolsGroupBy { + iterable, + key_func: key.flat_option(), + state: PyMutex::new(GroupByState { + current_key: None, + current_value: None, + grouper: None, + }), + } + .into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__next__")] + fn next( + zelf: PyRef, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, PyItertoolsGrouperRef)> { + let mut state = zelf.state.lock(); + state.grouper = None; + let current_key = state.current_key.clone(); + drop(state); + + let (value, key) = if let Some(old_key) = current_key { + loop { + let (value, new_key) = zelf.advance(vm)?; + if !vm.bool_eq(new_key.clone(), old_key.clone())? { + break (value, new_key); + } + } + } else { + zelf.advance(vm)? + }; + + let mut state = zelf.state.lock(); + state.current_value = Some(value.clone()); + state.current_key = Some(key); + + let grouper = PyItertoolsGrouper { + groupby: zelf.clone(), + } + .into_ref(vm); + + state.grouper = Some(PyRc::downgrade(&grouper.clone().into_typed_pyobj())); + Ok((value, grouper)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef) -> PyRef { + zelf + } + + pub(super) fn advance(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + let new_value = call_next(vm, &self.iterable)?; + let new_key = if let Some(ref kf) = self.key_func { + kf.invoke(new_value.clone(), vm)? + } else { + new_value.clone() + }; + Ok((new_value, new_key)) + } + } + + #[pyclass(name = "_grouper")] + #[derive(Debug)] + struct PyItertoolsGrouper { + groupby: PyRef, + } + + type PyItertoolsGrouperRef = PyRef; + + impl PyValue for PyItertoolsGrouper { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "_grouper") + } + } + + #[pyimpl] + impl PyItertoolsGrouper { + #[pymethod(name = "__next__")] + fn next(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let mut state = zelf.groupby.state.lock(); + + if !state.is_current(&zelf) { + return Err(new_stop_iteration(vm)); + } + + if let Some(val) = state.current_value.clone() { + state.current_value = None; + Ok(val) + } else { + let old_key = state.current_key.clone().unwrap(); + // Shouldn't mutex over call to advance as that executes arbitrary Python code + drop(state); + let (value, key) = zelf.groupby.advance(vm)?; + if vm.bool_eq(key.clone(), old_key)? { + Ok(value) + } else { + let mut state = zelf.groupby.state.lock(); + state.current_value = Some(value); + state.current_key = Some(key); + Err(new_stop_iteration(vm)) + } + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef) -> PyRef { + zelf + } + } + #[pyclass(name = "islice")] #[derive(Debug)] struct PyItertoolsIslice { From 611b4b4907fc862628210e5490be38199f92ebe3 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Wed, 15 Jul 2020 07:38:39 +1200 Subject: [PATCH 123/209] Add test_itertools.py from CPython 3.8.3. --- Lib/test/test_itertools.py | 2537 ++++++++++++++++++++++++++++++++++++ 1 file changed, 2537 insertions(+) create mode 100644 Lib/test/test_itertools.py diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py new file mode 100644 index 000000000..eaa6197be --- /dev/null +++ b/Lib/test/test_itertools.py @@ -0,0 +1,2537 @@ +import unittest +from test import support +from itertools import * +import weakref +from decimal import Decimal +from fractions import Fraction +import operator +import random +import copy +import pickle +from functools import reduce +import sys +import struct +import threading +maxsize = support.MAX_Py_ssize_t +minsize = -maxsize-1 + +def lzip(*args): + return list(zip(*args)) + +def onearg(x): + 'Test function of one argument' + return 2*x + +def errfunc(*args): + 'Test function that raises an error' + raise ValueError + +def gen3(): + 'Non-restartable source sequence' + for i in (0, 1, 2): + yield i + +def isEven(x): + 'Test predicate' + return x%2==0 + +def isOdd(x): + 'Test predicate' + return x%2==1 + +def tupleize(*args): + return args + +def irange(n): + for i in range(n): + yield i + +class StopNow: + 'Class emulating an empty iterable.' + def __iter__(self): + return self + def __next__(self): + raise StopIteration + +def take(n, seq): + 'Convenience function for partially consuming a long of infinite iterable' + return list(islice(seq, n)) + +def prod(iterable): + return reduce(operator.mul, iterable, 1) + +def fact(n): + 'Factorial' + return prod(range(1, n+1)) + +# root level methods for pickling ability +def testR(r): + return r[0] + +def testR2(r): + return r[2] + +def underten(x): + return x<10 + +picklecopiers = [lambda s, proto=proto: pickle.loads(pickle.dumps(s, proto)) + for proto in range(pickle.HIGHEST_PROTOCOL + 1)] + +class TestBasicOps(unittest.TestCase): + + def pickletest(self, protocol, it, stop=4, take=1, compare=None): + """Test that an iterator is the same after pickling, also when part-consumed""" + def expand(it, i=0): + # Recursively expand iterables, within sensible bounds + if i > 10: + raise RuntimeError("infinite recursion encountered") + if isinstance(it, str): + return it + try: + l = list(islice(it, stop)) + except TypeError: + return it # can't expand it + return [expand(e, i+1) for e in l] + + # Test the initial copy against the original + dump = pickle.dumps(it, protocol) + i2 = pickle.loads(dump) + self.assertEqual(type(it), type(i2)) + a, b = expand(it), expand(i2) + self.assertEqual(a, b) + if compare: + c = expand(compare) + self.assertEqual(a, c) + + # Take from the copy, and create another copy and compare them. + i3 = pickle.loads(dump) + took = 0 + try: + for i in range(take): + next(i3) + took += 1 + except StopIteration: + pass #in case there is less data than 'take' + dump = pickle.dumps(i3, protocol) + i4 = pickle.loads(dump) + a, b = expand(i3), expand(i4) + self.assertEqual(a, b) + if compare: + c = expand(compare[took:]) + self.assertEqual(a, c); + + def test_accumulate(self): + self.assertEqual(list(accumulate(range(10))), # one positional arg + [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) + self.assertEqual(list(accumulate(iterable=range(10))), # kw arg + [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) + for typ in int, complex, Decimal, Fraction: # multiple types + self.assertEqual( + list(accumulate(map(typ, range(10)))), + list(map(typ, [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]))) + self.assertEqual(list(accumulate('abc')), ['a', 'ab', 'abc']) # works with non-numeric + self.assertEqual(list(accumulate([])), []) # empty iterable + self.assertEqual(list(accumulate([7])), [7]) # iterable of length one + self.assertRaises(TypeError, accumulate, range(10), 5, 6) # too many args + self.assertRaises(TypeError, accumulate) # too few args + self.assertRaises(TypeError, accumulate, x=range(10)) # unexpected kwd arg + self.assertRaises(TypeError, list, accumulate([1, []])) # args that don't add + + s = [2, 8, 9, 5, 7, 0, 3, 4, 1, 6] + self.assertEqual(list(accumulate(s, min)), + [2, 2, 2, 2, 2, 0, 0, 0, 0, 0]) + self.assertEqual(list(accumulate(s, max)), + [2, 8, 9, 9, 9, 9, 9, 9, 9, 9]) + self.assertEqual(list(accumulate(s, operator.mul)), + [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) + with self.assertRaises(TypeError): + list(accumulate(s, chr)) # unary-operation + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, accumulate(range(10))) # test pickling + self.pickletest(proto, accumulate(range(10), initial=7)) + self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) + self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) + self.assertEqual(list(accumulate([], initial=100)), [100]) + with self.assertRaises(TypeError): + list(accumulate([10, 20], 100)) + + def test_chain(self): + + def chain2(*iterables): + 'Pure python version in the docs' + for it in iterables: + for element in it: + yield element + + for c in (chain, chain2): + self.assertEqual(list(c('abc', 'def')), list('abcdef')) + self.assertEqual(list(c('abc')), list('abc')) + self.assertEqual(list(c('')), []) + self.assertEqual(take(4, c('abc', 'def')), list('abcd')) + self.assertRaises(TypeError, list,c(2, 3)) + + def test_chain_from_iterable(self): + self.assertEqual(list(chain.from_iterable(['abc', 'def'])), list('abcdef')) + self.assertEqual(list(chain.from_iterable(['abc'])), list('abc')) + self.assertEqual(list(chain.from_iterable([''])), []) + self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd')) + self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) + + def test_chain_reducible(self): + for oper in [copy.deepcopy] + picklecopiers: + it = chain('abc', 'def') + self.assertEqual(list(oper(it)), list('abcdef')) + self.assertEqual(next(it), 'a') + self.assertEqual(list(oper(it)), list('bcdef')) + + self.assertEqual(list(oper(chain(''))), []) + self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) + self.assertRaises(TypeError, list, oper(chain(2, 3))) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) + + def test_chain_setstate(self): + self.assertRaises(TypeError, chain().__setstate__, ()) + self.assertRaises(TypeError, chain().__setstate__, []) + self.assertRaises(TypeError, chain().__setstate__, 0) + self.assertRaises(TypeError, chain().__setstate__, ([],)) + self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) + it = chain() + it.__setstate__((iter(['abc', 'def']),)) + self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) + it = chain() + it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) + self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) + + def test_combinations(self): + self.assertRaises(TypeError, combinations, 'abc') # missing r argument + self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments + self.assertRaises(TypeError, combinations, None) # pool is not iterable + self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative + + for op in [lambda a:a] + picklecopiers: + self.assertEqual(list(op(combinations('abc', 32))), []) # r > n + + self.assertEqual(list(op(combinations('ABCD', 2))), + [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) + testIntermediate = combinations('ABCD', 2) + next(testIntermediate) + self.assertEqual(list(op(testIntermediate)), + [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) + + self.assertEqual(list(op(combinations(range(4), 3))), + [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) + testIntermediate = combinations(range(4), 3) + next(testIntermediate) + self.assertEqual(list(op(testIntermediate)), + [(0,1,3), (0,2,3), (1,2,3)]) + + + def combinations1(iterable, r): + 'Pure python version shown in the docs' + pool = tuple(iterable) + n = len(pool) + if r > n: + return + indices = list(range(r)) + yield tuple(pool[i] for i in indices) + while 1: + for i in reversed(range(r)): + if indices[i] != i + n - r: + break + else: + return + indices[i] += 1 + for j in range(i+1, r): + indices[j] = indices[j-1] + 1 + yield tuple(pool[i] for i in indices) + + def combinations2(iterable, r): + 'Pure python version shown in the docs' + pool = tuple(iterable) + n = len(pool) + for indices in permutations(range(n), r): + if sorted(indices) == list(indices): + yield tuple(pool[i] for i in indices) + + def combinations3(iterable, r): + 'Pure python version from cwr()' + pool = tuple(iterable) + n = len(pool) + for indices in combinations_with_replacement(range(n), r): + if len(set(indices)) == r: + yield tuple(pool[i] for i in indices) + + for n in range(7): + values = [5*x-12 for x in range(n)] + for r in range(n+2): + result = list(combinations(values, r)) + self.assertEqual(len(result), 0 if r>n else fact(n) / fact(r) / fact(n-r)) # right number of combs + self.assertEqual(len(result), len(set(result))) # no repeats + self.assertEqual(result, sorted(result)) # lexicographic order + for c in result: + self.assertEqual(len(c), r) # r-length combinations + self.assertEqual(len(set(c)), r) # no duplicate elements + self.assertEqual(list(c), sorted(c)) # keep original ordering + self.assertTrue(all(e in values for e in c)) # elements taken from input iterable + self.assertEqual(list(c), + [e for e in values if e in c]) # comb is a subsequence of the input iterable + self.assertEqual(result, list(combinations1(values, r))) # matches first pure python version + self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version + self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, combinations(values, r)) # test pickling + + @support.bigaddrspacetest + def test_combinations_overflow(self): + with self.assertRaises((OverflowError, MemoryError)): + combinations("AA", 2**29) + + # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_tuple_reuse(self): + self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) + self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) + + def test_combinations_with_replacement(self): + cwr = combinations_with_replacement + self.assertRaises(TypeError, cwr, 'abc') # missing r argument + self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments + self.assertRaises(TypeError, cwr, None) # pool is not iterable + self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative + + for op in [lambda a:a] + picklecopiers: + self.assertEqual(list(op(cwr('ABC', 2))), + [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) + testIntermediate = cwr('ABC', 2) + next(testIntermediate) + self.assertEqual(list(op(testIntermediate)), + [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) + + + def cwr1(iterable, r): + 'Pure python version shown in the docs' + # number items returned: (n+r-1)! / r! / (n-1)! when n>0 + pool = tuple(iterable) + n = len(pool) + if not n and r: + return + indices = [0] * r + yield tuple(pool[i] for i in indices) + while 1: + for i in reversed(range(r)): + if indices[i] != n - 1: + break + else: + return + indices[i:] = [indices[i] + 1] * (r - i) + yield tuple(pool[i] for i in indices) + + def cwr2(iterable, r): + 'Pure python version shown in the docs' + pool = tuple(iterable) + n = len(pool) + for indices in product(range(n), repeat=r): + if sorted(indices) == list(indices): + yield tuple(pool[i] for i in indices) + + def numcombs(n, r): + if not n: + return 0 if r else 1 + return fact(n+r-1) / fact(r)/ fact(n-1) + + for n in range(7): + values = [5*x-12 for x in range(n)] + for r in range(n+2): + result = list(cwr(values, r)) + + self.assertEqual(len(result), numcombs(n, r)) # right number of combs + self.assertEqual(len(result), len(set(result))) # no repeats + self.assertEqual(result, sorted(result)) # lexicographic order + + regular_combs = list(combinations(values, r)) # compare to combs without replacement + if n == 0 or r <= 1: + self.assertEqual(result, regular_combs) # cases that should be identical + else: + self.assertTrue(set(result) >= set(regular_combs)) # rest should be supersets of regular combs + + for c in result: + self.assertEqual(len(c), r) # r-length combinations + noruns = [k for k,v in groupby(c)] # combo without consecutive repeats + self.assertEqual(len(noruns), len(set(noruns))) # no repeats other than consecutive + self.assertEqual(list(c), sorted(c)) # keep original ordering + self.assertTrue(all(e in values for e in c)) # elements taken from input iterable + self.assertEqual(noruns, + [e for e in values if e in c]) # comb is a subsequence of the input iterable + self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version + self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, cwr(values,r)) # test pickling + + @support.bigaddrspacetest + def test_combinations_with_replacement_overflow(self): + with self.assertRaises((OverflowError, MemoryError)): + combinations_with_replacement("AA", 2**30) + + # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_with_replacement_tuple_reuse(self): + cwr = combinations_with_replacement + self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) + self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) + + def test_permutations(self): + self.assertRaises(TypeError, permutations) # too few arguments + self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments + self.assertRaises(TypeError, permutations, None) # pool is not iterable + self.assertRaises(ValueError, permutations, 'abc', -2) # r is negative + self.assertEqual(list(permutations('abc', 32)), []) # r > n + self.assertRaises(TypeError, permutations, 'abc', 's') # r is not an int or None + self.assertEqual(list(permutations(range(3), 2)), + [(0,1), (0,2), (1,0), (1,2), (2,0), (2,1)]) + + def permutations1(iterable, r=None): + 'Pure python version shown in the docs' + pool = tuple(iterable) + n = len(pool) + r = n if r is None else r + if r > n: + return + indices = list(range(n)) + cycles = list(range(n-r+1, n+1))[::-1] + yield tuple(pool[i] for i in indices[:r]) + while n: + for i in reversed(range(r)): + cycles[i] -= 1 + if cycles[i] == 0: + indices[i:] = indices[i+1:] + indices[i:i+1] + cycles[i] = n - i + else: + j = cycles[i] + indices[i], indices[-j] = indices[-j], indices[i] + yield tuple(pool[i] for i in indices[:r]) + break + else: + return + + def permutations2(iterable, r=None): + 'Pure python version shown in the docs' + pool = tuple(iterable) + n = len(pool) + r = n if r is None else r + for indices in product(range(n), repeat=r): + if len(set(indices)) == r: + yield tuple(pool[i] for i in indices) + + for n in range(7): + values = [5*x-12 for x in range(n)] + for r in range(n+2): + result = list(permutations(values, r)) + self.assertEqual(len(result), 0 if r>n else fact(n) / fact(n-r)) # right number of perms + self.assertEqual(len(result), len(set(result))) # no repeats + self.assertEqual(result, sorted(result)) # lexicographic order + for p in result: + self.assertEqual(len(p), r) # r-length permutations + self.assertEqual(len(set(p)), r) # no duplicate elements + self.assertTrue(all(e in values for e in p)) # elements taken from input iterable + self.assertEqual(result, list(permutations1(values, r))) # matches first pure python version + self.assertEqual(result, list(permutations2(values, r))) # matches second pure python version + if r == n: + self.assertEqual(result, list(permutations(values, None))) # test r as None + self.assertEqual(result, list(permutations(values))) # test default r + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, permutations(values, r)) # test pickling + + @support.bigaddrspacetest + def test_permutations_overflow(self): + with self.assertRaises((OverflowError, MemoryError)): + permutations("A", 2**30) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_permutations_tuple_reuse(self): + self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1) + self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1) + + def test_combinatorics(self): + # Test relationships between product(), permutations(), + # combinations() and combinations_with_replacement(). + + for n in range(6): + s = 'ABCDEFG'[:n] + for r in range(8): + prod = list(product(s, repeat=r)) + cwr = list(combinations_with_replacement(s, r)) + perm = list(permutations(s, r)) + comb = list(combinations(s, r)) + + # Check size + self.assertEqual(len(prod), n**r) + self.assertEqual(len(cwr), (fact(n+r-1) / fact(r)/ fact(n-1)) if n else (not r)) + self.assertEqual(len(perm), 0 if r>n else fact(n) / fact(n-r)) + self.assertEqual(len(comb), 0 if r>n else fact(n) / fact(r) / fact(n-r)) + + # Check lexicographic order without repeated tuples + self.assertEqual(prod, sorted(set(prod))) + self.assertEqual(cwr, sorted(set(cwr))) + self.assertEqual(perm, sorted(set(perm))) + self.assertEqual(comb, sorted(set(comb))) + + # Check interrelationships + self.assertEqual(cwr, [t for t in prod if sorted(t)==list(t)]) # cwr: prods which are sorted + self.assertEqual(perm, [t for t in prod if len(set(t))==r]) # perm: prods with no dups + self.assertEqual(comb, [t for t in perm if sorted(t)==list(t)]) # comb: perms that are sorted + self.assertEqual(comb, [t for t in cwr if len(set(t))==r]) # comb: cwrs without dups + self.assertEqual(comb, list(filter(set(cwr).__contains__, perm))) # comb: perm that is a cwr + self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm + self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm + + def test_compress(self): + self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) + self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) + self.assertEqual(list(compress('ABCDEF', [0,0,0,0,0,0])), list('')) + self.assertEqual(list(compress('ABCDEF', [1,1,1,1,1,1])), list('ABCDEF')) + self.assertEqual(list(compress('ABCDEF', [1,0,1])), list('AC')) + self.assertEqual(list(compress('ABC', [0,1,1,1,1,1])), list('BC')) + n = 10000 + data = chain.from_iterable(repeat(range(6), n)) + selectors = chain.from_iterable(repeat((0, 1))) + self.assertEqual(list(compress(data, selectors)), [1,3,5] * n) + self.assertRaises(TypeError, compress, None, range(6)) # 1st arg not iterable + self.assertRaises(TypeError, compress, range(6), None) # 2nd arg not iterable + self.assertRaises(TypeError, compress, range(6)) # too few args + self.assertRaises(TypeError, compress, range(6), None) # too many args + + # check copy, deepcopy, pickle + for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: + for data, selectors, result1, result2 in [ + ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), + ('ABCDEF', [0,0,0,0,0,0], '', ''), + ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), + ('ABCDEF', [1,0,1], 'AC', 'C'), + ('ABC', [0,1,1,1,1,1], 'BC', 'C'), + ]: + + self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) + self.assertEqual(list(op(compress(data, selectors))), list(result1)) + testIntermediate = compress(data, selectors) + if result1: + next(testIntermediate) + self.assertEqual(list(op(testIntermediate)), list(result2)) + + + def test_count(self): + self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) + self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) + self.assertEqual(take(2, lzip('abc',count(3))), [('a', 3), ('b', 4)]) + self.assertEqual(take(2, zip('abc',count(-1))), [('a', -1), ('b', 0)]) + self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)]) + self.assertRaises(TypeError, count, 2, 3, 4) + self.assertRaises(TypeError, count, 'a') + self.assertEqual(take(10, count(maxsize-5)), + list(range(maxsize-5, maxsize+5))) + self.assertEqual(take(10, count(-maxsize-5)), + list(range(-maxsize-5, -maxsize+5))) + self.assertEqual(take(3, count(3.25)), [3.25, 4.25, 5.25]) + self.assertEqual(take(3, count(3.25-4j)), [3.25-4j, 4.25-4j, 5.25-4j]) + self.assertEqual(take(3, count(Decimal('1.1'))), + [Decimal('1.1'), Decimal('2.1'), Decimal('3.1')]) + self.assertEqual(take(3, count(Fraction(2, 3))), + [Fraction(2, 3), Fraction(5, 3), Fraction(8, 3)]) + BIGINT = 1<<1000 + self.assertEqual(take(3, count(BIGINT)), [BIGINT, BIGINT+1, BIGINT+2]) + c = count(3) + self.assertEqual(repr(c), 'count(3)') + next(c) + self.assertEqual(repr(c), 'count(4)') + c = count(-9) + self.assertEqual(repr(c), 'count(-9)') + next(c) + self.assertEqual(next(c), -8) + self.assertEqual(repr(count(10.25)), 'count(10.25)') + self.assertEqual(repr(count(10.0)), 'count(10.0)') + self.assertEqual(type(next(count(10.0))), float) + for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): + # Test repr + r1 = repr(count(i)) + r2 = 'count(%r)'.__mod__(i) + self.assertEqual(r1, r2) + + # check copy, deepcopy, pickle + for value in -3, 3, maxsize-5, maxsize+5: + c = count(value) + self.assertEqual(next(copy.copy(c)), value) + self.assertEqual(next(copy.deepcopy(c)), value) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, count(value)) + + #check proper internal error handling for large "step' sizes + count(1, maxsize+5); sys.exc_info() + + def test_count_with_stride(self): + self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) + self.assertEqual(lzip('abc',count(start=2,step=3)), + [('a', 2), ('b', 5), ('c', 8)]) + self.assertEqual(lzip('abc',count(step=-1)), + [('a', 0), ('b', -1), ('c', -2)]) + self.assertRaises(TypeError, count, 'a', 'b') + self.assertEqual(lzip('abc',count(2,0)), [('a', 2), ('b', 2), ('c', 2)]) + self.assertEqual(lzip('abc',count(2,1)), [('a', 2), ('b', 3), ('c', 4)]) + self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) + self.assertEqual(take(20, count(maxsize-15, 3)), take(20, range(maxsize-15, maxsize+100, 3))) + self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3))) + self.assertEqual(take(3, count(10, maxsize+5)), + list(range(10, 10+3*(maxsize+5), maxsize+5))) + self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5]) + self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j]) + self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))), + [Decimal('1.1'), Decimal('1.2'), Decimal('1.3')]) + self.assertEqual(take(3, count(Fraction(2,3), Fraction(1,7))), + [Fraction(2,3), Fraction(17,21), Fraction(20,21)]) + BIGINT = 1<<1000 + self.assertEqual(take(3, count(step=BIGINT)), [0, BIGINT, 2*BIGINT]) + self.assertEqual(repr(take(3, count(10, 2.5))), repr([10, 12.5, 15.0])) + c = count(3, 5) + self.assertEqual(repr(c), 'count(3, 5)') + next(c) + self.assertEqual(repr(c), 'count(8, 5)') + c = count(-9, 0) + self.assertEqual(repr(c), 'count(-9, 0)') + next(c) + self.assertEqual(repr(c), 'count(-9, 0)') + c = count(-9, -3) + self.assertEqual(repr(c), 'count(-9, -3)') + next(c) + self.assertEqual(repr(c), 'count(-12, -3)') + self.assertEqual(repr(c), 'count(-12, -3)') + self.assertEqual(repr(count(10.5, 1.25)), 'count(10.5, 1.25)') + self.assertEqual(repr(count(10.5, 1)), 'count(10.5)') # suppress step=1 when it's an int + self.assertEqual(repr(count(10.5, 1.00)), 'count(10.5, 1.0)') # do show float values lilke 1.0 + self.assertEqual(repr(count(10, 1.00)), 'count(10, 1.0)') + c = count(10, 1.0) + self.assertEqual(type(next(c)), int) + self.assertEqual(type(next(c)), float) + for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): + for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): + # Test repr + r1 = repr(count(i, j)) + if j == 1: + r2 = ('count(%r)' % i) + else: + r2 = ('count(%r, %r)' % (i, j)) + self.assertEqual(r1, r2) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, count(i, j)) + + def test_cycle(self): + self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) + self.assertEqual(list(cycle('')), []) + self.assertRaises(TypeError, cycle) + self.assertRaises(TypeError, cycle, 5) + self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + + # check copy, deepcopy, pickle + c = cycle('abc') + self.assertEqual(next(c), 'a') + #simple copy currently not supported, because __reduce__ returns + #an internal iterator + #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) + self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), + list('bcabcabcab')) + next(c) + self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), + list('cabcabcabc')) + next(c) + next(c) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, cycle('abc')) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap between the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple + with self.assertRaises(TypeError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') + c.__setstate__((tuple('defg'), 0)) + take(20, c) + + # The second argument in the setstate tuple must be an int + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + + self.assertRaises(TypeError, cycle('').__setstate__, ()) + self.assertRaises(TypeError, cycle('').__setstate__, ([],)) + + def test_groupby(self): + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + self.assertRaises(TypeError, groupby, None) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check normal pickled + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + dup = [] + for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested and pickled + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + dup = [] + for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): + for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + it = groupby(s, testR) + _, g = next(it) + next(it) + next(it) + self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + def test_filter(self): + self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) + self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) + self.assertEqual(list(filter(bool, [0,1,0,2,0])), [1,2]) + self.assertEqual(take(4, filter(isEven, count())), [0,2,4,6]) + self.assertRaises(TypeError, filter) + self.assertRaises(TypeError, filter, lambda x:x) + self.assertRaises(TypeError, filter, lambda x:x, range(6), 7) + self.assertRaises(TypeError, filter, isEven, 3) + self.assertRaises(TypeError, next, filter(range(6), range(6))) + + # check copy, deepcopy, pickle + ans = [0,2,4] + + c = filter(isEven, range(6)) + self.assertEqual(list(copy.copy(c)), ans) + c = filter(isEven, range(6)) + self.assertEqual(list(copy.deepcopy(c)), ans) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + c = filter(isEven, range(6)) + self.assertEqual(list(pickle.loads(pickle.dumps(c, proto))), ans) + next(c) + self.assertEqual(list(pickle.loads(pickle.dumps(c, proto))), ans[1:]) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + c = filter(isEven, range(6)) + self.pickletest(proto, c) + + def test_filterfalse(self): + self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) + self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) + self.assertEqual(list(filterfalse(bool, [0,1,0,2,0])), [0,0,0]) + self.assertEqual(take(4, filterfalse(isEven, count())), [1,3,5,7]) + self.assertRaises(TypeError, filterfalse) + self.assertRaises(TypeError, filterfalse, lambda x:x) + self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) + self.assertRaises(TypeError, filterfalse, isEven, 3) + self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, filterfalse(isEven, range(6))) + + def test_zip(self): + # XXX This is rather silly now that builtin zip() calls zip()... + ans = [(x,y) for x, y in zip('abc',count())] + self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) + self.assertEqual(list(zip('abc', range(6))), lzip('abc', range(6))) + self.assertEqual(list(zip('abcdef', range(3))), lzip('abcdef', range(3))) + self.assertEqual(take(3,zip('abcdef', count())), lzip('abcdef', range(3))) + self.assertEqual(list(zip('abcdef')), lzip('abcdef')) + self.assertEqual(list(zip()), lzip()) + self.assertRaises(TypeError, zip, 3) + self.assertRaises(TypeError, zip, range(3), 3) + self.assertEqual([tuple(list(pair)) for pair in zip('abc', 'def')], + lzip('abc', 'def')) + self.assertEqual([pair for pair in zip('abc', 'def')], + lzip('abc', 'def')) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_tuple_reuse(self): + ids = list(map(id, zip('abc', 'def'))) + self.assertEqual(min(ids), max(ids)) + ids = list(map(id, list(zip('abc', 'def')))) + self.assertEqual(len(dict.fromkeys(ids)), len(ids)) + + # check copy, deepcopy, pickle + ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] + self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) + + ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] + self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] + self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + testIntermediate = zip('abc',count()) + next(testIntermediate) + ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] + self.assertEqual(ans, [('b', 1), ('c', 2)]) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, zip('abc', count())) + + def test_ziplongest(self): + for args in [ + ['abc', range(6)], + [range(6), 'abc'], + [range(1000), range(2000,2100), range(3000,3050)], + [range(1000), range(0), range(3000,3050), range(1200), range(1500)], + [range(1000), range(0), range(3000,3050), range(1200), range(1500), range(0)], + ]: + target = [tuple([arg[i] if i < len(arg) else None for arg in args]) + for i in range(max(map(len, args)))] + self.assertEqual(list(zip_longest(*args)), target) + self.assertEqual(list(zip_longest(*args, **{})), target) + target = [tuple((e is None and 'X' or e) for e in t) for t in target] # Replace None fills with 'X' + self.assertEqual(list(zip_longest(*args, **dict(fillvalue='X'))), target) + + self.assertEqual(take(3,zip_longest('abcdef', count())), list(zip('abcdef', range(3)))) # take 3 from infinite input + + self.assertEqual(list(zip_longest()), list(zip())) + self.assertEqual(list(zip_longest([])), list(zip([]))) + self.assertEqual(list(zip_longest('abcdef')), list(zip('abcdef'))) + + self.assertEqual(list(zip_longest('abc', 'defg', **{})), + list(zip(list('abc')+[None], 'defg'))) # empty keyword dict + self.assertRaises(TypeError, zip_longest, 3) + self.assertRaises(TypeError, zip_longest, range(3), 3) + + for stmt in [ + "zip_longest('abc', fv=1)", + "zip_longest('abc', fillvalue=1, bogus_keyword=None)", + ]: + try: + eval(stmt, globals(), locals()) + except TypeError: + pass + else: + self.fail('Did not raise Type in: ' + stmt) + + self.assertEqual([tuple(list(pair)) for pair in zip_longest('abc', 'def')], + list(zip('abc', 'def'))) + self.assertEqual([pair for pair in zip_longest('abc', 'def')], + list(zip('abc', 'def'))) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_longest_tuple_reuse(self): + ids = list(map(id, zip_longest('abc', 'def'))) + self.assertEqual(min(ids), max(ids)) + ids = list(map(id, list(zip_longest('abc', 'def')))) + self.assertEqual(len(dict.fromkeys(ids)), len(ids)) + + def test_zip_longest_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, zip_longest("abc", "def")) + self.pickletest(proto, zip_longest("abc", "defgh")) + self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) + self.pickletest(proto, zip_longest("", "defgh")) + + def test_zip_longest_bad_iterable(self): + exception = TypeError() + + class BadIterable: + def __iter__(self): + raise exception + + with self.assertRaises(TypeError) as cm: + zip_longest(BadIterable()) + + self.assertIs(cm.exception, exception) + + def test_bug_7244(self): + + class Repeater: + # this class is similar to itertools.repeat + def __init__(self, o, t, e): + self.o = o + self.t = int(t) + self.e = e + def __iter__(self): # its iterator is itself + return self + def __next__(self): + if self.t > 0: + self.t -= 1 + return self.o + else: + raise self.e + + # Formerly this code in would fail in debug mode + # with Undetected Error and Stop Iteration + r1 = Repeater(1, 3, StopIteration) + r2 = Repeater(2, 4, StopIteration) + def run(r1, r2): + result = [] + for i, j in zip_longest(r1, r2, fillvalue=0): + with support.captured_output('stdout'): + print((i, j)) + result.append((i, j)) + return result + self.assertEqual(run(r1, r2), [(1,2), (1,2), (1,2), (0,2)]) + + # Formerly, the RuntimeError would be lost + # and StopIteration would stop as expected + r1 = Repeater(1, 3, RuntimeError) + r2 = Repeater(2, 4, StopIteration) + it = zip_longest(r1, r2, fillvalue=0) + self.assertEqual(next(it), (1, 2)) + self.assertEqual(next(it), (1, 2)) + self.assertEqual(next(it), (1, 2)) + self.assertRaises(RuntimeError, next, it) + + def test_product(self): + for args, result in [ + ([], [()]), # zero iterables + (['ab'], [('a',), ('b',)]), # one iterable + ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables + ([range(0), range(2), range(3)], []), # first iterable with zero length + ([range(2), range(0), range(3)], []), # middle iterable with zero length + ([range(2), range(3), range(0)], []), # last iterable with zero length + ]: + self.assertEqual(list(product(*args)), result) + for r in range(4): + self.assertEqual(list(product(*(args*r))), + list(product(*args, **dict(repeat=r)))) + self.assertEqual(len(list(product(*[range(7)]*6))), 7**6) + self.assertRaises(TypeError, product, range(6), None) + + def product1(*args, **kwds): + pools = list(map(tuple, args)) * kwds.get('repeat', 1) + n = len(pools) + if n == 0: + yield () + return + if any(len(pool) == 0 for pool in pools): + return + indices = [0] * n + yield tuple(pool[i] for pool, i in zip(pools, indices)) + while 1: + for i in reversed(range(n)): # right to left + if indices[i] == len(pools[i]) - 1: + continue + indices[i] += 1 + for j in range(i+1, n): + indices[j] = 0 + yield tuple(pool[i] for pool, i in zip(pools, indices)) + break + else: + return + + def product2(*args, **kwds): + 'Pure python version used in docs' + pools = list(map(tuple, args)) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) + + argtypes = ['', 'abc', '', range(0), range(4), dict(a=1, b=2, c=3), + set('abcdefg'), range(11), tuple(range(13))] + for i in range(100): + args = [random.choice(argtypes) for j in range(random.randrange(5))] + expected_len = prod(map(len, args)) + self.assertEqual(len(list(product(*args))), expected_len) + self.assertEqual(list(product(*args)), list(product1(*args))) + self.assertEqual(list(product(*args)), list(product2(*args))) + args = map(iter, args) + self.assertEqual(len(list(product(*args))), expected_len) + + @support.bigaddrspacetest + def test_product_overflow(self): + with self.assertRaises((OverflowError, MemoryError)): + product(*(['ab']*2**5), repeat=2**25) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_product_tuple_reuse(self): + self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) + self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) + + def test_product_pickling(self): + # check copy, deepcopy, pickle + for args, result in [ + ([], [()]), # zero iterables + (['ab'], [('a',), ('b',)]), # one iterable + ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables + ([range(0), range(2), range(3)], []), # first iterable with zero length + ([range(2), range(0), range(3)], []), # middle iterable with zero length + ([range(2), range(3), range(0)], []), # last iterable with zero length + ]: + self.assertEqual(list(copy.copy(product(*args))), result) + self.assertEqual(list(copy.deepcopy(product(*args))), result) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, product(*args)) + + def test_product_issue_25021(self): + # test that indices are properly clamped to the length of the tuples + p = product((1, 2),(3,)) + p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped + self.assertEqual(next(p), (2, 3)) + # test that empty tuple in the list will result in an immediate StopIteration + p = product((1, 2), (), (3,)) + p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped + self.assertRaises(StopIteration, next, p) + + def test_repeat(self): + self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) + self.assertEqual(lzip(range(3),repeat('a')), + [(0, 'a'), (1, 'a'), (2, 'a')]) + self.assertEqual(list(repeat('a', 3)), ['a', 'a', 'a']) + self.assertEqual(take(3, repeat('a')), ['a', 'a', 'a']) + self.assertEqual(list(repeat('a', 0)), []) + self.assertEqual(list(repeat('a', -3)), []) + self.assertRaises(TypeError, repeat) + self.assertRaises(TypeError, repeat, None, 3, 4) + self.assertRaises(TypeError, repeat, None, 'a') + r = repeat(1+0j) + self.assertEqual(repr(r), 'repeat((1+0j))') + r = repeat(1+0j, 5) + self.assertEqual(repr(r), 'repeat((1+0j), 5)') + list(r) + self.assertEqual(repr(r), 'repeat((1+0j), 0)') + + # check copy, deepcopy, pickle + c = repeat(object='a', times=10) + self.assertEqual(next(c), 'a') + self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) + self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, repeat(object='a', times=10)) + + def test_repeat_with_negative_times(self): + self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") + self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") + self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") + self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") + + def test_map(self): + self.assertEqual(list(map(operator.pow, range(3), range(1,7))), + [0**1, 1**2, 2**3]) + self.assertEqual(list(map(tupleize, 'abc', range(5))), + [('a',0),('b',1),('c',2)]) + self.assertEqual(list(map(tupleize, 'abc', count())), + [('a',0),('b',1),('c',2)]) + self.assertEqual(take(2,map(tupleize, 'abc', count())), + [('a',0),('b',1)]) + self.assertEqual(list(map(operator.pow, [])), []) + self.assertRaises(TypeError, map) + self.assertRaises(TypeError, list, map(None, range(3), range(3))) + self.assertRaises(TypeError, map, operator.neg) + self.assertRaises(TypeError, next, map(10, range(5))) + self.assertRaises(ValueError, next, map(errfunc, [4], [5])) + self.assertRaises(TypeError, next, map(onearg, [4], [5])) + + # check copy, deepcopy, pickle + ans = [('a',0),('b',1),('c',2)] + + c = map(tupleize, 'abc', count()) + self.assertEqual(list(copy.copy(c)), ans) + + c = map(tupleize, 'abc', count()) + self.assertEqual(list(copy.deepcopy(c)), ans) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + c = map(tupleize, 'abc', count()) + self.pickletest(proto, c) + + def test_starmap(self): + self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), + [0**1, 1**2, 2**3]) + self.assertEqual(take(3, starmap(operator.pow, zip(count(), count(1)))), + [0**1, 1**2, 2**3]) + self.assertEqual(list(starmap(operator.pow, [])), []) + self.assertEqual(list(starmap(operator.pow, [iter([4,5])])), [4**5]) + self.assertRaises(TypeError, list, starmap(operator.pow, [None])) + self.assertRaises(TypeError, starmap) + self.assertRaises(TypeError, starmap, operator.pow, [(4,5)], 'extra') + self.assertRaises(TypeError, next, starmap(10, [(4,5)])) + self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) + self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) + + # check copy, deepcopy, pickle + ans = [0**1, 1**2, 2**3] + + c = starmap(operator.pow, zip(range(3), range(1,7))) + self.assertEqual(list(copy.copy(c)), ans) + + c = starmap(operator.pow, zip(range(3), range(1,7))) + self.assertEqual(list(copy.deepcopy(c)), ans) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + c = starmap(operator.pow, zip(range(3), range(1,7))) + self.pickletest(proto, c) + + def test_islice(self): + for args in [ # islice(args) should agree with range(args) + (10, 20, 3), + (10, 3, 20), + (10, 20), + (10, 10), + (10, 3), + (20,) + ]: + self.assertEqual(list(islice(range(100), *args)), + list(range(*args))) + + for args, tgtargs in [ # Stop when seqn is exhausted + ((10, 110, 3), ((10, 100, 3))), + ((10, 110), ((10, 100))), + ((110,), (100,)) + ]: + self.assertEqual(list(islice(range(100), *args)), + list(range(*tgtargs))) + + # Test stop=None + self.assertEqual(list(islice(range(10), None)), list(range(10))) + self.assertEqual(list(islice(range(10), None, None)), list(range(10))) + self.assertEqual(list(islice(range(10), None, None, None)), list(range(10))) + self.assertEqual(list(islice(range(10), 2, None)), list(range(2, 10))) + self.assertEqual(list(islice(range(10), 1, None, 2)), list(range(1, 10, 2))) + + # Test number of items consumed SF #1171417 + it = iter(range(10)) + self.assertEqual(list(islice(it, 3)), list(range(3))) + self.assertEqual(list(it), list(range(3, 10))) + + it = iter(range(10)) + self.assertEqual(list(islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + + # Test invalid arguments + ra = range(10) + self.assertRaises(TypeError, islice, ra) + self.assertRaises(TypeError, islice, ra, 1, 2, 3, 4) + self.assertRaises(ValueError, islice, ra, -5, 10, 1) + self.assertRaises(ValueError, islice, ra, 1, -5, -1) + self.assertRaises(ValueError, islice, ra, 1, 10, -1) + self.assertRaises(ValueError, islice, ra, 1, 10, 0) + self.assertRaises(ValueError, islice, ra, 'a') + self.assertRaises(ValueError, islice, ra, 'a', 1) + self.assertRaises(ValueError, islice, ra, 1, 'a') + self.assertRaises(ValueError, islice, ra, 'a', 1, 1) + self.assertRaises(ValueError, islice, ra, 1, 'a', 1) + self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1) + + # Issue #10323: Less islice in a predictable state + c = count() + self.assertEqual(list(islice(c, 1, 3, 50)), [1]) + self.assertEqual(next(c), 3) + + # check copy, deepcopy, pickle + for args in [ # islice(args) should agree with range(args) + (10, 20, 3), + (10, 3, 20), + (10, 20), + (10, 3), + (20,) + ]: + self.assertEqual(list(copy.copy(islice(range(100), *args))), + list(range(*args))) + self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), + list(range(*args))) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, islice(range(100), *args)) + + # Issue #21321: check source iterator is not referenced + # from islice() after the latter has been exhausted + it = (x for x in (1, 2)) + wr = weakref.ref(it) + it = islice(it, 1) + self.assertIsNotNone(wr()) + list(it) # exhaust the iterator + support.gc_collect() + self.assertIsNone(wr()) + + # Issue #30537: islice can accept integer-like objects as + # arguments + class IntLike(object): + def __init__(self, val): + self.val = val + def __index__(self): + return self.val + self.assertEqual(list(islice(range(100), IntLike(10))), list(range(10))) + self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50))), + list(range(10, 50))) + self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), + list(range(10,50,5))) + + def test_takewhile(self): + data = [1, 3, 5, 20, 2, 4, 6, 8] + self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) + self.assertEqual(list(takewhile(underten, [])), []) + self.assertRaises(TypeError, takewhile) + self.assertRaises(TypeError, takewhile, operator.pow) + self.assertRaises(TypeError, takewhile, operator.pow, [(4,5)], 'extra') + self.assertRaises(TypeError, next, takewhile(10, [(4,5)])) + self.assertRaises(ValueError, next, takewhile(errfunc, [(4,5)])) + t = takewhile(bool, [1, 1, 1, 0, 0, 0]) + self.assertEqual(list(t), [1, 1, 1]) + self.assertRaises(StopIteration, next, t) + + # check copy, deepcopy, pickle + self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) + self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), + [1, 3, 5]) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, takewhile(underten, data)) + + def test_dropwhile(self): + data = [1, 3, 5, 20, 2, 4, 6, 8] + self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) + self.assertEqual(list(dropwhile(underten, [])), []) + self.assertRaises(TypeError, dropwhile) + self.assertRaises(TypeError, dropwhile, operator.pow) + self.assertRaises(TypeError, dropwhile, operator.pow, [(4,5)], 'extra') + self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) + self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) + + # check copy, deepcopy, pickle + self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) + self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), + [20, 2, 4, 6, 8]) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, dropwhile(underten, data)) + + def test_tee(self): + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # test values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + # tee pass-through to copyable iterator + a, b = tee('abc') + c, d = tee(a) + self.assertTrue(a is c) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # check deepcopy + a, b = tee('abc') + self.assertEqual(list(copy.deepcopy(a)), ans) + self.assertEqual(list(copy.deepcopy(b)), ans) + self.assertEqual(list(a), ans) + self.assertEqual(list(b), ans) + a, b = tee(range(10000)) + self.assertEqual(list(copy.deepcopy(a)), long_ans) + self.assertEqual(list(copy.deepcopy(b)), long_ans) + self.assertEqual(list(a), long_ans) + self.assertEqual(list(b), long_ans) + + # check partially consumed deepcopy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.deepcopy(a)), ans[2:]) + self.assertEqual(list(copy.deepcopy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) + self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # check pickle + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.pickletest(proto, iter(tee('abc'))) + a, b = tee('abc') + self.pickletest(proto, a, compare=ans) + self.pickletest(proto, b, compare=ans) + + # Issue 13454: Crash when deleting backward iterator from tee() + def test_tee_del_backward(self): + forward, backward = tee(repeat(None, 20000000)) + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + def test_tee_reenter(self): + class I: + first = True + def __iter__(self): + return self + def __next__(self): + first = self.first + self.first = False + if first: + return next(b) + + a, b = tee(I()) + with self.assertRaisesRegex(RuntimeError, "tee"): + next(a) + + def test_tee_concurrent(self): + start = threading.Event() + finish = threading.Event() + class I: + def __iter__(self): + return self + def __next__(self): + start.set() + finish.wait() + + a, b = tee(I()) + thread = threading.Thread(target=next, args=[a]) + thread.start() + try: + start.wait() + with self.assertRaisesRegex(RuntimeError, "tee"): + next(b) + finally: + finish.set() + thread.join() + + def test_StopIteration(self): + self.assertRaises(StopIteration, next, zip()) + + for f in (chain, cycle, zip, groupby): + self.assertRaises(StopIteration, next, f([])) + self.assertRaises(StopIteration, next, f(StopNow())) + + self.assertRaises(StopIteration, next, islice([], None)) + self.assertRaises(StopIteration, next, islice(StopNow(), None)) + + p, q = tee([]) + self.assertRaises(StopIteration, next, p) + self.assertRaises(StopIteration, next, q) + p, q = tee(StopNow()) + self.assertRaises(StopIteration, next, p) + self.assertRaises(StopIteration, next, q) + + self.assertRaises(StopIteration, next, repeat(None, 0)) + + for f in (filter, filterfalse, map, takewhile, dropwhile, starmap): + self.assertRaises(StopIteration, next, f(lambda x:x, [])) + self.assertRaises(StopIteration, next, f(lambda x:x, StopNow())) + +class TestExamples(unittest.TestCase): + + def test_accumulate(self): + self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) + + def test_accumulate_reducible(self): + # check copy, deepcopy, pickle + data = [1, 2, 3, 4, 5] + accumulated = [1, 3, 6, 10, 15] + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + it = accumulate(data) + self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) + self.assertEqual(next(it), 1) + self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) + it = accumulate(data) + self.assertEqual(next(it), 1) + self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) + self.assertEqual(list(copy.copy(it)), accumulated[1:]) + + def test_accumulate_reducible_none(self): + # Issue #25718: total is None + it = accumulate([None, None, None], operator.is_) + self.assertEqual(next(it), None) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + it_copy = pickle.loads(pickle.dumps(it, proto)) + self.assertEqual(list(it_copy), [True, False]) + self.assertEqual(list(copy.deepcopy(it)), [True, False]) + self.assertEqual(list(copy.copy(it)), [True, False]) + + def test_chain(self): + self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') + + def test_chain_from_iterable(self): + self.assertEqual(''.join(chain.from_iterable(['ABC', 'DEF'])), 'ABCDEF') + + def test_combinations(self): + self.assertEqual(list(combinations('ABCD', 2)), + [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) + self.assertEqual(list(combinations(range(4), 3)), + [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) + + def test_combinations_with_replacement(self): + self.assertEqual(list(combinations_with_replacement('ABC', 2)), + [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) + + def test_compress(self): + self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) + + def test_count(self): + self.assertEqual(list(islice(count(10), 5)), [10, 11, 12, 13, 14]) + + def test_cycle(self): + self.assertEqual(list(islice(cycle('ABCD'), 12)), list('ABCDABCDABCD')) + + def test_dropwhile(self): + self.assertEqual(list(dropwhile(lambda x: x<5, [1,4,6,4,1])), [6,4,1]) + + def test_groupby(self): + self.assertEqual([k for k, g in groupby('AAAABBBCCDAABBB')], + list('ABCDAB')) + self.assertEqual([(list(g)) for k, g in groupby('AAAABBBCCD')], + [list('AAAA'), list('BBB'), list('CC'), list('D')]) + + def test_filter(self): + self.assertEqual(list(filter(lambda x: x%2, range(10))), [1,3,5,7,9]) + + def test_filterfalse(self): + self.assertEqual(list(filterfalse(lambda x: x%2, range(10))), [0,2,4,6,8]) + + def test_map(self): + self.assertEqual(list(map(pow, (2,3,10), (5,2,3))), [32, 9, 1000]) + + def test_islice(self): + self.assertEqual(list(islice('ABCDEFG', 2)), list('AB')) + self.assertEqual(list(islice('ABCDEFG', 2, 4)), list('CD')) + self.assertEqual(list(islice('ABCDEFG', 2, None)), list('CDEFG')) + self.assertEqual(list(islice('ABCDEFG', 0, None, 2)), list('ACEG')) + + def test_zip(self): + self.assertEqual(list(zip('ABCD', 'xy')), [('A', 'x'), ('B', 'y')]) + + def test_zip_longest(self): + self.assertEqual(list(zip_longest('ABCD', 'xy', fillvalue='-')), + [('A', 'x'), ('B', 'y'), ('C', '-'), ('D', '-')]) + + def test_permutations(self): + self.assertEqual(list(permutations('ABCD', 2)), + list(map(tuple, 'AB AC AD BA BC BD CA CB CD DA DB DC'.split()))) + self.assertEqual(list(permutations(range(3))), + [(0,1,2), (0,2,1), (1,0,2), (1,2,0), (2,0,1), (2,1,0)]) + + def test_product(self): + self.assertEqual(list(product('ABCD', 'xy')), + list(map(tuple, 'Ax Ay Bx By Cx Cy Dx Dy'.split()))) + self.assertEqual(list(product(range(2), repeat=3)), + [(0,0,0), (0,0,1), (0,1,0), (0,1,1), + (1,0,0), (1,0,1), (1,1,0), (1,1,1)]) + + def test_repeat(self): + self.assertEqual(list(repeat(10, 3)), [10, 10, 10]) + + def test_stapmap(self): + self.assertEqual(list(starmap(pow, [(2,5), (3,2), (10,3)])), + [32, 9, 1000]) + + def test_takewhile(self): + self.assertEqual(list(takewhile(lambda x: x<5, [1,4,6,4,1])), [1,4]) + + +class TestPurePythonRoughEquivalents(unittest.TestCase): + + @staticmethod + def islice(iterable, *args): + s = slice(*args) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) + try: + nexti = next(it) + except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass + return + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass + + def test_islice_recipe(self): + self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) + self.assertEqual(list(self.islice('ABCDEFG', 2, 4)), list('CD')) + self.assertEqual(list(self.islice('ABCDEFG', 2, None)), list('CDEFG')) + self.assertEqual(list(self.islice('ABCDEFG', 0, None, 2)), list('ACEG')) + # Test items consumed. + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3)), list(range(3))) + self.assertEqual(list(it), list(range(3, 10))) + it = iter(range(10)) + self.assertEqual(list(self.islice(it, 3, 3)), []) + self.assertEqual(list(it), list(range(3, 10))) + # Test that slice finishes in predictable state. + c = count() + self.assertEqual(list(self.islice(c, 1, 3, 50)), [1]) + self.assertEqual(next(c), 3) + + +class TestGC(unittest.TestCase): + + def makecycle(self, iterator, container): + container.append(iterator) + next(iterator) + del container, iterator + + def test_accumulate(self): + a = [] + self.makecycle(accumulate([1,2,a,3]), a) + + def test_chain(self): + a = [] + self.makecycle(chain(a), a) + + def test_chain_from_iterable(self): + a = [] + self.makecycle(chain.from_iterable([a]), a) + + def test_combinations(self): + a = [] + self.makecycle(combinations([1,2,a,3], 3), a) + + def test_combinations_with_replacement(self): + a = [] + self.makecycle(combinations_with_replacement([1,2,a,3], 3), a) + + def test_compress(self): + a = [] + self.makecycle(compress('ABCDEF', [1,0,1,0,1,0]), a) + + def test_count(self): + a = [] + Int = type('Int', (int,), dict(x=a)) + self.makecycle(count(Int(0), Int(1)), a) + + def test_cycle(self): + a = [] + self.makecycle(cycle([a]*2), a) + + def test_dropwhile(self): + a = [] + self.makecycle(dropwhile(bool, [0, a, a]), a) + + def test_groupby(self): + a = [] + self.makecycle(groupby([a]*2, lambda x:x), a) + + def test_issue2246(self): + # Issue 2246 -- the _grouper iterator was not included in GC + n = 10 + keyfunc = lambda x: x + for i, j in groupby(range(n), key=keyfunc): + keyfunc.__dict__.setdefault('x',[]).append(j) + + def test_filter(self): + a = [] + self.makecycle(filter(lambda x:True, [a]*2), a) + + def test_filterfalse(self): + a = [] + self.makecycle(filterfalse(lambda x:False, a), a) + + def test_zip(self): + a = [] + self.makecycle(zip([a]*2, [a]*3), a) + + def test_zip_longest(self): + a = [] + self.makecycle(zip_longest([a]*2, [a]*3), a) + b = [a, None] + self.makecycle(zip_longest([a]*2, [a]*3, fillvalue=b), a) + + def test_map(self): + a = [] + self.makecycle(map(lambda x:x, [a]*2), a) + + def test_islice(self): + a = [] + self.makecycle(islice([a]*2, None), a) + + def test_permutations(self): + a = [] + self.makecycle(permutations([1,2,a,3], 3), a) + + def test_product(self): + a = [] + self.makecycle(product([1,2,a,3], repeat=3), a) + + def test_repeat(self): + a = [] + self.makecycle(repeat(a), a) + + def test_starmap(self): + a = [] + self.makecycle(starmap(lambda *t: t, [(a,a)]*2), a) + + def test_takewhile(self): + a = [] + self.makecycle(takewhile(bool, [1, 0, a, a]), a) + +def R(seqn): + 'Regular generator' + for i in seqn: + yield i + +class G: + 'Sequence using __getitem__' + def __init__(self, seqn): + self.seqn = seqn + def __getitem__(self, i): + return self.seqn[i] + +class I: + 'Sequence using iterator protocol' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class Ig: + 'Sequence using iterator protocol defined with a generator' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + for val in self.seqn: + yield val + +class X: + 'Missing __getitem__ and __iter__' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class N: + 'Iterator missing __next__()' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + +class E: + 'Test propagation of exceptions' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + 3 // 0 + +class S: + 'Test immediate stop' + def __init__(self, seqn): + pass + def __iter__(self): + return self + def __next__(self): + raise StopIteration + +def L(seqn): + 'Test multiple tiers of iterators' + return chain(map(lambda x:x, R(Ig(G(seqn))))) + + +class TestVariousIteratorArgs(unittest.TestCase): + + def test_accumulate(self): + s = [1,2,3,4,5] + r = [1,3,6,10,15] + n = len(s) + for g in (G, I, Ig, L, R): + self.assertEqual(list(accumulate(g(s))), r) + self.assertEqual(list(accumulate(S(s))), []) + self.assertRaises(TypeError, accumulate, X(s)) + self.assertRaises(TypeError, accumulate, N(s)) + self.assertRaises(ZeroDivisionError, list, accumulate(E(s))) + + def test_chain(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(chain(g(s))), list(g(s))) + self.assertEqual(list(chain(g(s), g(s))), list(g(s))+list(g(s))) + self.assertRaises(TypeError, list, chain(X(s))) + self.assertRaises(TypeError, list, chain(N(s))) + self.assertRaises(ZeroDivisionError, list, chain(E(s))) + + def test_compress(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + n = len(s) + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(compress(g(s), repeat(1))), list(g(s))) + self.assertRaises(TypeError, compress, X(s), repeat(1)) + self.assertRaises(TypeError, compress, N(s), repeat(1)) + self.assertRaises(ZeroDivisionError, list, compress(E(s), repeat(1))) + + def test_product(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + self.assertRaises(TypeError, product, X(s)) + self.assertRaises(TypeError, product, N(s)) + self.assertRaises(ZeroDivisionError, product, E(s)) + + def test_cycle(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + tgtlen = len(s) * 3 + expected = list(g(s))*3 + actual = list(islice(cycle(g(s)), tgtlen)) + self.assertEqual(actual, expected) + self.assertRaises(TypeError, cycle, X(s)) + self.assertRaises(TypeError, cycle, N(s)) + self.assertRaises(ZeroDivisionError, list, cycle(E(s))) + + def test_groupby(self): + for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual([k for k, sb in groupby(g(s))], list(g(s))) + self.assertRaises(TypeError, groupby, X(s)) + self.assertRaises(TypeError, groupby, N(s)) + self.assertRaises(ZeroDivisionError, list, groupby(E(s))) + + def test_filter(self): + for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(filter(isEven, g(s))), + [x for x in g(s) if isEven(x)]) + self.assertRaises(TypeError, filter, isEven, X(s)) + self.assertRaises(TypeError, filter, isEven, N(s)) + self.assertRaises(ZeroDivisionError, list, filter(isEven, E(s))) + + def test_filterfalse(self): + for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(filterfalse(isEven, g(s))), + [x for x in g(s) if isOdd(x)]) + self.assertRaises(TypeError, filterfalse, isEven, X(s)) + self.assertRaises(TypeError, filterfalse, isEven, N(s)) + self.assertRaises(ZeroDivisionError, list, filterfalse(isEven, E(s))) + + def test_zip(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(zip(g(s))), lzip(g(s))) + self.assertEqual(list(zip(g(s), g(s))), lzip(g(s), g(s))) + self.assertRaises(TypeError, zip, X(s)) + self.assertRaises(TypeError, zip, N(s)) + self.assertRaises(ZeroDivisionError, list, zip(E(s))) + + def test_ziplongest(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(zip_longest(g(s))), list(zip(g(s)))) + self.assertEqual(list(zip_longest(g(s), g(s))), list(zip(g(s), g(s)))) + self.assertRaises(TypeError, zip_longest, X(s)) + self.assertRaises(TypeError, zip_longest, N(s)) + self.assertRaises(ZeroDivisionError, list, zip_longest(E(s))) + + def test_map(self): + for s in (range(10), range(0), range(100), (7,11), range(20,50,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(map(onearg, g(s))), + [onearg(x) for x in g(s)]) + self.assertEqual(list(map(operator.pow, g(s), g(s))), + [x**x for x in g(s)]) + self.assertRaises(TypeError, map, onearg, X(s)) + self.assertRaises(TypeError, map, onearg, N(s)) + self.assertRaises(ZeroDivisionError, list, map(onearg, E(s))) + + def test_islice(self): + for s in ("12345", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(list(islice(g(s),1,None,2)), list(g(s))[1::2]) + self.assertRaises(TypeError, islice, X(s), 10) + self.assertRaises(TypeError, islice, N(s), 10) + self.assertRaises(ZeroDivisionError, list, islice(E(s), 10)) + + def test_starmap(self): + for s in (range(10), range(0), range(100), (7,11), range(20,50,5)): + for g in (G, I, Ig, S, L, R): + ss = lzip(s, s) + self.assertEqual(list(starmap(operator.pow, g(ss))), + [x**x for x in g(s)]) + self.assertRaises(TypeError, starmap, operator.pow, X(ss)) + self.assertRaises(TypeError, starmap, operator.pow, N(ss)) + self.assertRaises(ZeroDivisionError, list, starmap(operator.pow, E(ss))) + + def test_takewhile(self): + for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + tgt = [] + for elem in g(s): + if not isEven(elem): break + tgt.append(elem) + self.assertEqual(list(takewhile(isEven, g(s))), tgt) + self.assertRaises(TypeError, takewhile, isEven, X(s)) + self.assertRaises(TypeError, takewhile, isEven, N(s)) + self.assertRaises(ZeroDivisionError, list, takewhile(isEven, E(s))) + + def test_dropwhile(self): + for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + tgt = [] + for elem in g(s): + if not tgt and isOdd(elem): continue + tgt.append(elem) + self.assertEqual(list(dropwhile(isOdd, g(s))), tgt) + self.assertRaises(TypeError, dropwhile, isOdd, X(s)) + self.assertRaises(TypeError, dropwhile, isOdd, N(s)) + self.assertRaises(ZeroDivisionError, list, dropwhile(isOdd, E(s))) + + def test_tee(self): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + it1, it2 = tee(g(s)) + self.assertEqual(list(it1), list(g(s))) + self.assertEqual(list(it2), list(g(s))) + self.assertRaises(TypeError, tee, X(s)) + self.assertRaises(TypeError, tee, N(s)) + self.assertRaises(ZeroDivisionError, list, tee(E(s))[0]) + +class LengthTransparency(unittest.TestCase): + + def test_repeat(self): + self.assertEqual(operator.length_hint(repeat(None, 50)), 50) + self.assertEqual(operator.length_hint(repeat(None, 0)), 0) + self.assertEqual(operator.length_hint(repeat(None), 12), 12) + + def test_repeat_with_negative_times(self): + self.assertEqual(operator.length_hint(repeat(None, -1)), 0) + self.assertEqual(operator.length_hint(repeat(None, -2)), 0) + self.assertEqual(operator.length_hint(repeat(None, times=-1)), 0) + self.assertEqual(operator.length_hint(repeat(None, times=-2)), 0) + +class RegressionTests(unittest.TestCase): + + def test_sf_793826(self): + # Fix Armin Rigo's successful efforts to wreak havoc + + def mutatingtuple(tuple1, f, tuple2): + # this builds a tuple t which is a copy of tuple1, + # then calls f(t), then mutates t to be equal to tuple2 + # (needs len(tuple1) == len(tuple2)). + def g(value, first=[1]): + if first: + del first[:] + f(next(z)) + return value + items = list(tuple2) + items[1:1] = list(tuple1) + gen = map(g, items) + z = zip(*[gen]*len(tuple1)) + next(z) + + def f(t): + global T + T = t + first[:] = list(T) + + first = [] + mutatingtuple((1,2,3), f, (4,5,6)) + second = list(T) + self.assertEqual(first, second) + + + def test_sf_950057(self): + # Make sure that chain() and cycle() catch exceptions immediately + # rather than when shifting between input sources + + def gen1(): + hist.append(0) + yield 1 + hist.append(1) + raise AssertionError + hist.append(2) + + def gen2(x): + hist.append(3) + yield 2 + hist.append(4) + + hist = [] + self.assertRaises(AssertionError, list, chain(gen1(), gen2(False))) + self.assertEqual(hist, [0,1]) + + hist = [] + self.assertRaises(AssertionError, list, chain(gen1(), gen2(True))) + self.assertEqual(hist, [0,1]) + + hist = [] + self.assertRaises(AssertionError, list, cycle(gen1())) + self.assertEqual(hist, [0,1]) + + @support.skip_if_pgo_task + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + + def test_issue30347_1(self): + def f(n): + if n == 5: + list(b) + return n != 6 + for (k, b) in groupby(range(10), f): + list(b) # shouldn't crash + + def test_issue30347_2(self): + class K: + def __init__(self, v): + pass + def __eq__(self, other): + nonlocal i + i += 1 + if i == 1: + next(g, None) + return True + i = 0 + g = next(groupby(range(10), K))[1] + for j in range(2): + next(g, None) # shouldn't crash + + +class SubclassWithKwargsTest(unittest.TestCase): + def test_keywords_in_subclass(self): + # count is not subclassable... + for cls in (repeat, zip, filter, filterfalse, chain, map, + starmap, islice, takewhile, dropwhile, cycle, compress): + class Subclass(cls): + def __init__(self, newarg=None, *args): + cls.__init__(self, *args) + try: + Subclass(newarg=1) + except TypeError as err: + # we expect type errors because of wrong argument count + self.assertNotIn("keyword arguments", err.args[0]) + +@support.cpython_only +class SizeofTest(unittest.TestCase): + def setUp(self): + self.ssize_t = struct.calcsize('n') + + check_sizeof = support.check_sizeof + + def test_product_sizeof(self): + basesize = support.calcobjsize('3Pi') + check = self.check_sizeof + check(product('ab', '12'), basesize + 2 * self.ssize_t) + check(product(*(('abc',) * 10)), basesize + 10 * self.ssize_t) + + def test_combinations_sizeof(self): + basesize = support.calcobjsize('3Pni') + check = self.check_sizeof + check(combinations('abcd', 3), basesize + 3 * self.ssize_t) + check(combinations(range(10), 4), basesize + 4 * self.ssize_t) + + def test_combinations_with_replacement_sizeof(self): + cwr = combinations_with_replacement + basesize = support.calcobjsize('3Pni') + check = self.check_sizeof + check(cwr('abcd', 3), basesize + 3 * self.ssize_t) + check(cwr(range(10), 4), basesize + 4 * self.ssize_t) + + def test_permutations_sizeof(self): + basesize = support.calcobjsize('4Pni') + check = self.check_sizeof + check(permutations('abcd'), + basesize + 4 * self.ssize_t + 4 * self.ssize_t) + check(permutations('abcd', 3), + basesize + 4 * self.ssize_t + 3 * self.ssize_t) + check(permutations('abcde', 3), + basesize + 5 * self.ssize_t + 3 * self.ssize_t) + check(permutations(range(10), 4), + basesize + 10 * self.ssize_t + 4 * self.ssize_t) + + +libreftest = """ Doctest for examples in the library reference: libitertools.tex + + +>>> amounts = [120.15, 764.05, 823.14] +>>> for checknum, amount in zip(count(1200), amounts): +... print('Check %d is for $%.2f' % (checknum, amount)) +... +Check 1200 is for $120.15 +Check 1201 is for $764.05 +Check 1202 is for $823.14 + +>>> import operator +>>> for cube in map(operator.pow, range(1,4), repeat(3)): +... print(cube) +... +1 +8 +27 + +>>> reportlines = ['EuroPython', 'Roster', '', 'alex', '', 'laura', '', 'martin', '', 'walter', '', 'samuele'] +>>> for name in islice(reportlines, 3, None, 2): +... print(name.title()) +... +Alex +Laura +Martin +Walter +Samuele + +>>> from operator import itemgetter +>>> d = dict(a=1, b=2, c=1, d=2, e=1, f=2, g=3) +>>> di = sorted(sorted(d.items()), key=itemgetter(1)) +>>> for k, g in groupby(di, itemgetter(1)): +... print(k, list(map(itemgetter(0), g))) +... +1 ['a', 'c', 'e'] +2 ['b', 'd', 'f'] +3 ['g'] + +# Find runs of consecutive numbers using groupby. The key to the solution +# is differencing with a range so that consecutive numbers all appear in +# same group. +>>> data = [ 1, 4,5,6, 10, 15,16,17,18, 22, 25,26,27,28] +>>> for k, g in groupby(enumerate(data), lambda t:t[0]-t[1]): +... print(list(map(operator.itemgetter(1), g))) +... +[1] +[4, 5, 6] +[10] +[15, 16, 17, 18] +[22] +[25, 26, 27, 28] + +>>> def take(n, iterable): +... "Return first n items of the iterable as a list" +... return list(islice(iterable, n)) + +>>> def prepend(value, iterator): +... "Prepend a single value in front of an iterator" +... # prepend(1, [2, 3, 4]) -> 1 2 3 4 +... return chain([value], iterator) + +>>> def enumerate(iterable, start=0): +... return zip(count(start), iterable) + +>>> def tabulate(function, start=0): +... "Return function(0), function(1), ..." +... return map(function, count(start)) + +>>> import collections +>>> def consume(iterator, n=None): +... "Advance the iterator n-steps ahead. If n is None, consume entirely." +... # Use functions that consume iterators at C speed. +... if n is None: +... # feed the entire iterator into a zero-length deque +... collections.deque(iterator, maxlen=0) +... else: +... # advance to the empty slice starting at position n +... next(islice(iterator, n, n), None) + +>>> def nth(iterable, n, default=None): +... "Returns the nth item or a default value" +... return next(islice(iterable, n, None), default) + +>>> def all_equal(iterable): +... "Returns True if all the elements are equal to each other" +... g = groupby(iterable) +... return next(g, True) and not next(g, False) + +>>> def quantify(iterable, pred=bool): +... "Count how many times the predicate is true" +... return sum(map(pred, iterable)) + +>>> def padnone(iterable): +... "Returns the sequence elements and then returns None indefinitely" +... return chain(iterable, repeat(None)) + +>>> def ncycles(iterable, n): +... "Returns the sequence elements n times" +... return chain(*repeat(iterable, n)) + +>>> def dotproduct(vec1, vec2): +... return sum(map(operator.mul, vec1, vec2)) + +>>> def flatten(listOfLists): +... return list(chain.from_iterable(listOfLists)) + +>>> def repeatfunc(func, times=None, *args): +... "Repeat calls to func with specified arguments." +... " Example: repeatfunc(random.random)" +... if times is None: +... return starmap(func, repeat(args)) +... else: +... return starmap(func, repeat(args, times)) + +>>> def pairwise(iterable): +... "s -> (s0,s1), (s1,s2), (s2, s3), ..." +... a, b = tee(iterable) +... try: +... next(b) +... except StopIteration: +... pass +... return zip(a, b) + +>>> def grouper(n, iterable, fillvalue=None): +... "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" +... args = [iter(iterable)] * n +... return zip_longest(*args, fillvalue=fillvalue) + +>>> def roundrobin(*iterables): +... "roundrobin('ABC', 'D', 'EF') --> A D E B F C" +... # Recipe credited to George Sakkis +... pending = len(iterables) +... nexts = cycle(iter(it).__next__ for it in iterables) +... while pending: +... try: +... for next in nexts: +... yield next() +... except StopIteration: +... pending -= 1 +... nexts = cycle(islice(nexts, pending)) + +>>> def powerset(iterable): +... "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" +... s = list(iterable) +... return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) + +>>> def unique_everseen(iterable, key=None): +... "List unique elements, preserving order. Remember all elements ever seen." +... # unique_everseen('AAAABBBCCDAABBB') --> A B C D +... # unique_everseen('ABBCcAD', str.lower) --> A B C D +... seen = set() +... seen_add = seen.add +... if key is None: +... for element in iterable: +... if element not in seen: +... seen_add(element) +... yield element +... else: +... for element in iterable: +... k = key(element) +... if k not in seen: +... seen_add(k) +... yield element + +>>> def unique_justseen(iterable, key=None): +... "List unique elements, preserving order. Remember only the element just seen." +... # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B +... # unique_justseen('ABBCcAD', str.lower) --> A B C A D +... return map(next, map(itemgetter(1), groupby(iterable, key))) + +>>> def first_true(iterable, default=False, pred=None): +... '''Returns the first true value in the iterable. +... +... If no true value is found, returns *default* +... +... If *pred* is not None, returns the first item +... for which pred(item) is true. +... +... ''' +... # first_true([a,b,c], x) --> a or b or c or x +... # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x +... return next(filter(pred, iterable), default) + +>>> def nth_combination(iterable, r, index): +... 'Equivalent to list(combinations(iterable, r))[index]' +... pool = tuple(iterable) +... n = len(pool) +... if r < 0 or r > n: +... raise ValueError +... c = 1 +... k = min(r, n-r) +... for i in range(1, k+1): +... c = c * (n - k + i) // i +... if index < 0: +... index += c +... if index < 0 or index >= c: +... raise IndexError +... result = [] +... while r: +... c, n, r = c*r//n, n-1, r-1 +... while index >= c: +... index -= c +... c, n = c*(n-r)//n, n-1 +... result.append(pool[-1-n]) +... return tuple(result) + + +This is not part of the examples but it tests to make sure the definitions +perform as purported. + +>>> take(10, count()) +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +>>> list(prepend(1, [2, 3, 4])) +[1, 2, 3, 4] + +>>> list(enumerate('abc')) +[(0, 'a'), (1, 'b'), (2, 'c')] + +>>> list(islice(tabulate(lambda x: 2*x), 4)) +[0, 2, 4, 6] + +>>> it = iter(range(10)) +>>> consume(it, 3) +>>> next(it) +3 +>>> consume(it) +>>> next(it, 'Done') +'Done' + +>>> nth('abcde', 3) +'d' + +>>> nth('abcde', 9) is None +True + +>>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] +[True, True, True, False, False] + +>>> quantify(range(99), lambda x: x%2==0) +50 + +>>> a = [[1, 2, 3], [4, 5, 6]] +>>> flatten(a) +[1, 2, 3, 4, 5, 6] + +>>> list(repeatfunc(pow, 5, 2, 3)) +[8, 8, 8, 8, 8] + +>>> import random +>>> take(5, map(int, repeatfunc(random.random))) +[0, 0, 0, 0, 0] + +>>> list(pairwise('abcd')) +[('a', 'b'), ('b', 'c'), ('c', 'd')] + +>>> list(pairwise([])) +[] + +>>> list(pairwise('a')) +[] + +>>> list(islice(padnone('abc'), 0, 6)) +['a', 'b', 'c', None, None, None] + +>>> list(ncycles('abc', 3)) +['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] + +>>> dotproduct([1,2,3], [4,5,6]) +32 + +>>> list(grouper(3, 'abcdefg', 'x')) +[('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] + +>>> list(roundrobin('abc', 'd', 'ef')) +['a', 'd', 'e', 'b', 'f', 'c'] + +>>> list(powerset([1,2,3])) +[(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] + +>>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) +True + +>>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len) +True + +>>> list(unique_everseen('AAAABBBCCDAABBB')) +['A', 'B', 'C', 'D'] + +>>> list(unique_everseen('ABBCcAD', str.lower)) +['A', 'B', 'C', 'D'] + +>>> list(unique_justseen('AAAABBBCCDAABBB')) +['A', 'B', 'C', 'D', 'A', 'B'] + +>>> list(unique_justseen('ABBCcAD', str.lower)) +['A', 'B', 'C', 'A', 'D'] + +>>> first_true('ABC0DEF1', '9', str.isdigit) +'0' + +>>> population = 'ABCDEFGH' +>>> for r in range(len(population) + 1): +... seq = list(combinations(population, r)) +... for i in range(len(seq)): +... assert nth_combination(population, r, i) == seq[i] +... for i in range(-len(seq), 0): +... assert nth_combination(population, r, i) == seq[i] + + +""" + +__test__ = {'libreftest' : libreftest} + +def test_main(verbose=None): + test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC, + RegressionTests, LengthTransparency, + SubclassWithKwargsTest, TestExamples, + TestPurePythonRoughEquivalents, + SizeofTest) + support.run_unittest(*test_classes) + + # verify reference counting + if verbose and hasattr(sys, "gettotalrefcount"): + import gc + counts = [None] * 5 + for i in range(len(counts)): + support.run_unittest(*test_classes) + gc.collect() + counts[i] = sys.gettotalrefcount() + print(counts) + + # doctest the examples in the library reference + support.run_doctest(sys.modules[__name__], verbose) + +if __name__ == "__main__": + test_main(verbose=True) From 2d6a1af8e271ffa812dabc705f633a124936d9b6 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Thu, 16 Jul 2020 07:38:22 +1200 Subject: [PATCH 124/209] Fix/mark itertools tests. --- Lib/test/test_itertools.py | 77 ++++++++++++++++++++++++++++++++++++-- vm/src/obj/objiter.rs | 10 ++++- vm/src/pyobject.rs | 4 ++ vm/src/stdlib/itertools.rs | 60 ++++++++++++++++++----------- 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index eaa6197be..422fa904c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -120,6 +120,8 @@ class TestBasicOps(unittest.TestCase): c = expand(compare[took:]) self.assertEqual(a, c); + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -177,6 +179,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd')) self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_chain_reducible(self): for oper in [copy.deepcopy] + picklecopiers: it = chain('abc', 'def') @@ -190,6 +194,8 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_chain_setstate(self): self.assertRaises(TypeError, chain().__setstate__, ()) self.assertRaises(TypeError, chain().__setstate__, []) @@ -203,6 +209,8 @@ class TestBasicOps(unittest.TestCase): it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments @@ -294,6 +302,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -382,6 +392,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) + # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615' + @unittest.skip("panics") def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -455,6 +467,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1) + # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615' + @unittest.skip("panics") def test_combinatorics(self): # Test relationships between product(), permutations(), # combinations() and combinations_with_replacement(). @@ -488,6 +502,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -521,7 +537,8 @@ class TestBasicOps(unittest.TestCase): next(testIntermediate) self.assertEqual(list(op(testIntermediate)), list(result2)) - + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -570,6 +587,8 @@ class TestBasicOps(unittest.TestCase): #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_count_with_stride(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), @@ -625,6 +644,8 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, count(i, j)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) self.assertEqual(list(cycle('')), []) @@ -667,6 +688,8 @@ class TestBasicOps(unittest.TestCase): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cycle_setstate(self): # Verify both modes for restoring state @@ -703,6 +726,8 @@ class TestBasicOps(unittest.TestCase): self.assertRaises(TypeError, cycle('').__setstate__, ()) self.assertRaises(TypeError, cycle('').__setstate__, ([],)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -833,6 +858,8 @@ class TestBasicOps(unittest.TestCase): keyfunc.skip = 1 self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) @@ -860,6 +887,8 @@ class TestBasicOps(unittest.TestCase): c = filter(isEven, range(6)) self.pickletest(proto, c) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -965,6 +994,8 @@ class TestBasicOps(unittest.TestCase): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_zip_longest_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, zip_longest("abc", "def")) @@ -1024,6 +1055,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(next(it), (1, 2)) self.assertRaises(RuntimeError, next, it) + # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615' + @unittest.skip("panics") def test_product(self): for args, result in [ ([], [()]), # zero iterables @@ -1092,6 +1125,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) + # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615' + @unittest.skip("panics") def test_product_pickling(self): # check copy, deepcopy, pickle for args, result in [ @@ -1107,6 +1142,8 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, product(*args)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_product_issue_25021(self): # test that indices are properly clamped to the length of the tuples p = product((1, 2),(3,)) @@ -1117,6 +1154,8 @@ class TestBasicOps(unittest.TestCase): p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped self.assertRaises(StopIteration, next, p) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1143,12 +1182,16 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, repeat(object='a', times=10)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1179,6 +1222,8 @@ class TestBasicOps(unittest.TestCase): c = map(tupleize, 'abc', count()) self.pickletest(proto, c) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1206,6 +1251,8 @@ class TestBasicOps(unittest.TestCase): c = starmap(operator.pow, zip(range(3), range(1,7))) self.pickletest(proto, c) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1300,6 +1347,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1320,6 +1369,8 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, takewhile(underten, data)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1337,6 +1388,8 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, dropwhile(underten, data)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_tee(self): n = 200 @@ -1486,6 +1539,8 @@ class TestBasicOps(unittest.TestCase): self.pickletest(proto, b, compare=ans) # Issue 13454: Crash when deleting backward iterator from tee() + # TODO: RUSTPYTHON + @unittest.skip("hangs") def test_tee_del_backward(self): forward, backward = tee(repeat(None, 20000000)) try: @@ -1495,6 +1550,8 @@ class TestBasicOps(unittest.TestCase): del forward, backward raise + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_tee_reenter(self): class I: first = True @@ -1510,6 +1567,8 @@ class TestBasicOps(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "tee"): next(a) + # TODO: RUSTPYTHON - hangs + @unittest.skip("hangs") def test_tee_concurrent(self): start = threading.Event() finish = threading.Event() @@ -1559,6 +1618,8 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_accumulate_reducible(self): # check copy, deepcopy, pickle data = [1, 2, 3, 4, 5] @@ -1574,6 +1635,8 @@ class TestExamples(unittest.TestCase): self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) self.assertEqual(list(copy.copy(it)), accumulated[1:]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_accumulate_reducible_none(self): # Issue #25718: total is None it = accumulate([None, None, None], operator.is_) @@ -1627,6 +1690,8 @@ class TestExamples(unittest.TestCase): def test_map(self): self.assertEqual(list(map(pow, (2,3,10), (5,2,3))), [32, 9, 1000]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_islice(self): self.assertEqual(list(islice('ABCDEFG', 2)), list('AB')) self.assertEqual(list(islice('ABCDEFG', 2, 4)), list('CD')) @@ -1885,7 +1950,6 @@ def L(seqn): class TestVariousIteratorArgs(unittest.TestCase): - def test_accumulate(self): s = [1,2,3,4,5] r = [1,3,6,10,15] @@ -2041,11 +2105,15 @@ class TestVariousIteratorArgs(unittest.TestCase): class LengthTransparency(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repeat(self): self.assertEqual(operator.length_hint(repeat(None, 50)), 50) self.assertEqual(operator.length_hint(repeat(None, 0)), 0) self.assertEqual(operator.length_hint(repeat(None), 12), 12) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repeat_with_negative_times(self): self.assertEqual(operator.length_hint(repeat(None, -1)), 0) self.assertEqual(operator.length_hint(repeat(None, -2)), 0) @@ -2145,6 +2213,8 @@ class RegressionTests(unittest.TestCase): class SubclassWithKwargsTest(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_keywords_in_subclass(self): # count is not subclassable... for cls in (repeat, zip, filter, filterfalse, chain, map, @@ -2530,8 +2600,9 @@ def test_main(verbose=None): counts[i] = sys.gettotalrefcount() print(counts) + # TODO: RUSTPYTHON this hangs or is very slow # doctest the examples in the library reference - support.run_doctest(sys.modules[__name__], verbose) + # support.run_doctest(sys.modules[__name__], verbose) if __name__ == "__main__": test_main(verbose=True) diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 450678ba8..c4e844785 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -23,7 +23,15 @@ use crate::vm::VirtualMachine; pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { if let Some(method_or_err) = vm.get_method(iter_target.clone(), "__iter__") { let method = method_or_err?; - vm.invoke(&method, vec![]) + let iter = vm.invoke(&method, vec![])?; + if iter.has_class_attr("__next__") { + Ok(iter) + } else { + Err(vm.new_type_error(format!( + "iter() returned non-iterator of type '{}'", + iter.lease_class().name + ))) + } } else { vm.get_method_or_type_error(iter_target.clone(), "__getitem__", || { format!( diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 639fd85d8..d6e20f959 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -915,6 +915,10 @@ pub trait TypeProtocol { fn get_class_attr(&self, attr_name: &str) -> Option { self.lease_class().get_attr(attr_name) } + + fn has_class_attr(&self, attr_name: &str) -> bool { + self.lease_class().has_attr(attr_name) + } } impl TypeProtocol for PyObjectRef { diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 5cf834f11..6c11a34bd 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -493,6 +493,7 @@ mod decl { struct GroupByState { current_value: Option, current_key: Option, + next_group: bool, grouper: Option>>, } @@ -501,6 +502,7 @@ mod decl { f.debug_struct("PyItertoolsGroupBy") .field("current_value", &self.current_value) .field("current_key", &self.current_key) + .field("next_group", &self.next_group) .finish() } } @@ -522,7 +524,7 @@ mod decl { #[pyclass(name = "groupby")] struct PyItertoolsGroupBy { iterable: PyObjectRef, - key_func: Option, + key_func: Option, state: PyMutex, } @@ -542,21 +544,30 @@ mod decl { } } + #[derive(FromArgs)] + struct GroupByArgs { + iterable: PyObjectRef, + #[pyarg(positional_or_keyword, optional = true)] + key: OptionalOption, + } + #[pyimpl] impl PyItertoolsGroupBy { #[pyslot] fn tp_new( cls: PyClassRef, - iterable: PyObjectRef, - key: OptionalOption, + args: GroupByArgs, vm: &VirtualMachine, ) -> PyResult> { + let iter = get_iter(vm, &args.iterable)?; + PyItertoolsGroupBy { - iterable, - key_func: key.flat_option(), + iterable: iter, + key_func: args.key.flat_option(), state: PyMutex::new(GroupByState { current_key: None, current_value: None, + next_group: false, grouper: None, }), } @@ -570,23 +581,28 @@ mod decl { ) -> PyResult<(PyObjectRef, PyItertoolsGrouperRef)> { let mut state = zelf.state.lock(); state.grouper = None; - let current_key = state.current_key.clone(); - drop(state); - let (value, key) = if let Some(old_key) = current_key { - loop { - let (value, new_key) = zelf.advance(vm)?; - if !vm.bool_eq(new_key.clone(), old_key.clone())? { - break (value, new_key); + if !state.next_group { + let current_key = state.current_key.clone(); + drop(state); + + let (value, key) = if let Some(old_key) = current_key { + loop { + let (value, new_key) = zelf.advance(vm)?; + if !vm.bool_eq(new_key.clone(), old_key.clone())? { + break (value, new_key); + } } - } - } else { - zelf.advance(vm)? - }; + } else { + zelf.advance(vm)? + }; - let mut state = zelf.state.lock(); - state.current_value = Some(value.clone()); - state.current_key = Some(key); + state = zelf.state.lock(); + state.current_value = Some(value); + state.current_key = Some(key); + } + + state.next_group = false; let grouper = PyItertoolsGrouper { groupby: zelf.clone(), @@ -594,7 +610,7 @@ mod decl { .into_ref(vm); state.grouper = Some(PyRc::downgrade(&grouper.clone().into_typed_pyobj())); - Ok((value, grouper)) + Ok((state.current_key.clone().unwrap(), grouper)) } #[pymethod(name = "__iter__")] @@ -605,7 +621,7 @@ mod decl { pub(super) fn advance(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { let new_value = call_next(vm, &self.iterable)?; let new_key = if let Some(ref kf) = self.key_func { - kf.invoke(new_value.clone(), vm)? + vm.invoke(kf, new_value.clone())? } else { new_value.clone() }; @@ -651,6 +667,8 @@ mod decl { let mut state = zelf.groupby.state.lock(); state.current_value = Some(value); state.current_key = Some(key); + state.next_group = true; + state.grouper = None; Err(new_stop_iteration(vm)) } } From 020c4c9fd67b1a893962dc3c5d3648e6df63509b Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Thu, 16 Jul 2020 19:24:07 +1200 Subject: [PATCH 125/209] Un-mark test_new_style_iter_class, as it now passes. --- Lib/test/test_iter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index e5e7f0eba..4f1aa68e4 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -226,8 +226,6 @@ class TestCase(unittest.TestCase): self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6]) # Test a new_style class with __iter__ but no next() method - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_style_iter_class(self): class IterClass(object): def __iter__(self): From 73a2b74a061eddcb181bffe8e47ef1219933c259 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Fri, 17 Jul 2020 20:24:10 +1200 Subject: [PATCH 126/209] Code review changes. --- vm/src/stdlib/itertools.rs | 61 +++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 6c11a34bd..9892a36a8 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -499,7 +499,7 @@ mod decl { impl fmt::Debug for GroupByState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PyItertoolsGroupBy") + f.debug_struct("GroupByState") .field("current_value", &self.current_value) .field("current_key", &self.current_key) .field("next_group", &self.next_group) @@ -509,15 +509,10 @@ mod decl { impl GroupByState { fn is_current(&self, grouper: &PyItertoolsGrouperRef) -> bool { - if let Some(ref current_weak_grouper) = self.grouper { - if let Some(current_grouper) = current_weak_grouper.upgrade() { - grouper.is(¤t_grouper) - } else { - false - } - } else { - false - } + self.grouper + .as_ref() + .and_then(|g| g.upgrade()) + .map_or(false, |ref current_grouper| grouper.is(current_grouper)) } } @@ -610,7 +605,7 @@ mod decl { .into_ref(vm); state.grouper = Some(PyRc::downgrade(&grouper.clone().into_typed_pyobj())); - Ok((state.current_key.clone().unwrap(), grouper)) + Ok((state.current_key.as_ref().unwrap().clone(), grouper)) } #[pymethod(name = "__iter__")] @@ -647,30 +642,30 @@ mod decl { impl PyItertoolsGrouper { #[pymethod(name = "__next__")] fn next(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - let mut state = zelf.groupby.state.lock(); + let old_key = { + let mut state = zelf.groupby.state.lock(); - if !state.is_current(&zelf) { - return Err(new_stop_iteration(vm)); - } - - if let Some(val) = state.current_value.clone() { - state.current_value = None; - Ok(val) - } else { - let old_key = state.current_key.clone().unwrap(); - // Shouldn't mutex over call to advance as that executes arbitrary Python code - drop(state); - let (value, key) = zelf.groupby.advance(vm)?; - if vm.bool_eq(key.clone(), old_key)? { - Ok(value) - } else { - let mut state = zelf.groupby.state.lock(); - state.current_value = Some(value); - state.current_key = Some(key); - state.next_group = true; - state.grouper = None; - Err(new_stop_iteration(vm)) + if !state.is_current(&zelf) { + return Err(new_stop_iteration(vm)); } + + // check to see if the value has already been retrieved from the iterator + if let Some(val) = state.current_value.take() { + return Ok(val); + } + + state.current_key.as_ref().unwrap().clone() + }; + let (value, key) = zelf.groupby.advance(vm)?; + if vm.bool_eq(key.clone(), old_key)? { + Ok(value) + } else { + let mut state = zelf.groupby.state.lock(); + state.current_value = Some(value); + state.current_key = Some(key); + state.next_group = true; + state.grouper = None; + Err(new_stop_iteration(vm)) } } From 5b26281a3d46a387ffd7d33a4a8864874c5c9c9d Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Sun, 26 Jul 2020 18:09:30 -0700 Subject: [PATCH 127/209] Fixed bug where sys.executable may follow symlinks --- vm/src/sysmodule.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index f130b7081..b8efb3bba 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,4 +1,4 @@ -use std::{env, mem}; +use std::{env, mem, path}; use crate::builtins; use crate::frame::FrameRef; @@ -28,9 +28,18 @@ fn argv(vm: &VirtualMachine) -> PyObjectRef { } fn executable(ctx: &PyContext) -> PyObjectRef { - if let Ok(path) = env::current_exe() { - if let Ok(path) = path.into_os_string().into_string() { - return ctx.new_str(path); + if let Some(exec_path) = env::args().next() { + if path::Path::new(&exec_path).is_absolute() { + return ctx.new_str(exec_path); + } + if let Ok(dir) = env::current_dir() { + if let Ok(dir) = dir.into_os_string().into_string() { + return ctx.new_str(format!( + "{}/{}", + dir, + exec_path.strip_prefix("./").unwrap_or(&exec_path) + )); + } } } ctx.none() From 03e7825ace9d56b7109d563e9e7c359fa1439da3 Mon Sep 17 00:00:00 2001 From: Minjun Shin Date: Tue, 28 Jul 2020 06:13:51 +0900 Subject: [PATCH 128/209] Minor typo fix for panic! macro --- vm/src/stdlib/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b66a284d4..f9ac48d5f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1012,7 +1012,7 @@ fn os_symlink( } else if meta.is_dir() { win_fs::symlink_dir(src.path, dst.path) } else { - panic!("Uknown file type"); + panic!("Unknown file type"); }; ret.map_err(|err| convert_io_error(vm, err)) } From 2f77be17d827da1a0ba6288f4e63748034a6b4c9 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 27 Jul 2020 18:47:09 -0500 Subject: [PATCH 129/209] Don't put the freeze example as a workspace member --- Cargo.lock | 7 ------- Cargo.toml | 2 +- examples/freeze/Cargo.toml | 16 ---------------- examples/freeze/{src => }/main.rs | 4 +++- 4 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 examples/freeze/Cargo.toml rename examples/freeze/{src => }/main.rs (62%) diff --git a/Cargo.lock b/Cargo.lock index 37bf69972..da5ef537f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1465,13 +1465,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rustpython_freeze" -version = "0.1.2" -dependencies = [ - "rustpython-vm", -] - [[package]] name = "rustpython_wasm" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index 8491aea3f..2c6bdefe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace] -members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate", "common"] +members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common"] [[bench]] name = "bench" diff --git a/examples/freeze/Cargo.toml b/examples/freeze/Cargo.toml deleted file mode 100644 index eafbed069..000000000 --- a/examples/freeze/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "rustpython_freeze" -version = "0.1.2" -authors = ["RustPython Team"] -license = "MIT" -description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to frozen executable" -repository = "https://github.com/RustPython/RustPython/tree/master/freeze" -edition = "2018" - -[[bin]] -name = "rustpython_freeze" -path = "src/main.rs" - - -[dependencies] -rustpython-vm = { path = "../../vm", default-features = false } diff --git a/examples/freeze/src/main.rs b/examples/freeze/main.rs similarity index 62% rename from examples/freeze/src/main.rs rename to examples/freeze/main.rs index 0bd9f8f5f..81e8028b2 100644 --- a/examples/freeze/src/main.rs +++ b/examples/freeze/main.rs @@ -7,8 +7,10 @@ fn main() -> vm::pyobject::PyResult<()> { let scope = vm.new_scope_with_builtins(); + // the file parameter is relevant to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR: + // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates let modules: HashMap = - vm::py_compile_bytecode!(file = "freeze.py"); + vm::py_compile_bytecode!(file = "examples/freeze/freeze.py"); let res = vm.run_code_obj( vm.ctx From 1a653562b8a964e79a4776b46bc53ff17b4f48b8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 29 Jul 2020 02:05:48 +0900 Subject: [PATCH 130/209] flat_option -> flatten by following std::option::Option --- vm/src/function.rs | 2 +- vm/src/obj/objfloat.rs | 2 +- vm/src/obj/objmodule.rs | 2 +- vm/src/pystr.rs | 2 +- vm/src/stdlib/io.rs | 20 ++++++++++---------- vm/src/stdlib/itertools.rs | 2 +- vm/src/stdlib/random.rs | 2 +- vm/src/stdlib/select.rs | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index 76b0fad40..b26318ced 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -404,7 +404,7 @@ pub type OptionalOption = OptionalArg>; impl OptionalOption { #[inline] - pub fn flat_option(self) -> Option { + pub fn flatten(self) -> Option { match self { Present(Some(value)) => Some(value), _ => None, diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index d3c162f5a..74c2925b2 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -381,7 +381,7 @@ impl PyFloat { #[pymethod(name = "__round__")] fn round(&self, ndigits: OptionalOption, vm: &VirtualMachine) -> PyResult { - let ndigits = ndigits.flat_option(); + let ndigits = ndigits.flatten(); if let Some(ndigits) = ndigits { let ndigits = ndigits.as_bigint(); if ndigits.is_zero() { diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index cd43681dd..abed0794f 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -57,7 +57,7 @@ impl PyModuleRef { vm, &zelf.as_object().dict().unwrap(), name.into_object(), - doc.flat_option() + doc.flatten() .map_or_else(|| vm.get_none(), PyRef::into_object), ); Ok(zelf) diff --git a/vm/src/pystr.rs b/vm/src/pystr.rs index 874934888..9c1a7af31 100644 --- a/vm/src/pystr.rs +++ b/vm/src/pystr.rs @@ -251,7 +251,7 @@ where FC: Fn(&'a Self, &Self) -> &'a Self, FD: Fn(&'a Self) -> &'a Self, { - let chars = chars.flat_option(); + let chars = chars.flatten(); match chars { Some(chars) => func_chars(self, chars.as_ref()), None => func_default(self), diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 226011e32..3e93f0549 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -24,7 +24,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; fn byte_count(bytes: OptionalOption) -> i64 { - bytes.flat_option().unwrap_or(-1 as i64) + bytes.flatten().unwrap_or(-1 as i64) } fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] @@ -204,7 +204,7 @@ impl PyStringIORef { fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { let mut buffer = self.buffer(vm)?; - let size = size.flat_option().unwrap_or_else(|| buffer.tell() as usize); + let size = size.flatten().unwrap_or_else(|| buffer.tell() as usize); buffer.cursor.get_mut().truncate(size); Ok(()) } @@ -232,7 +232,7 @@ fn string_io_new( _args: StringIOArgs, vm: &VirtualMachine, ) -> PyResult { - let flatten = object.flat_option(); + let flatten = object.flatten(); let input = flatten.map_or_else(Vec::new, |v| objstr::borrow_value(&v).as_bytes().to_vec()); PyStringIO { @@ -316,7 +316,7 @@ impl PyBytesIORef { fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { let mut buffer = self.buffer(vm)?; - let size = size.flat_option().unwrap_or_else(|| buffer.tell() as usize); + let size = size.flatten().unwrap_or_else(|| buffer.tell() as usize); buffer.cursor.get_mut().truncate(size); Ok(()) } @@ -411,7 +411,7 @@ fn io_base_checkclosed( ) -> PyResult<()> { if objbool::boolval(vm, vm.get_attribute(instance, "closed")?)? { let msg = msg - .flat_option() + .flatten() .unwrap_or_else(|| vm.new_str("I/O operation on closed file.".to_owned())); Err(vm.new_exception(vm.ctx.exceptions.value_error.clone(), vec![msg])) } else { @@ -426,7 +426,7 @@ fn io_base_checkreadable( ) -> PyResult<()> { if !objbool::boolval(vm, vm.call_method(&instance, "readable", vec![])?)? { let msg = msg - .flat_option() + .flatten() .unwrap_or_else(|| vm.new_str("File or stream is not readable.".to_owned())); Err(vm.new_exception(vm.ctx.exceptions.value_error.clone(), vec![msg])) } else { @@ -441,7 +441,7 @@ fn io_base_checkwritable( ) -> PyResult<()> { if !objbool::boolval(vm, vm.call_method(&instance, "writable", vec![])?)? { let msg = msg - .flat_option() + .flatten() .unwrap_or_else(|| vm.new_str("File or stream is not writable.".to_owned())); Err(vm.new_exception(vm.ctx.exceptions.value_error.clone(), vec![msg])) } else { @@ -456,7 +456,7 @@ fn io_base_checkseekable( ) -> PyResult<()> { if !objbool::boolval(vm, vm.call_method(&instance, "seekable", vec![])?)? { let msg = msg - .flat_option() + .flatten() .unwrap_or_else(|| vm.new_str("File or stream is not seekable.".to_owned())); Err(vm.new_exception(vm.ctx.exceptions.value_error.clone(), vec![msg])) } else { @@ -929,7 +929,7 @@ fn text_io_wrapper_read( let bytes = vm.call_method( &raw, "read", - vec![size.flat_option().unwrap_or_else(|| vm.get_none())], + vec![size.flatten().unwrap_or_else(|| vm.get_none())], )?; let bytes = PyBytesLike::try_from_object(vm, bytes)?; //format bytes into string @@ -988,7 +988,7 @@ fn text_io_wrapper_readline( let bytes = vm.call_method( &raw, "readline", - vec![size.flat_option().unwrap_or_else(|| vm.get_none())], + vec![size.flatten().unwrap_or_else(|| vm.get_none())], )?; let bytes = PyBytesLike::try_from_object(vm, bytes)?; //format bytes into string diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index eb9752f5d..1ef5e6176 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -1169,7 +1169,7 @@ mod decl { let n = pool.len(); // If r is not provided, r == n. If provided, r must be a positive integer, or None. // If None, it behaves the same as if it was not provided. - let r = match r.flat_option() { + let r = match r.flatten() { Some(r) => { let val = r .payload::() diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index 1cdab8ec3..43fd18c27 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -83,7 +83,7 @@ mod _random { #[pymethod] fn seed(&self, n: OptionalOption) { - let new_rng = match n.flat_option() { + let new_rng = match n.flatten() { None => PyRng::default(), Some(n) => { let (_, mut key) = n.as_bigint().abs().to_u32_digits(); diff --git a/vm/src/stdlib/select.rs b/vm/src/stdlib/select.rs index 0423ae43d..ed4d0b8fa 100644 --- a/vm/src/stdlib/select.rs +++ b/vm/src/stdlib/select.rs @@ -113,7 +113,7 @@ fn select_select( timeout: OptionalOption>, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, PyObjectRef, PyObjectRef)> { - let mut timeout = timeout.flat_option().map(|e| match e { + let mut timeout = timeout.flatten().map(|e| match e { Either::A(f) => f, Either::B(i) => i as f64, }); From fa2d6175c5f89b073589a38e4a95cadab6dccf5a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 28 Jul 2020 13:16:05 -0500 Subject: [PATCH 131/209] =?UTF-8?q?Fix=20flat=5Foption->flatten=20merge=20?= =?UTF-8?q?=F0=9F=98=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vm/src/stdlib/itertools.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 2d981152e..98b57f0f0 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -558,7 +558,7 @@ mod decl { PyItertoolsGroupBy { iterable: iter, - key_func: args.key.flat_option(), + key_func: args.key.flatten(), state: PyMutex::new(GroupByState { current_key: None, current_value: None, From e20a7673e35cd44601e40c072c8a506e10e3375f Mon Sep 17 00:00:00 2001 From: Minjun Shin Date: Wed, 29 Jul 2020 13:09:27 +0900 Subject: [PATCH 132/209] os.execv implementation for unix systems Uses nix::unistd::execv for implementation. --- vm/src/stdlib/os.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 2b1c07eaf..f22991c50 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -47,6 +47,8 @@ use crate::vm::VirtualMachine; #[cfg(unix)] use crate::obj::objdict::PyMapping; #[cfg(unix)] +use crate::obj::objlist::PyListRef; +#[cfg(unix)] use crate::pyobject::PyIterable; #[cfg(unix)] use std::convert::TryFrom; @@ -1271,6 +1273,33 @@ fn os_chmod( Ok(()) } +#[cfg(unix)] +fn os_execv( + path: PyStringRef, + argv_list: Either, + vm: &VirtualMachine, +) -> PyResult<()> { + let path = ffi::CString::new(path.as_str()) + .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; + + let argv: Vec = match argv_list { + Either::A(list) => vm.extract_elements(list.as_object())?, + Either::B(tuple) => vm.extract_elements(tuple.as_object())?, + }; + let argv: Vec<&ffi::CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); + + if argv.is_empty() { + return Err(vm.new_value_error("execv() arg 2 must not be empty".to_owned())); + } + if argv.first().unwrap().to_bytes().is_empty() { + return Err(vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned())); + } + + unistd::execv(&path, &argv) + .map(|_ok| ()) + .map_err(|err| convert_nix_error(vm, err)) +} + fn os_fspath(path: PyPathLike, vm: &VirtualMachine) -> PyResult { path.mode.process_path(path.path, vm) } @@ -2171,6 +2200,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { SupportFunc::new(vm, "lchown", os_lchown, None, None, None), SupportFunc::new(vm, "fchown", os_fchown, Some(true), None, Some(true)), SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "execv", os_execv, None, None, None), ]); let supports_fd = PySet::default().into_ref(vm); let supports_dir_fd = PySet::default().into_ref(vm); @@ -2268,6 +2298,7 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { extend_module!(vm, module, { "chmod" => ctx.new_function(os_chmod), + "execv" => ctx.new_function(os_execv), // TODO: windows "get_inheritable" => ctx.new_function(os_get_inheritable), // TODO: windows "get_blocking" => ctx.new_function(os_get_blocking), "getppid" => ctx.new_function(os_getppid), From 9b48296fd7dde5d44b55a23ba00743a796a7cb9a Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 19 Jul 2020 13:49:51 +1200 Subject: [PATCH 133/209] Copy cgi.py and test_cgi.py from CPython 3.8.3. --- Lib/cgi.py | 992 +++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_cgi.py | 603 ++++++++++++++++++++++++++ 2 files changed, 1595 insertions(+) create mode 100755 Lib/cgi.py create mode 100644 Lib/test/test_cgi.py diff --git a/Lib/cgi.py b/Lib/cgi.py new file mode 100755 index 000000000..c22c71b38 --- /dev/null +++ b/Lib/cgi.py @@ -0,0 +1,992 @@ +#! /usr/local/bin/python + +# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is +# intentionally NOT "/usr/bin/env python". On many systems +# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI +# scripts, and /usr/local/bin is the default directory where Python is +# installed, so /usr/bin/env would be unable to find python. Granted, +# binary installations by Linux vendors often install Python in +# /usr/bin. So let those vendors patch cgi.py to match their choice +# of installation. + +"""Support module for CGI (Common Gateway Interface) scripts. + +This module defines a number of utilities for use by CGI scripts +written in Python. +""" + +# History +# ------- +# +# Michael McLay started this module. Steve Majewski changed the +# interface to SvFormContentDict and FormContentDict. The multipart +# parsing was inspired by code submitted by Andreas Paepcke. Guido van +# Rossum rewrote, reformatted and documented the module and is currently +# responsible for its maintenance. +# + +__version__ = "2.6" + + +# Imports +# ======= + +from io import StringIO, BytesIO, TextIOWrapper +from collections.abc import Mapping +import sys +import os +import urllib.parse +from email.parser import FeedParser +from email.message import Message +import html +import locale +import tempfile + +__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", + "parse_header", "test", "print_exception", "print_environ", + "print_form", "print_directory", "print_arguments", + "print_environ_usage"] + +# Logging support +# =============== + +logfile = "" # Filename to log to, if not empty +logfp = None # File object to log to, if not None + +def initlog(*allargs): + """Write a log message, if there is a log file. + + Even though this function is called initlog(), you should always + use log(); log is a variable that is set either to initlog + (initially), to dolog (once the log file has been opened), or to + nolog (when logging is disabled). + + The first argument is a format string; the remaining arguments (if + any) are arguments to the % operator, so e.g. + log("%s: %s", "a", "b") + will write "a: b" to the log file, followed by a newline. + + If the global logfp is not None, it should be a file object to + which log data is written. + + If the global logfp is None, the global logfile may be a string + giving a filename to open, in append mode. This file should be + world writable!!! If the file can't be opened, logging is + silently disabled (since there is no safe place where we could + send an error message). + + """ + global log, logfile, logfp + if logfile and not logfp: + try: + logfp = open(logfile, "a") + except OSError: + pass + if not logfp: + log = nolog + else: + log = dolog + log(*allargs) + +def dolog(fmt, *args): + """Write a log message to the log file. See initlog() for docs.""" + logfp.write(fmt%args + "\n") + +def nolog(*allargs): + """Dummy function, assigned to log when logging is disabled.""" + pass + +def closelog(): + """Close the log file.""" + global log, logfile, logfp + logfile = '' + if logfp: + logfp.close() + logfp = None + log = initlog + +log = initlog # The current logging function + + +# Parsing functions +# ================= + +# Maximum input we will accept when REQUEST_METHOD is POST +# 0 ==> unlimited input +maxlen = 0 + +def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): + """Parse a query in the environment or from a file (default stdin) + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + """ + if fp is None: + fp = sys.stdin + + # field keys and values (except for files) are returned as strings + # an encoding is required to decode the bytes read from self.fp + if hasattr(fp,'encoding'): + encoding = fp.encoding + else: + encoding = 'latin-1' + + # fp.read() must return bytes + if isinstance(fp, TextIOWrapper): + fp = fp.buffer + + if not 'REQUEST_METHOD' in environ: + environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone + if environ['REQUEST_METHOD'] == 'POST': + ctype, pdict = parse_header(environ['CONTENT_TYPE']) + if ctype == 'multipart/form-data': + return parse_multipart(fp, pdict) + elif ctype == 'application/x-www-form-urlencoded': + clength = int(environ['CONTENT_LENGTH']) + if maxlen and clength > maxlen: + raise ValueError('Maximum content length exceeded') + qs = fp.read(clength).decode(encoding) + else: + qs = '' # Unknown content-type + if 'QUERY_STRING' in environ: + if qs: qs = qs + '&' + qs = qs + environ['QUERY_STRING'] + elif sys.argv[1:]: + if qs: qs = qs + '&' + qs = qs + sys.argv[1] + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + elif 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + else: + if sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, + encoding=encoding) + + +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): + """Parse multipart input. + + Arguments: + fp : input file + pdict: dictionary containing other parameters of content-type header + encoding, errors: request encoding and error handler, passed to + FieldStorage + + Returns a dictionary just like parse_qs(): keys are the field names, each + value is a list of values for that field. For non-file fields, the value + is a list of strings. + """ + # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always + # represented as 7bit US-ASCII. + boundary = pdict['boundary'].decode('ascii') + ctype = "multipart/form-data; boundary={}".format(boundary) + headers = Message() + headers.set_type(ctype) + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, + environ={'REQUEST_METHOD': 'POST'}) + return {k: fs.getlist(k) for k in fs} + +def _parseparam(s): + while s[:1] == ';': + s = s[1:] + end = s.find(';') + while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: + end = s.find(';', end + 1) + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] + +def parse_header(line): + """Parse a Content-type like header. + + Return the main content-type and a dictionary of options. + + """ + parts = _parseparam(';' + line) + key = parts.__next__() + pdict = {} + for p in parts: + i = p.find('=') + if i >= 0: + name = p[:i].strip().lower() + value = p[i+1:].strip() + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + value = value.replace('\\\\', '\\').replace('\\"', '"') + pdict[name] = value + return key, pdict + + +# Classes for field storage +# ========================= + +class MiniFieldStorage: + + """Like FieldStorage, for use when no file uploads are possible.""" + + # Dummy attributes + filename = None + list = None + type = None + file = None + type_options = {} + disposition = None + disposition_options = {} + headers = {} + + def __init__(self, name, value): + """Constructor from field name and value.""" + self.name = name + self.value = value + # self.file = StringIO(value) + + def __repr__(self): + """Return printable representation.""" + return "MiniFieldStorage(%r, %r)" % (self.name, self.value) + + +class FieldStorage: + + """Store a sequence of fields, reading multipart/form-data. + + This class provides naming, typing, files stored on disk, and + more. At the top level, it is accessible like a dictionary, whose + keys are the field names. (Note: None can occur as a field name.) + The items are either a Python list (if there's multiple values) or + another FieldStorage or MiniFieldStorage object. If it's a single + object, it has the following attributes: + + name: the field name, if specified; otherwise None + + filename: the filename, if specified; otherwise None; this is the + client side filename, *not* the file name on which it is + stored (that's a temporary file you don't deal with) + + value: the value as a *string*; for file uploads, this + transparently reads the file every time you request the value + and returns *bytes* + + file: the file(-like) object from which you can read the data *as + bytes* ; None if the data is stored a simple string + + type: the content-type, or None if not specified + + type_options: dictionary of options specified on the content-type + line + + disposition: content-disposition, or None if not specified + + disposition_options: dictionary of corresponding options + + headers: a dictionary(-like) object (sometimes email.message.Message or a + subclass thereof) containing *all* headers + + The class is subclassable, mostly for the purpose of overriding + the make_file() method, which is called internally to come up with + a file open for reading and writing. This makes it possible to + override the default choice of storing all files in a temporary + directory and unlinking them as soon as they have been opened. + + """ + def __init__(self, fp=None, headers=None, outerboundary=b'', + environ=os.environ, keep_blank_values=0, strict_parsing=0, + limit=None, encoding='utf-8', errors='replace', + max_num_fields=None): + """Constructor. Read multipart/* until last part. + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + (not used when the request method is GET) + Can be : + 1. a TextIOWrapper object + 2. an object whose read() and readline() methods return bytes + + headers : header dictionary-like object; default: + taken from environ as per CGI spec + + outerboundary : terminating multipart boundary + (for internal use only) + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + + limit : used internally to read parts of multipart/form-data forms, + to exit from the reading loop when reached. It is the difference + between the form content-length and the number of bytes already + read + + encoding, errors : the encoding and error handler used to decode the + binary stream to strings. Must be the same as the charset defined + for the page sending the form (content-type : meta http-equiv or + header) + + max_num_fields: int. If set, then __init__ throws a ValueError + if there are more than n fields read by parse_qsl(). + + """ + method = 'GET' + self.keep_blank_values = keep_blank_values + self.strict_parsing = strict_parsing + self.max_num_fields = max_num_fields + if 'REQUEST_METHOD' in environ: + method = environ['REQUEST_METHOD'].upper() + self.qs_on_post = None + if method == 'GET' or method == 'HEAD': + if 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + elif sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') + fp = BytesIO(qs) + if headers is None: + headers = {'content-type': + "application/x-www-form-urlencoded"} + if headers is None: + headers = {} + if method == 'POST': + # Set default content-type for POST to what's traditional + headers['content-type'] = "application/x-www-form-urlencoded" + if 'CONTENT_TYPE' in environ: + headers['content-type'] = environ['CONTENT_TYPE'] + if 'QUERY_STRING' in environ: + self.qs_on_post = environ['QUERY_STRING'] + if 'CONTENT_LENGTH' in environ: + headers['content-length'] = environ['CONTENT_LENGTH'] + else: + if not (isinstance(headers, (Mapping, Message))): + raise TypeError("headers must be mapping or an instance of " + "email.message.Message") + self.headers = headers + if fp is None: + self.fp = sys.stdin.buffer + # self.fp.read() must return bytes + elif isinstance(fp, TextIOWrapper): + self.fp = fp.buffer + else: + if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): + raise TypeError("fp must be file pointer") + self.fp = fp + + self.encoding = encoding + self.errors = errors + + if not isinstance(outerboundary, bytes): + raise TypeError('outerboundary must be bytes, not %s' + % type(outerboundary).__name__) + self.outerboundary = outerboundary + + self.bytes_read = 0 + self.limit = limit + + # Process content-disposition header + cdisp, pdict = "", {} + if 'content-disposition' in self.headers: + cdisp, pdict = parse_header(self.headers['content-disposition']) + self.disposition = cdisp + self.disposition_options = pdict + self.name = None + if 'name' in pdict: + self.name = pdict['name'] + self.filename = None + if 'filename' in pdict: + self.filename = pdict['filename'] + self._binary_file = self.filename is not None + + # Process content-type header + # + # Honor any existing content-type header. But if there is no + # content-type header, use some sensible defaults. Assume + # outerboundary is "" at the outer level, but something non-false + # inside a multi-part. The default for an inner part is text/plain, + # but for an outer part it should be urlencoded. This should catch + # bogus clients which erroneously forget to include a content-type + # header. + # + # See below for what we do if there does exist a content-type header, + # but it happens to be something we don't understand. + if 'content-type' in self.headers: + ctype, pdict = parse_header(self.headers['content-type']) + elif self.outerboundary or method != 'POST': + ctype, pdict = "text/plain", {} + else: + ctype, pdict = 'application/x-www-form-urlencoded', {} + self.type = ctype + self.type_options = pdict + if 'boundary' in pdict: + self.innerboundary = pdict['boundary'].encode(self.encoding, + self.errors) + else: + self.innerboundary = b"" + + clen = -1 + if 'content-length' in self.headers: + try: + clen = int(self.headers['content-length']) + except ValueError: + pass + if maxlen and clen > maxlen: + raise ValueError('Maximum content length exceeded') + self.length = clen + if self.limit is None and clen >= 0: + self.limit = clen + + self.list = self.file = None + self.done = 0 + if ctype == 'application/x-www-form-urlencoded': + self.read_urlencoded() + elif ctype[:10] == 'multipart/': + self.read_multi(environ, keep_blank_values, strict_parsing) + else: + self.read_single() + + def __del__(self): + try: + self.file.close() + except AttributeError: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.file.close() + + def __repr__(self): + """Return a printable representation.""" + return "FieldStorage(%r, %r, %r)" % ( + self.name, self.filename, self.value) + + def __iter__(self): + return iter(self.keys()) + + def __getattr__(self, name): + if name != 'value': + raise AttributeError(name) + if self.file: + self.file.seek(0) + value = self.file.read() + self.file.seek(0) + elif self.list is not None: + value = self.list + else: + value = None + return value + + def __getitem__(self, key): + """Dictionary style indexing.""" + if self.list is None: + raise TypeError("not indexable") + found = [] + for item in self.list: + if item.name == key: found.append(item) + if not found: + raise KeyError(key) + if len(found) == 1: + return found[0] + else: + return found + + def getvalue(self, key, default=None): + """Dictionary style get() method, including 'value' lookup.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return value.value + else: + return default + + def getfirst(self, key, default=None): + """ Return the first value received.""" + if key in self: + value = self[key] + if isinstance(value, list): + return value[0].value + else: + return value.value + else: + return default + + def getlist(self, key): + """ Return list of received values.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return [value.value] + else: + return [] + + def keys(self): + """Dictionary style keys() method.""" + if self.list is None: + raise TypeError("not indexable") + return list(set(item.name for item in self.list)) + + def __contains__(self, key): + """Dictionary style __contains__ method.""" + if self.list is None: + raise TypeError("not indexable") + return any(item.name == key for item in self.list) + + def __len__(self): + """Dictionary style len(x) support.""" + return len(self.keys()) + + def __bool__(self): + if self.list is None: + raise TypeError("Cannot be converted to bool.") + return bool(self.list) + + def read_urlencoded(self): + """Internal: read data in query string format.""" + qs = self.fp.read(self.length) + if not isinstance(qs, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(qs).__name__)) + qs = qs.decode(self.encoding, self.errors) + if self.qs_on_post: + qs += '&' + self.qs_on_post + query = urllib.parse.parse_qsl( + qs, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields) + self.list = [MiniFieldStorage(key, value) for key, value in query] + self.skip_lines() + + FieldStorageClass = None + + def read_multi(self, environ, keep_blank_values, strict_parsing): + """Internal: read a part that is itself multipart.""" + ib = self.innerboundary + if not valid_boundary(ib): + raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) + self.list = [] + if self.qs_on_post: + query = urllib.parse.parse_qsl( + self.qs_on_post, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields) + self.list.extend(MiniFieldStorage(key, value) for key, value in query) + + klass = self.FieldStorageClass or self.__class__ + first_line = self.fp.readline() # bytes + if not isinstance(first_line, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(first_line).__name__)) + self.bytes_read += len(first_line) + + # Ensure that we consume the file until we've hit our inner boundary + while (first_line.strip() != (b"--" + self.innerboundary) and + first_line): + first_line = self.fp.readline() + self.bytes_read += len(first_line) + + # Propagate max_num_fields into the sub class appropriately + max_num_fields = self.max_num_fields + if max_num_fields is not None: + max_num_fields -= len(self.list) + + while True: + parser = FeedParser() + hdr_text = b"" + while True: + data = self.fp.readline() + hdr_text += data + if not data.strip(): + break + if not hdr_text: + break + # parser takes strings, not bytes + self.bytes_read += len(hdr_text) + parser.feed(hdr_text.decode(self.encoding, self.errors)) + headers = parser.close() + + # Some clients add Content-Length for part headers, ignore them + if 'content-length' in headers: + del headers['content-length'] + + limit = None if self.limit is None \ + else self.limit - self.bytes_read + part = klass(self.fp, headers, ib, environ, keep_blank_values, + strict_parsing, limit, + self.encoding, self.errors, max_num_fields) + + if max_num_fields is not None: + max_num_fields -= 1 + if part.list: + max_num_fields -= len(part.list) + if max_num_fields < 0: + raise ValueError('Max number of fields exceeded') + + self.bytes_read += part.bytes_read + self.list.append(part) + if part.done or self.bytes_read >= self.length > 0: + break + self.skip_lines() + + def read_single(self): + """Internal: read an atomic part.""" + if self.length >= 0: + self.read_binary() + self.skip_lines() + else: + self.read_lines() + self.file.seek(0) + + bufsize = 8*1024 # I/O buffering size for copy to file + + def read_binary(self): + """Internal: read binary data.""" + self.file = self.make_file() + todo = self.length + if todo >= 0: + while todo > 0: + data = self.fp.read(min(todo, self.bufsize)) # bytes + if not isinstance(data, bytes): + raise ValueError("%s should return bytes, got %s" + % (self.fp, type(data).__name__)) + self.bytes_read += len(data) + if not data: + self.done = -1 + break + self.file.write(data) + todo = todo - len(data) + + def read_lines(self): + """Internal: read lines until EOF or outerboundary.""" + if self._binary_file: + self.file = self.__file = BytesIO() # store data as bytes for files + else: + self.file = self.__file = StringIO() # as strings for other fields + if self.outerboundary: + self.read_lines_to_outerboundary() + else: + self.read_lines_to_eof() + + def __write(self, line): + """line is always bytes, not string""" + if self.__file is not None: + if self.__file.tell() + len(line) > 1000: + self.file = self.make_file() + data = self.__file.getvalue() + self.file.write(data) + self.__file = None + if self._binary_file: + # keep bytes + self.file.write(line) + else: + # decode to string + self.file.write(line.decode(self.encoding, self.errors)) + + def read_lines_to_eof(self): + """Internal: read lines until EOF.""" + while 1: + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + if not line: + self.done = -1 + break + self.__write(line) + + def read_lines_to_outerboundary(self): + """Internal: read lines until outerboundary. + Data is read as bytes: boundaries and line ends must be converted + to bytes for comparisons. + """ + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + delim = b"" + last_line_lfend = True + _read = 0 + while 1: + if self.limit is not None and _read >= self.limit: + break + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + _read += len(line) + if not line: + self.done = -1 + break + if delim == b"\r": + line = delim + line + delim = b"" + if line.startswith(b"--") and last_line_lfend: + strippedline = line.rstrip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + odelim = delim + if line.endswith(b"\r\n"): + delim = b"\r\n" + line = line[:-2] + last_line_lfend = True + elif line.endswith(b"\n"): + delim = b"\n" + line = line[:-1] + last_line_lfend = True + elif line.endswith(b"\r"): + # We may interrupt \r\n sequences if they span the 2**16 + # byte boundary + delim = b"\r" + line = line[:-1] + last_line_lfend = False + else: + delim = b"" + last_line_lfend = False + self.__write(odelim + line) + + def skip_lines(self): + """Internal: skip lines until outer boundary if defined.""" + if not self.outerboundary or self.done: + return + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + last_line_lfend = True + while True: + line = self.fp.readline(1<<16) + self.bytes_read += len(line) + if not line: + self.done = -1 + break + if line.endswith(b"--") and last_line_lfend: + strippedline = line.strip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + last_line_lfend = line.endswith(b'\n') + + def make_file(self): + """Overridable: return a readable & writable file. + + The file will be used as follows: + - data is written to it + - seek(0) + - data is read from it + + The file is opened in binary mode for files, in text mode + for other fields + + This version opens a temporary file for reading and writing, + and immediately deletes (unlinks) it. The trick (on Unix!) is + that the file can still be used, but it can't be opened by + another process, and it will automatically be deleted when it + is closed or when the current process terminates. + + If you want a more permanent file, you derive a class which + overrides this method. If you want a visible temporary file + that is nevertheless automatically deleted when the script + terminates, try defining a __del__ method in a derived class + which unlinks the temporary files you have created. + + """ + if self._binary_file: + return tempfile.TemporaryFile("wb+") + else: + return tempfile.TemporaryFile("w+", + encoding=self.encoding, newline = '\n') + + +# Test/debug code +# =============== + +def test(environ=os.environ): + """Robust test CGI script, usable as main program. + + Write minimal HTTP headers and dump all information provided to + the script in HTML form. + + """ + print("Content-type: text/html") + print() + sys.stderr = sys.stdout + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + print_environ_usage() + def f(): + exec("testing print_exception() -- italics?") + def g(f=f): + f() + print("

What follows is a test, not an actual exception:

") + g() + except: + print_exception() + + print("

Second try with a small maxlen...

") + + global maxlen + maxlen = 50 + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + except: + print_exception() + +def print_exception(type=None, value=None, tb=None, limit=None): + if type is None: + type, value, tb = sys.exc_info() + import traceback + print() + print("

Traceback (most recent call last):

") + list = traceback.format_tb(tb, limit) + \ + traceback.format_exception_only(type, value) + print("
%s%s
" % ( + html.escape("".join(list[:-1])), + html.escape(list[-1]), + )) + del tb + +def print_environ(environ=os.environ): + """Dump the shell environment as HTML.""" + keys = sorted(environ.keys()) + print() + print("

Shell Environment:

") + print("
") + for key in keys: + print("
", html.escape(key), "
", html.escape(environ[key])) + print("
") + print() + +def print_form(form): + """Dump the contents of a form as HTML.""" + keys = sorted(form.keys()) + print() + print("

Form Contents:

") + if not keys: + print("

No form fields.") + print("

") + for key in keys: + print("
" + html.escape(key) + ":", end=' ') + value = form[key] + print("" + html.escape(repr(type(value))) + "") + print("
" + html.escape(repr(value))) + print("
") + print() + +def print_directory(): + """Dump the current directory as HTML.""" + print() + print("

Current Working Directory:

") + try: + pwd = os.getcwd() + except OSError as msg: + print("OSError:", html.escape(str(msg))) + else: + print(html.escape(pwd)) + print() + +def print_arguments(): + print() + print("

Command Line Arguments:

") + print() + print(sys.argv) + print() + +def print_environ_usage(): + """Dump a list of environment variables used by CGI as HTML.""" + print(""" +

These environment variables could have been set:

+
    +
  • AUTH_TYPE +
  • CONTENT_LENGTH +
  • CONTENT_TYPE +
  • DATE_GMT +
  • DATE_LOCAL +
  • DOCUMENT_NAME +
  • DOCUMENT_ROOT +
  • DOCUMENT_URI +
  • GATEWAY_INTERFACE +
  • LAST_MODIFIED +
  • PATH +
  • PATH_INFO +
  • PATH_TRANSLATED +
  • QUERY_STRING +
  • REMOTE_ADDR +
  • REMOTE_HOST +
  • REMOTE_IDENT +
  • REMOTE_USER +
  • REQUEST_METHOD +
  • SCRIPT_NAME +
  • SERVER_NAME +
  • SERVER_PORT +
  • SERVER_PROTOCOL +
  • SERVER_ROOT +
  • SERVER_SOFTWARE +
+In addition, HTTP headers sent by the server may be passed in the +environment as well. Here are some common variable names: +
    +
  • HTTP_ACCEPT +
  • HTTP_CONNECTION +
  • HTTP_HOST +
  • HTTP_PRAGMA +
  • HTTP_REFERER +
  • HTTP_USER_AGENT +
+""") + + +# Utilities +# ========= + +def valid_boundary(s): + import re + if isinstance(s, bytes): + _vb_pattern = b"^[ -~]{0,200}[!-~]$" + else: + _vb_pattern = "^[ -~]{0,200}[!-~]$" + return re.match(_vb_pattern, s) + +# Invoke mainline +# =============== + +# Call test() when this file is run as a script (not imported as a module) +if __name__ == '__main__': + test() diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py new file mode 100644 index 000000000..ab8677199 --- /dev/null +++ b/Lib/test/test_cgi.py @@ -0,0 +1,603 @@ +import cgi +import os +import sys +import tempfile +import unittest +from collections import namedtuple +from io import StringIO, BytesIO +from test import support + +class HackedSysModule: + # The regression test will have real values in sys.argv, which + # will completely confuse the test of the cgi module + argv = [] + stdin = sys.stdin + +cgi.sys = HackedSysModule() + +class ComparableException: + def __init__(self, err): + self.err = err + + def __str__(self): + return str(self.err) + + def __eq__(self, anExc): + if not isinstance(anExc, Exception): + return NotImplemented + return (self.err.__class__ == anExc.__class__ and + self.err.args == anExc.args) + + def __getattr__(self, attr): + return getattr(self.err, attr) + +def do_test(buf, method): + env = {} + if method == "GET": + fp = None + env['REQUEST_METHOD'] = 'GET' + env['QUERY_STRING'] = buf + elif method == "POST": + fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes + env['REQUEST_METHOD'] = 'POST' + env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded' + env['CONTENT_LENGTH'] = str(len(buf)) + else: + raise ValueError("unknown method: %s" % method) + try: + return cgi.parse(fp, env, strict_parsing=1) + except Exception as err: + return ComparableException(err) + +parse_strict_test_cases = [ + ("", ValueError("bad query field: ''")), + ("&", ValueError("bad query field: ''")), + ("&&", ValueError("bad query field: ''")), + (";", ValueError("bad query field: ''")), + (";&;", ValueError("bad query field: ''")), + # Should the next few really be valid? + ("=", {}), + ("=&=", {}), + ("=;=", {}), + # This rest seem to make sense + ("=a", {'': ['a']}), + ("&=a", ValueError("bad query field: ''")), + ("=a&", ValueError("bad query field: ''")), + ("=&a", ValueError("bad query field: 'a'")), + ("b=a", {'b': ['a']}), + ("b+=a", {'b ': ['a']}), + ("a=b=a", {'a': ['b=a']}), + ("a=+b=a", {'a': [' b=a']}), + ("&b=a", ValueError("bad query field: ''")), + ("b&=a", ValueError("bad query field: 'b'")), + ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), + ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", + {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], + 'cuyer': ['r'], + 'expire': ['964546263'], + 'kid': ['130003.300038'], + 'lobale': ['en-US'], + 'order_id': ['0bb2e248638833d48cb7fed300000f1b'], + 'ss': ['env'], + 'view': ['bustomer'], + }), + + ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse", + {'SUBMIT': ['Browse'], + '_assigned_to': ['31392'], + '_category': ['100'], + '_status': ['1'], + 'group_id': ['5470'], + 'set': ['custom'], + }) + ] + +def norm(seq): + return sorted(seq, key=repr) + +def first_elts(list): + return [p[0] for p in list] + +def first_second_elts(list): + return [(p[0], p[1][0]) for p in list] + +def gen_result(data, environ): + encoding = 'latin-1' + fake_stdin = BytesIO(data.encode(encoding)) + fake_stdin.seek(0) + form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding) + + result = {} + for k, v in dict(form).items(): + result[k] = isinstance(v, list) and form.getlist(k) or v.value + + return result + +class CgiTests(unittest.TestCase): + + def test_parse_multipart(self): + fp = BytesIO(POSTDATA.encode('latin1')) + env = {'boundary': BOUNDARY.encode('latin1'), + 'CONTENT-LENGTH': '558'} + result = cgi.parse_multipart(fp, env) + expected = {'submit': [' Add '], 'id': ['1234'], + 'file': [b'Testing 123.\n'], 'title': ['']} + self.assertEqual(result, expected) + + def test_parse_multipart_invalid_encoding(self): + BOUNDARY = "JfISa01" + POSTDATA = """--JfISa01 +Content-Disposition: form-data; name="submit-name" +Content-Length: 3 + +\u2603 +--JfISa01""" + fp = BytesIO(POSTDATA.encode('utf8')) + env = {'boundary': BOUNDARY.encode('latin1'), + 'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))} + result = cgi.parse_multipart(fp, env, encoding="ascii", + errors="surrogateescape") + expected = {'submit-name': ["\udce2\udc98\udc83"]} + self.assertEqual(result, expected) + self.assertEqual("\u2603".encode('utf8'), + result["submit-name"][0].encode('utf8', 'surrogateescape')) + + def test_fieldstorage_properties(self): + fs = cgi.FieldStorage() + self.assertFalse(fs) + self.assertIn("FieldStorage", repr(fs)) + self.assertEqual(list(fs), list(fs.keys())) + fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue')) + self.assertTrue(fs) + + def test_fieldstorage_invalid(self): + self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj", + environ={"REQUEST_METHOD":"PUT"}) + self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar") + fs = cgi.FieldStorage(headers={'content-type':'text/plain'}) + self.assertRaises(TypeError, bool, fs) + + def test_strict(self): + for orig, expect in parse_strict_test_cases: + # Test basic parsing + d = do_test(orig, "GET") + self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig)) + d = do_test(orig, "POST") + self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig)) + + env = {'QUERY_STRING': orig} + fs = cgi.FieldStorage(environ=env) + if isinstance(expect, dict): + # test dict interface + self.assertEqual(len(expect), len(fs)) + self.assertCountEqual(expect.keys(), fs.keys()) + ##self.assertEqual(norm(expect.values()), norm(fs.values())) + ##self.assertEqual(norm(expect.items()), norm(fs.items())) + self.assertEqual(fs.getvalue("nonexistent field", "default"), "default") + # test individual fields + for key in expect.keys(): + expect_val = expect[key] + self.assertIn(key, fs) + if len(expect_val) > 1: + self.assertEqual(fs.getvalue(key), expect_val) + else: + self.assertEqual(fs.getvalue(key), expect_val[0]) + + def test_log(self): + cgi.log("Testing") + + cgi.logfp = StringIO() + cgi.initlog("%s", "Testing initlog 1") + cgi.log("%s", "Testing log 2") + self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n") + if os.path.exists(os.devnull): + cgi.logfp = None + cgi.logfile = os.devnull + cgi.initlog("%s", "Testing log 3") + self.addCleanup(cgi.closelog) + cgi.log("Testing log 4") + + def test_fieldstorage_readline(self): + # FieldStorage uses readline, which has the capacity to read all + # contents of the input file into memory; we use readline's size argument + # to prevent that for files that do not contain any newlines in + # non-GET/HEAD requests + class TestReadlineFile: + def __init__(self, file): + self.file = file + self.numcalls = 0 + + def readline(self, size=None): + self.numcalls += 1 + if size: + return self.file.readline(size) + else: + return self.file.readline() + + def __getattr__(self, name): + file = self.__dict__['file'] + a = getattr(file, name) + if not isinstance(a, int): + setattr(self, name, a) + return a + + f = TestReadlineFile(tempfile.TemporaryFile("wb+")) + self.addCleanup(f.close) + f.write(b'x' * 256 * 1024) + f.seek(0) + env = {'REQUEST_METHOD':'PUT'} + fs = cgi.FieldStorage(fp=f, environ=env) + self.addCleanup(fs.file.close) + # if we're not chunking properly, readline is only called twice + # (by read_binary); if we are chunking properly, it will be called 5 times + # as long as the chunksize is 1 << 16. + self.assertGreater(f.numcalls, 2) + f.close() + + def test_fieldstorage_multipart(self): + #Test basic FieldStorage multipart parsing + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': '558'} + fp = BytesIO(POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 4) + expect = [{'name':'id', 'filename':None, 'value':'1234'}, + {'name':'title', 'filename':None, 'value':''}, + {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, + {'name':'submit', 'filename':None, 'value':' Add '}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_leading_whitespace(self): + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': '560'} + # Add some leading whitespace to our post data that will cause the + # first line to not be the innerboundary. + fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 4) + expect = [{'name':'id', 'filename':None, 'value':'1234'}, + {'name':'title', 'filename':None, 'value':''}, + {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, + {'name':'submit', 'filename':None, 'value':' Add '}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_non_ascii(self): + #Test basic FieldStorage multipart parsing + env = {'REQUEST_METHOD':'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH':'558'} + for encoding in ['iso-8859-1','utf-8']: + fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding)) + fs = cgi.FieldStorage(fp, environ=env,encoding=encoding) + self.assertEqual(len(fs.list), 1) + expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_maxline(self): + # Issue #18167 + maxline = 1 << 16 + self.maxDiff = None + def check(content): + data = """---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +%s +---123-- +""".replace('\n', '\r\n') % content + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'REQUEST_METHOD': 'POST', + } + self.assertEqual(gen_result(data, environ), + {'upload': content.encode('latin1')}) + check('x' * (maxline - 1)) + check('x' * (maxline - 1) + '\r') + check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) + + def test_fieldstorage_multipart_w3c(self): + # Test basic FieldStorage multipart parsing (W3C sample) + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3), + 'CONTENT_LENGTH': str(len(POSTDATA_W3))} + fp = BytesIO(POSTDATA_W3.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 2) + self.assertEqual(fs.list[0].name, 'submit-name') + self.assertEqual(fs.list[0].value, 'Larry') + self.assertEqual(fs.list[1].name, 'files') + files = fs.list[1].value + self.assertEqual(len(files), 2) + expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'}, + {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}] + for x in range(len(files)): + for k, exp in expect[x].items(): + got = getattr(files[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_part_content_length(self): + BOUNDARY = "JfISa01" + POSTDATA = """--JfISa01 +Content-Disposition: form-data; name="submit-name" +Content-Length: 5 + +Larry +--JfISa01""" + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': str(len(POSTDATA))} + fp = BytesIO(POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 1) + self.assertEqual(fs.list[0].name, 'submit-name') + self.assertEqual(fs.list[0].value, 'Larry') + + def test_field_storage_multipart_no_content_length(self): + fp = BytesIO(b"""--MyBoundary +Content-Disposition: form-data; name="my-arg"; filename="foo" + +Test + +--MyBoundary-- +""") + env = { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary", + "wsgi.input": fp, + } + fields = cgi.FieldStorage(fp, environ=env) + + self.assertEqual(len(fields["my-arg"].file.read()), 5) + + def test_fieldstorage_as_context_manager(self): + fp = BytesIO(b'x' * 10) + env = {'REQUEST_METHOD': 'PUT'} + with cgi.FieldStorage(fp=fp, environ=env) as fs: + content = fs.file.read() + self.assertFalse(fs.file.closed) + self.assertTrue(fs.file.closed) + self.assertEqual(content, 'x' * 10) + with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'): + fs.file.read() + + _qs_result = { + 'key1': 'value1', + 'key2': ['value2x', 'value2y'], + 'key3': 'value3', + 'key4': 'value4' + } + def testQSAndUrlEncode(self): + data = "key2=value2x&key3=value3&key4=value4" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded', + 'QUERY_STRING': 'key1=value1&key2=value2y', + 'REQUEST_METHOD': 'POST', + } + v = gen_result(data, environ) + self.assertEqual(self._qs_result, v) + + def test_max_num_fields(self): + # For application/x-www-form-urlencoded + data = '&'.join(['a=a']*11) + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded', + 'REQUEST_METHOD': 'POST', + } + + with self.assertRaises(ValueError): + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=10, + ) + + # For multipart/form-data + data = """---123 +Content-Disposition: form-data; name="a" + +3 +---123 +Content-Type: application/x-www-form-urlencoded + +a=4 +---123 +Content-Type: application/x-www-form-urlencoded + +a=5 +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'a=1&a=2', + 'REQUEST_METHOD': 'POST', + } + + # 2 GET entities + # 1 top level POST entities + # 1 entity within the second POST entity + # 1 entity within the third POST entity + with self.assertRaises(ValueError): + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=4, + ) + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=5, + ) + + def testQSAndFormData(self): + data = """---123 +Content-Disposition: form-data; name="key2" + +value2y +---123 +Content-Disposition: form-data; name="key3" + +value3 +---123 +Content-Disposition: form-data; name="key4" + +value4 +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'key1=value1&key2=value2x', + 'REQUEST_METHOD': 'POST', + } + v = gen_result(data, environ) + self.assertEqual(self._qs_result, v) + + def testQSAndFormDataFile(self): + data = """---123 +Content-Disposition: form-data; name="key2" + +value2y +---123 +Content-Disposition: form-data; name="key3" + +value3 +---123 +Content-Disposition: form-data; name="key4" + +value4 +---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +this is the content of the fake file + +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'key1=value1&key2=value2x', + 'REQUEST_METHOD': 'POST', + } + result = self._qs_result.copy() + result.update({ + 'upload': b'this is the content of the fake file\n' + }) + v = gen_result(data, environ) + self.assertEqual(result, v) + + def test_parse_header(self): + self.assertEqual( + cgi.parse_header("text/plain"), + ("text/plain", {})) + self.assertEqual( + cgi.parse_header("text/vnd.just.made.this.up ; "), + ("text/vnd.just.made.this.up", {})) + self.assertEqual( + cgi.parse_header("text/plain;charset=us-ascii"), + ("text/plain", {"charset": "us-ascii"})) + self.assertEqual( + cgi.parse_header('text/plain ; charset="us-ascii"'), + ("text/plain", {"charset": "us-ascii"})) + self.assertEqual( + cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'), + ("text/plain", {"charset": "us-ascii", "another": "opt"})) + self.assertEqual( + cgi.parse_header('attachment; filename="silly.txt"'), + ("attachment", {"filename": "silly.txt"})) + self.assertEqual( + cgi.parse_header('attachment; filename="strange;name"'), + ("attachment", {"filename": "strange;name"})) + self.assertEqual( + cgi.parse_header('attachment; filename="strange;name";size=123;'), + ("attachment", {"filename": "strange;name", "size": "123"})) + self.assertEqual( + cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'), + ("form-data", {"name": "files", "filename": 'fo"o;bar'})) + + def test_all(self): + blacklist = {"logfile", "logfp", "initlog", "dolog", "nolog", + "closelog", "log", "maxlen", "valid_boundary"} + support.check__all__(self, cgi, blacklist=blacklist) + + +BOUNDARY = "---------------------------721837373350705526688164684" + +POSTDATA = """-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="id" + +1234 +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="title" + + +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="file"; filename="test.txt" +Content-Type: text/plain + +Testing 123. + +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="submit" + + Add\x20 +-----------------------------721837373350705526688164684-- +""" + +POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="id" + +\xe7\xf1\x80 +-----------------------------721837373350705526688164684 +""" + +# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4 +BOUNDARY_W3 = "AaB03x" +POSTDATA_W3 = """--AaB03x +Content-Disposition: form-data; name="submit-name" + +Larry +--AaB03x +Content-Disposition: form-data; name="files" +Content-Type: multipart/mixed; boundary=BbC04y + +--BbC04y +Content-Disposition: file; filename="file1.txt" +Content-Type: text/plain + +... contents of file1.txt ... +--BbC04y +Content-Disposition: file; filename="file2.gif" +Content-Type: image/gif +Content-Transfer-Encoding: binary + +...contents of file2.gif... +--BbC04y-- +--AaB03x-- +""" + +if __name__ == '__main__': + unittest.main() From cfc7a7c7343506bebd7bb5908fbb2995c9b851c5 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 19 Jul 2020 13:53:04 +1200 Subject: [PATCH 134/209] Copy updated urllib from CPython 3.8.3. --- Lib/urllib/error.py | 6 +- Lib/urllib/parse.py | 268 ++++++++++++++++++++++++++++++-------- Lib/urllib/request.py | 248 +++++++++++++++++++---------------- Lib/urllib/robotparser.py | 46 +++++-- 4 files changed, 386 insertions(+), 182 deletions(-) diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16..8cd901f13 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,14 +16,10 @@ import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. - # It sets self.args for compatibility with other EnvironmentError + # It sets self.args for compatibility with other OSError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. def __init__(self, reason, filename=None): diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08..e2b6f133e 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -30,6 +30,7 @@ test_urlparse.py provides a good indicator of parsing behavior. import re import sys import collections +import warnings __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -38,29 +39,37 @@ __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' @@ -147,16 +156,22 @@ class _NetlocResultMixinBase(object): def hostname(self): hostname = self._hostinfo[0] if not hostname: - hostname = None - elif hostname is not None: - hostname = hostname.lower() - return hostname + return None + # Scoped IPv6 address may have zone info, which must not be lowercased + # like http://[fe80::822a:a8ff:fe49:470c%tESt]:1234/keys + separator = '%' if isinstance(hostname, str) else b'%' + hostname, percent, zone = hostname.partition(separator) + return hostname.lower() + percent + zone @property def port(self): port = self._hostinfo[1] if port is not None: - port = int(port, 10) + try: + port = int(port, 10) + except ValueError: + message = f'Port could not be cast to integer value as {port!r}' + raise ValueError(message) from None if not ( 0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port @@ -274,7 +289,7 @@ by reference to a primary resource and additional identifying information. """ _ParseResultBase.__doc__ = """ -ParseResult(scheme, netloc, path, params, query, fragment) +ParseResult(scheme, netloc, path, params, query, fragment) A 6-tuple that contains components of a parsed URL. """ @@ -381,6 +396,24 @@ def _splitnetloc(url, start=0): delim = min(delim, wdelim) # use earliest delim position return url[start:delim], url[delim:] # return (domain, rest) +def _checknetloc(netloc): + if not netloc or netloc.isascii(): + return + # looking for characters like \u2100 that expand to 'a/c' + # IDNA uses NFKC equivalence, so normalize for this check + import unicodedata + n = netloc.replace('@', '') # ignore characters already included + n = n.replace(':', '') # but not the surrounding text + n = n.replace('#', '') + n = n.replace('?', '') + netloc2 = unicodedata.normalize('NFKC', n) + if n == netloc2: + return + for c in '/?#@:': + if c in netloc2: + raise ValueError("netloc '" + netloc + "' contains invalid " + + "characters under NFKC normalization") + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -399,7 +432,6 @@ def urlsplit(url, scheme='', allow_fragments=True): i = url.find(':') if i > 0: if url[:i] == 'http': # optimize the common case - scheme = url[:i].lower() url = url[i+1:] if url[:2] == '//': netloc, url = _splitnetloc(url, 2) @@ -410,7 +442,8 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) + _checknetloc(netloc) + v = SplitResult('http', netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) for c in url[:i]: @@ -433,6 +466,7 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) + _checknetloc(netloc) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) @@ -574,7 +608,7 @@ def unquote_to_bytes(string): # if the function is never called global _hextobyte if _hextobyte is None: - _hextobyte = {(a + b).encode(): bytes([int(a + b, 16)]) + _hextobyte = {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig for b in _hexdig} for item in bits[1:]: try: @@ -612,8 +646,9 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): + encoding='utf-8', errors='replace', max_num_fields=None): """Parse a query given as a string argument. Arguments: @@ -633,10 +668,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + max_num_fields: int. If set, then throws a ValueError if there + are more than n fields read by parse_qsl(). + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, - encoding=encoding, errors=errors) + encoding=encoding, errors=errors, + max_num_fields=max_num_fields) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -644,30 +685,43 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): + encoding='utf-8', errors='replace', max_num_fields=None): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + max_num_fields: int. If set, then throws a ValueError + if there are more than n fields read by parse_qsl(). + + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) + + # If max_num_fields is defined then check that the number of fields + # is less than max_num_fields. This prevents a memory exhaustion DOS + # attack via post bodies with many fields. + if max_num_fields is not None: + num_fields = 1 + qs.count('&') + qs.count(';') + if max_num_fields < num_fields: + raise ValueError('Max number of fields exceeded') + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] r = [] for name_value in pairs: @@ -704,7 +758,7 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): _ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' b'abcdefghijklmnopqrstuvwxyz' b'0123456789' - b'_.-') + b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) _safe_quoters = {} @@ -734,22 +788,32 @@ def quote(string, safe='/', encoding=None, errors=None): """quote('abc def') -> 'abc%20def' Each part of a URL, e.g. the path info, the query, etc., has a - different set of reserved characters that must be quoted. + different set of reserved characters that must be quoted. The + quote function offers a cautious (not minimal) way to quote a + string for most of these parts. - RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists - the following reserved characters. + RFC 3986 Uniform Resource Identifier (URI): Generic Syntax lists + the following (un)reserved characters. - reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | - "$" | "," + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + reserved = gen-delims / sub-delims + gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" - Each of these characters is reserved in some component of a URL, + Each of the reserved characters is reserved in some component of a URL, but not necessarily in all of them. - By default, the quote function is intended for quoting the path - section of a URL. Thus, it will not encode '/'. This character - is reserved, but in typical usage the quote function is being - called on a path where the existing slash characters are used as - reserved characters. + The quote function %-escapes all characters that are neither in the + unreserved chars ("always safe") nor the additional chars set via the + safe arg. + + The default for the safe arg is '/'. The character is reserved, but in + typical usage the quote function is being called on a path where the + existing slash characters are to be preserved. + + Python 3.7 updates from using RFC 2396 to RFC 3986 to quote URL strings. + Now, "~" is included in the set of unreserved characters. string and safe may be either str or bytes objects. encoding and errors must not be specified if string is a bytes object. @@ -893,7 +957,14 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None, l.append(k + '=' + elt) return '&'.join(l) + def to_bytes(url): + warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8", + DeprecationWarning, stacklevel=2) + return _to_bytes(url) + + +def _to_bytes(url): """to_bytes(u"URL") --> 'URL'.""" # Most URL schemes require ASCII. If that changes, the conversion # can be relaxed. @@ -906,16 +977,29 @@ def to_bytes(url): " contains non-ASCII characters") return url + def unwrap(url): - """unwrap('') --> 'type://host/path'.""" + """Transform a string like '' into 'scheme://host/path'. + + The string is returned unchanged if it's not a wrapped URL. + """ url = str(url).strip() if url[:1] == '<' and url[-1:] == '>': url = url[1:-1].strip() - if url[:4] == 'URL:': url = url[4:].strip() + if url[:4] == 'URL:': + url = url[4:].strip() return url -_typeprog = None + def splittype(url): + warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splittype(url) + + +_typeprog = None +def _splittype(url): """splittype('type:opaquestring') --> 'type', 'opaquestring'.""" global _typeprog if _typeprog is None: @@ -927,12 +1011,20 @@ def splittype(url): return scheme.lower(), data return None, url -_hostprog = None + def splithost(url): + warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splithost(url) + + +_hostprog = None +def _splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: @@ -942,32 +1034,64 @@ def splithost(url): return host_port, path return None, url + def splituser(host): + warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splituser(host) + + +def _splituser(host): """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" user, delim, host = host.rpartition('@') return (user if delim else None), host + def splitpasswd(user): + warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitpasswd(user) + + +def _splitpasswd(user): """splitpasswd('user:passwd') -> 'user', 'passwd'.""" user, delim, passwd = user.partition(':') return user, (passwd if delim else None) + +def splitport(host): + warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitport(host) + + # splittag('/path#tag') --> '/path', 'tag' _portprog = None -def splitport(host): +def _splitport(host): """splitport('host:port') --> 'host', 'port'.""" global _portprog if _portprog is None: - _portprog = re.compile('(.*):([0-9]*)$', re.DOTALL) + _portprog = re.compile('(.*):([0-9]*)', re.DOTALL) - match = _portprog.match(host) + match = _portprog.fullmatch(host) if match: host, port = match.groups() if port: return host, port return host, None + def splitnport(host, defport=-1): + warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitnport(host, defport) + + +def _splitnport(host, defport=-1): """Split host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. Return numerical port if a valid number are found after ':'. @@ -983,27 +1107,59 @@ def splitnport(host, defport=-1): return host, nport return host, defport + def splitquery(url): + warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitquery(url) + + +def _splitquery(url): """splitquery('/path?query') --> '/path', 'query'.""" path, delim, query = url.rpartition('?') if delim: return path, query return url, None + def splittag(url): + warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splittag(url) + + +def _splittag(url): """splittag('/path#tag') --> '/path', 'tag'.""" path, delim, tag = url.rpartition('#') if delim: return path, tag return url, None + def splitattr(url): + warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, " + "use urllib.parse.urlparse() instead", + DeprecationWarning, stacklevel=2) + return _splitattr(url) + + +def _splitattr(url): """splitattr('/path;attr1=value1;attr2=value2;...') -> '/path', ['attr1=value1', 'attr2=value2', ...].""" words = url.split(';') return words[0], words[1:] + def splitvalue(attr): + warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, " + "use urllib.parse.parse_qsl() instead", + DeprecationWarning, stacklevel=2) + return _splitvalue(attr) + + +def _splitvalue(attr): """splitvalue('attr=value') --> 'attr', 'value'.""" attr, delim, value = attr.partition('=') return attr, (value if delim else None) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 5f15b74f4..e44073886 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -94,7 +94,6 @@ import socket import string import sys import time -import collections import tempfile import contextlib import warnings @@ -103,8 +102,8 @@ import warnings from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, - splittype, splithost, splitport, splituser, splitpasswd, - splitattr, splitquery, splitvalue, splittag, to_bytes, + _splittype, _splithost, _splitport, _splituser, _splitpasswd, + _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook @@ -199,7 +198,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, global _opener if cafile or capath or cadefault: import warnings - warnings.warn("cafile, cpath and cadefault are deprecated, use a " + warnings.warn("cafile, capath and cadefault are deprecated, use a " "custom context instead.", DeprecationWarning, 2) if context is not None: raise ValueError( @@ -243,7 +242,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None): Returns a tuple containing the path to the newly created data file as well as the resulting HTTPMessage object. """ - url_type, path = splittype(url) + url_type, path = _splittype(url) with contextlib.closing(urlopen(url, data)) as fp: headers = fp.info() @@ -351,7 +350,7 @@ class Request: def full_url(self, url): # unwrap('') --> 'type://host/path' self._full_url = unwrap(url) - self._full_url, self.fragment = splittag(self._full_url) + self._full_url, self.fragment = _splittag(self._full_url) self._parse() @full_url.deleter @@ -379,10 +378,10 @@ class Request: self.data = None def _parse(self): - self.type, rest = splittype(self._full_url) + self.type, rest = _splittype(self._full_url) if self.type is None: raise ValueError("unknown url type: %r" % self.full_url) - self.host, self.selector = splithost(rest) + self.host, self.selector = _splithost(rest) if self.host: self.host = unquote(self.host) @@ -427,8 +426,7 @@ class Request: self.unredirected_hdrs.pop(header_name, None) def header_items(self): - hdrs = self.unredirected_hdrs.copy() - hdrs.update(self.headers) + hdrs = {**self.unredirected_hdrs, **self.headers} return list(hdrs.items()) class OpenerDirector: @@ -523,6 +521,7 @@ class OpenerDirector: meth = getattr(processor, meth_name) req = meth(req) + sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method()) response = self._open(req, data) # post-process response @@ -684,8 +683,8 @@ class HTTPRedirectHandler(BaseHandler): newurl = newurl.replace(' ', '%20') CONTENT_HEADERS = ("content-length", "content-type") - newheaders = dict((k, v) for k, v in req.headers.items() - if k.lower() not in CONTENT_HEADERS) + newheaders = {k: v for k, v in req.headers.items() + if k.lower() not in CONTENT_HEADERS} return Request(newurl, headers=newheaders, origin_req_host=req.origin_req_host, @@ -769,7 +768,7 @@ def _parse_proxy(proxy): According to RFC 3986, having an authority component means the URL must have two slashes after the scheme. """ - scheme, r_scheme = splittype(proxy) + scheme, r_scheme = _splittype(proxy) if not r_scheme.startswith("/"): # authority scheme = None @@ -784,9 +783,9 @@ def _parse_proxy(proxy): if end == -1: end = None authority = r_scheme[2:end] - userinfo, hostport = splituser(authority) + userinfo, hostport = _splituser(authority) if userinfo is not None: - user, password = splitpasswd(userinfo) + user, password = _splitpasswd(userinfo) else: user = password = None return scheme, user, password, hostport @@ -801,6 +800,7 @@ class ProxyHandler(BaseHandler): assert hasattr(proxies, 'keys'), "proxies must be a mapping" self.proxies = proxies for type, url in proxies.items(): + type = type.lower() setattr(self, '%s_open' % type, lambda r, proxy=url, type=type, meth=self.proxy_open: meth(r, proxy, type)) @@ -846,7 +846,7 @@ class HTTPPasswordMgr: self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - [self.reduce_uri(u, default_port) for u in uri]) + self.reduce_uri(u, default_port) for u in uri) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): @@ -873,7 +873,7 @@ class HTTPPasswordMgr: scheme = None authority = uri path = '/' - host, port = splitport(authority) + host, port = _splitport(authority) if default_port and port is None and scheme is not None: dport = {"http": 80, "https": 443, @@ -945,8 +945,15 @@ class AbstractBasicAuthHandler: # allow for double- and single-quoted realm values # (single quotes are a violation of the RFC, but appear in the wild) - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\']?)([^"\']*)\\2', re.I) + rx = re.compile('(?:^|,)' # start of the string or ',' + '[ \t]*' # optional whitespaces + '([^ \t]+)' # scheme like "Basic" + '[ \t]+' # mandatory whitespaces + # realm=xxx + # realm='xxx' + # realm="xxx" + 'realm=(["\']?)([^"\']*)\\2', + re.I) # XXX could pre-emptively send auth info already accepted (RFC 2617, # end of section 2, and section 1.2 immediately after "credentials" @@ -958,27 +965,51 @@ class AbstractBasicAuthHandler: self.passwd = password_mgr self.add_password = self.passwd.add_password + def _parse_realm(self, header): + # parse WWW-Authenticate header: accept multiple challenges per header + found_challenge = False + for mo in AbstractBasicAuthHandler.rx.finditer(header): + scheme, quote, realm = mo.groups() + if quote not in ['"', "'"]: + warnings.warn("Basic Auth Realm was unquoted", + UserWarning, 3) + + yield (scheme, realm) + + found_challenge = True + + if not found_challenge: + if header: + scheme = header.split()[0] + else: + scheme = '' + yield (scheme, None) + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority - # XXX could be multiple headers - authreq = headers.get(authreq, None) + headers = headers.get_all(authreq) + if not headers: + # no header found + return - if authreq: - scheme = authreq.split()[0] - if scheme.lower() != 'basic': - raise ValueError("AbstractBasicAuthHandler does not" - " support the following scheme: '%s'" % - scheme) - else: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if quote not in ['"',"'"]: - warnings.warn("Basic Auth Realm was unquoted", - UserWarning, 2) - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + unsupported = None + for header in headers: + for scheme, realm in self._parse_realm(header): + if scheme.lower() != 'basic': + unsupported = scheme + continue + + if realm is not None: + # Use the first matching Basic challenge. + # Ignore following challenges even if they use the Basic + # scheme. + return self.retry_http_basic_auth(host, req, realm) + + if unsupported is not None: + raise ValueError("AbstractBasicAuthHandler does not " + "support the following scheme: %r" + % (scheme,)) def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -1144,7 +1175,11 @@ class AbstractDigestAuthHandler: A2 = "%s:%s" % (req.get_method(), # XXX selector: what about proxies and full urls req.selector) - if qop == 'auth': + # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` + # or `auth-int` to the response back. we use `auth` to send the response back. + if qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + elif 'auth' in qop.split(','): if nonce == self.last_nonce: self.nonce_count += 1 else: @@ -1152,10 +1187,8 @@ class AbstractDigestAuthHandler: self.last_nonce = nonce ncvalue = '%08x' % self.nonce_count cnonce = self.get_cnonce(nonce) - noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2)) respdig = KD(H(A1), noncebit) - elif qop is None: - respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) else: # XXX handle auth-int. raise URLError("qop '%s' is not supported." % qop) @@ -1262,8 +1295,8 @@ class AbstractHTTPHandler(BaseHandler): sel_host = host if request.has_proxy(): - scheme, sel = splittype(request.selector) - sel_host, sel_path = splithost(sel) + scheme, sel = _splittype(request.selector) + sel_host, sel_path = _splithost(sel) if not request.has_header('Host'): request.add_unredirected_header('Host', sel_host) for name, value in self.parent.addheaders: @@ -1287,8 +1320,8 @@ class AbstractHTTPHandler(BaseHandler): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update(dict((k, v) for k, v in req.headers.items() - if k not in headers)) + headers.update({k: v for k, v in req.headers.items() + if k not in headers}) # TODO(jhylton): Should this be redesigned to handle # persistent connections? @@ -1300,7 +1333,7 @@ class AbstractHTTPHandler(BaseHandler): # So make sure the connection gets closed after the (only) # request. headers["Connection"] = "close" - headers = dict((name.title(), val) for name, val in headers.items()) + headers = {name.title(): val for name, val in headers.items()} if req._tunnel_host: tunnel_headers = {} @@ -1479,7 +1512,7 @@ class FileHandler(BaseHandler): 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified)) if host: - host, port = splitport(host) + host, port = _splitport(host) if not host or \ (not port and _safe_gethostbyname(host) in self.get_names()): if host: @@ -1488,7 +1521,6 @@ class FileHandler(BaseHandler): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1505,16 +1537,16 @@ class FTPHandler(BaseHandler): host = req.host if not host: raise URLError('ftp error: no host given') - host, port = splitport(host) + host, port = _splitport(host) if port is None: port = ftplib.FTP_PORT else: port = int(port) # username/password handling - user, host = splituser(host) + user, host = _splituser(host) if user: - user, passwd = splitpasswd(user) + user, passwd = _splitpasswd(user) else: passwd = None host = unquote(host) @@ -1525,7 +1557,7 @@ class FTPHandler(BaseHandler): host = socket.gethostbyname(host) except OSError as msg: raise URLError(msg) - path, attrs = splitattr(req.selector) + path, attrs = _splitattr(req.selector) dirs = path.split('/') dirs = list(map(unquote, dirs)) dirs, file = dirs[:-1], dirs[-1] @@ -1535,7 +1567,7 @@ class FTPHandler(BaseHandler): fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' for attr in attrs: - attr, value = splitvalue(attr) + attr, value = _splitvalue(attr) if attr.lower() == 'type' and \ value in ('a', 'A', 'i', 'I', 'd', 'D'): type = value.upper() @@ -1658,14 +1690,10 @@ else: of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need @@ -1733,26 +1761,26 @@ class URLopener: # External interface def open(self, fullurl, data=None): """Use URLopener().open(file) instead of open(file, 'r').""" - fullurl = unwrap(to_bytes(fullurl)) + fullurl = unwrap(_to_bytes(fullurl)) fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|") if self.tempcache and fullurl in self.tempcache: filename, headers = self.tempcache[fullurl] fp = open(filename, 'rb') return addinfourl(fp, headers, fullurl) - urltype, url = splittype(fullurl) + urltype, url = _splittype(fullurl) if not urltype: urltype = 'file' if urltype in self.proxies: proxy = self.proxies[urltype] - urltype, proxyhost = splittype(proxy) - host, selector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + host, selector = _splithost(proxyhost) url = (host, fullurl) # Signal special case to open_*() else: proxy = None name = 'open_' + urltype self.type = urltype name = name.replace('-', '_') - if not hasattr(self, name): + if not hasattr(self, name) or name == 'open_local_file': if proxy: return self.open_unknown_proxy(proxy, fullurl, data) else: @@ -1769,28 +1797,28 @@ class URLopener: def open_unknown(self, fullurl, data=None): """Overridable interface to open unknown URL type.""" - type, url = splittype(fullurl) + type, url = _splittype(fullurl) raise OSError('url error', 'unknown url type', type) def open_unknown_proxy(self, proxy, fullurl, data=None): """Overridable interface to open unknown URL type.""" - type, url = splittype(fullurl) + type, url = _splittype(fullurl) raise OSError('url error', 'invalid proxy for %s' % type, proxy) # External interface def retrieve(self, url, filename=None, reporthook=None, data=None): """retrieve(url) returns (filename, headers) for a local object or (tempfilename, headers) for a remote object.""" - url = unwrap(to_bytes(url)) + url = unwrap(_to_bytes(url)) if self.tempcache and url in self.tempcache: return self.tempcache[url] - type, url1 = splittype(url) + type, url1 = _splittype(url) if filename is None and (not type or type == 'file'): try: fp = self.open_local_file(url1) hdrs = fp.info() fp.close() - return url2pathname(splithost(url1)[1]), hdrs + return url2pathname(_splithost(url1)[1]), hdrs except OSError as msg: pass fp = self.open(url, data) @@ -1799,11 +1827,10 @@ class URLopener: if filename: tfp = open(filename, 'wb') else: - import tempfile - garbage, path = splittype(url) - garbage, path = splithost(path or "") - path, garbage = splitquery(path or "") - path, garbage = splitattr(path or "") + garbage, path = _splittype(url) + garbage, path = _splithost(path or "") + path, garbage = _splitquery(path or "") + path, garbage = _splitattr(path or "") suffix = os.path.splitext(path)[1] (fd, filename) = tempfile.mkstemp(suffix) self.__tempfiles.append(filename) @@ -1860,25 +1887,25 @@ class URLopener: user_passwd = None proxy_passwd= None if isinstance(url, str): - host, selector = splithost(url) + host, selector = _splithost(url) if host: - user_passwd, host = splituser(host) + user_passwd, host = _splituser(host) host = unquote(host) realhost = host else: host, selector = url # check whether the proxy contains authorization information - proxy_passwd, host = splituser(host) + proxy_passwd, host = _splituser(host) # now we proceed with the url we want to obtain - urltype, rest = splittype(selector) + urltype, rest = _splittype(selector) url = rest user_passwd = None if urltype.lower() != 'http': realhost = None else: - realhost, rest = splithost(rest) + realhost, rest = _splithost(rest) if realhost: - user_passwd, realhost = splituser(realhost) + user_passwd, realhost = _splituser(realhost) if user_passwd: selector = "%s://%s%s" % (urltype, realhost, rest) if proxy_bypass(realhost): @@ -1984,7 +2011,7 @@ class URLopener: """Use local file.""" import email.utils import mimetypes - host, file = splithost(url) + host, file = _splithost(url) localname = url2pathname(file) try: stats = os.stat(localname) @@ -2001,7 +2028,7 @@ class URLopener: if file[:1] == '/': urlfile = 'file://' + file return addinfourl(open(localname, 'rb'), headers, urlfile) - host, port = splitport(host) + host, port = _splitport(host) if (not port and socket.gethostbyname(host) in ((localhost(),) + thishost())): urlfile = file @@ -2017,11 +2044,11 @@ class URLopener: if not isinstance(url, str): raise URLError('ftp error: proxy support for ftp protocol currently not implemented') import mimetypes - host, path = splithost(url) + host, path = _splithost(url) if not host: raise URLError('ftp error: no host given') - host, port = splitport(host) - user, host = splituser(host) - if user: user, passwd = splitpasswd(user) + host, port = _splitport(host) + user, host = _splituser(host) + if user: user, passwd = _splitpasswd(user) else: passwd = None host = unquote(host) user = unquote(user or '') @@ -2032,7 +2059,7 @@ class URLopener: port = ftplib.FTP_PORT else: port = int(port) - path, attrs = splitattr(path) + path, attrs = _splitattr(path) path = unquote(path) dirs = path.split('/') dirs, file = dirs[:-1], dirs[-1] @@ -2054,7 +2081,7 @@ class URLopener: if not file: type = 'D' else: type = 'I' for attr in attrs: - attr, value = splitvalue(attr) + attr, value = _splitvalue(attr) if attr.lower() == 'type' and \ value in ('a', 'A', 'i', 'I', 'd', 'D'): type = value.upper() @@ -2237,11 +2264,11 @@ class FancyURLopener(URLopener): return getattr(self,name)(url, realm, data) def retry_proxy_http_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) newurl = 'http://' + host + selector proxy = self.proxies['http'] - urltype, proxyhost = splittype(proxy) - proxyhost, proxyselector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + proxyhost, proxyselector = _splithost(proxyhost) i = proxyhost.find('@') + 1 proxyhost = proxyhost[i:] user, passwd = self.get_user_passwd(proxyhost, realm, i) @@ -2255,11 +2282,11 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_proxy_https_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) newurl = 'https://' + host + selector proxy = self.proxies['https'] - urltype, proxyhost = splittype(proxy) - proxyhost, proxyselector = splithost(proxyhost) + urltype, proxyhost = _splittype(proxy) + proxyhost, proxyselector = _splithost(proxyhost) i = proxyhost.find('@') + 1 proxyhost = proxyhost[i:] user, passwd = self.get_user_passwd(proxyhost, realm, i) @@ -2273,7 +2300,7 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_http_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) i = host.find('@') + 1 host = host[i:] user, passwd = self.get_user_passwd(host, realm, i) @@ -2287,7 +2314,7 @@ class FancyURLopener(URLopener): return self.open(newurl, data) def retry_https_basic_auth(self, url, realm, data=None): - host, selector = splithost(url) + host, selector = _splithost(url) i = host.find('@') + 1 host = host[i:] user, passwd = self.get_user_passwd(host, realm, i) @@ -2504,23 +2531,26 @@ def proxy_bypass_environment(host, proxies=None): try: no_proxy = proxies['no'] except KeyError: - return 0 + return False # '*' is special case for always bypass if no_proxy == '*': - return 1 + return True + host = host.lower() # strip port off host - hostonly, port = splitport(host) + hostonly, port = _splitport(host) # check if the host ends with any of the DNS suffixes - no_proxy_list = [proxy.strip() for proxy in no_proxy.split(',')] - for name in no_proxy_list: + for name in no_proxy.split(','): + name = name.strip() if name: - name = re.escape(name) - pattern = r'(.+\.)?%s$' % name - if (re.match(pattern, hostonly, re.I) - or re.match(pattern, host, re.I)): - return 1 + name = name.lstrip('.') # ignore leading dots + name = name.lower() + if hostonly == name or host == name: + return True + name = '.' + name + if hostonly.endswith(name) or host.endswith(name): + return True # otherwise, don't bypass - return 0 + return False # This code tests an OSX specific data structure but is testable on all @@ -2539,7 +2569,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): """ from fnmatch import fnmatch - hostonly, port = splitport(host) + hostonly, port = _splitport(host) def ip2num(ipAddr): parts = ipAddr.split('.') @@ -2646,7 +2676,7 @@ elif os.name == 'nt': for p in proxyServer.split(';'): protocol, address = p.split('=', 1) # See if address has a type:// prefix - if not re.match('^([^/:]+)://', address): + if not re.match('(?:[^/:]+)://', address): address = '%s://%s' % (protocol, address) proxies[protocol] = address else: @@ -2693,7 +2723,7 @@ elif os.name == 'nt': if not proxyEnable or not proxyOverride: return 0 # try to make a host list from name and IP address. - rawHost, port = splitport(host) + rawHost, port = _splitport(host) host = [rawHost] try: addr = socket.gethostbyname(rawHost) diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9dab4c1c3..c58565e39 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -16,6 +16,9 @@ import urllib.request __all__ = ["RobotFileParser"] +RequestRate = collections.namedtuple("RequestRate", "requests seconds") + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -24,6 +27,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.sitemaps = [] self.default_entry = None self.disallow_all = False self.allow_all = False @@ -136,12 +140,14 @@ class RobotFileParser: # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): - req_rate = collections.namedtuple('req_rate', - 'requests seconds') - entry.req_rate = req_rate - entry.req_rate.requests = int(numbers[0]) - entry.req_rate.seconds = int(numbers[1]) + entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 + elif line[0] == "sitemap": + # According to http://www.sitemaps.org/protocol.html + # "This directive is independent of the user-agent line, + # so it doesn't matter where you place it in your file." + # Therefore we do not change the state of the parser. + self.sitemaps.append(line[1]) if state == 2: self._add_entry(entry) @@ -180,7 +186,9 @@ class RobotFileParser: for entry in self.entries: if entry.applies_to(useragent): return entry.delay - return self.default_entry.delay + if self.default_entry: + return self.default_entry.delay + return None def request_rate(self, useragent): if not self.mtime(): @@ -188,10 +196,20 @@ class RobotFileParser: for entry in self.entries: if entry.applies_to(useragent): return entry.req_rate - return self.default_entry.req_rate + if self.default_entry: + return self.default_entry.req_rate + return None + + def site_maps(self): + if not self.sitemaps: + return None + return self.sitemaps def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n\n'.join(map(str, entries)) class RuleLine: @@ -223,10 +241,14 @@ class Entry: def __str__(self): ret = [] for agent in self.useragents: - ret.extend(["User-agent: ", agent, "\n"]) - for line in self.rulelines: - ret.extend([str(line), "\n"]) - return ''.join(ret) + ret.append(f"User-agent: {agent}") + if self.delay is not None: + ret.append(f"Crawl-delay: {self.delay}") + if self.req_rate is not None: + rate = self.req_rate + ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") + ret.extend(map(str, self.rulelines)) + return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" From 9da9b9263309c111e18e7c42afc85864252e674a Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Tue, 28 Jul 2020 12:50:47 +1200 Subject: [PATCH 135/209] Fix some cgi tests and mark some other as expected failures. --- Cargo.lock | 1 + Lib/test/test_cgi.py | 6 ++++ vm/Cargo.toml | 1 + vm/src/stdlib/io.rs | 67 +++++++++++++++++++++++++++++++++----------- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da5ef537f..0fa13b5f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1412,6 +1412,7 @@ dependencies = [ "log", "maplit", "md-5", + "memchr", "mt19937", "nix", "num-bigint", diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index ab8677199..8ea2484ae 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -128,6 +128,8 @@ class CgiTests(unittest.TestCase): 'file': [b'Testing 123.\n'], 'title': ['']} self.assertEqual(result, expected) + # TODO RUSTPYTHON - see https://github.com/RustPython/RustPython/issues/935 + @unittest.expectedFailure def test_parse_multipart_invalid_encoding(self): BOUNDARY = "JfISa01" POSTDATA = """--JfISa01 @@ -201,6 +203,8 @@ Content-Length: 3 self.addCleanup(cgi.closelog) cgi.log("Testing log 4") + # TODO RUSTPYTHON + @unittest.expectedFailure def test_fieldstorage_readline(self): # FieldStorage uses readline, which has the capacity to read all # contents of the input file into memory; we use readline's size argument @@ -290,6 +294,8 @@ Content-Length: 3 got = getattr(fs.list[x], k) self.assertEqual(got, exp) + # TODO RUSTPYTHON + @unittest.expectedFailure def test_fieldstorage_multipart_maxline(self): # Issue #18167 maxline = 1 << 16 diff --git a/vm/Cargo.toml b/vm/Cargo.toml index f7a9ac5a8..559685033 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -77,6 +77,7 @@ generational-arena = "0.2" parking_lot = "0.11" thread_local = "1.0" cfg-if = "0.1.10" +memchr = "2.1.0" ## unicode stuff unicode_names2 = "0.4" diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3e93f0549..587e89c89 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -5,6 +5,7 @@ use std::fs; use std::io::{self, prelude::*, Cursor, SeekFrom}; use crossbeam_utils::atomic::AtomicCell; +use memchr; use num_traits::ToPrimitive; use crate::byteslike::PyBytesLike; @@ -109,12 +110,35 @@ impl BufferedIO { self.cursor.position() } - fn readline(&mut self) -> Option { - let mut buf = String::new(); + fn readline(&mut self, size: OptionalOption, vm: &VirtualMachine) -> PyResult> { + let mut buf = Vec::new(); + let mut left = size.flat_option().filter(|s| *s >= 0).unwrap_or(i64::MAX); + if left == 0 { + return Ok(buf); + } + loop { + let (done, used) = { + let mut available = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; + if left < available.len() as i64 { + available = &available[..left as usize]; + } - match self.cursor.read_line(&mut buf) { - Ok(_) => Some(buf), - Err(_) => None, + match memchr::memchr(b'\n', available) { + Some(i) => { + buf.extend_from_slice(&available[..=i]); + (true, i + 1) + } + _ => { + buf.extend_from_slice(available); + (false, available.len()) + } + } + }; + self.cursor.consume(used); + left -= used as i64; + if done || used == 0 || left == 0 { + return Ok(buf); + } } } } @@ -195,10 +219,12 @@ impl PyStringIORef { Ok(self.buffer(vm)?.tell()) } - fn readline(self, vm: &VirtualMachine) -> PyResult { - match self.buffer(vm)?.readline() { - Some(line) => Ok(line), - None => Err(vm.new_value_error("Error Performing Operation".to_owned())), + fn readline(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult { + // TODO size should correspond to the number of characters, at the moments its the number of + // bytes. + match String::from_utf8(self.buffer(vm)?.readline(size, vm)?) { + Ok(value) => Ok(value), + Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_owned())), } } @@ -307,11 +333,8 @@ impl PyBytesIORef { Ok(self.buffer(vm)?.tell()) } - fn readline(self, vm: &VirtualMachine) -> PyResult> { - match self.buffer(vm)?.readline() { - Some(line) => Ok(line.as_bytes().to_vec()), - None => Err(vm.new_value_error("Error Performing Operation".to_owned())), - } + fn readline(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult> { + self.buffer(vm)?.readline(size, vm) } fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { @@ -812,6 +835,17 @@ fn buffered_writer_seekable(_self: PyObjectRef) -> bool { true } +fn buffered_writer_seek( + instance: PyObjectRef, + offset: PyObjectRef, + how: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let raw = vm.get_attribute(instance, "raw")?; + let args: Vec<_> = std::iter::once(offset).chain(how.into_option()).collect(); + vm.invoke(&vm.get_attribute(raw, "seek")?, args) +} + #[derive(FromArgs)] struct TextIOWrapperArgs { #[pyarg(positional_or_keyword, optional = false)] @@ -1149,7 +1183,7 @@ pub fn io_open( // There are 3 possible classes here, each inheriting from the RawBaseIO // creating || writing || appending => BufferedWriter let buffered = match mode.chars().next().unwrap() { - 'w' => { + 'w' | 'a' => { let buffered_writer_class = vm .get_attribute(io_module.clone(), "BufferedWriter") .unwrap(); @@ -1162,7 +1196,7 @@ pub fn io_open( vm.invoke(&buffered_reader_class, vec![file_io_obj.clone()]) } //TODO: updating => PyBufferedRandom - _ => unimplemented!("'a' mode is not yet implemented"), + _ => unimplemented!("'+' modes is not yet implemented"), }; match typ.chars().next().unwrap() { @@ -1237,6 +1271,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "__init__" => ctx.new_method(buffered_io_base_init), "write" => ctx.new_method(buffered_writer_write), "seekable" => ctx.new_method(buffered_writer_seekable), + "seek" => ctx.new_method(buffered_writer_seek), "fileno" => ctx.new_method(buffered_io_base_fileno), "tell" => ctx.new_method(buffered_io_base_tell), "close" => ctx.new_method(buffered_io_base_close), From ddb960654476ce26cac59c400561d739394f6c01 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Tue, 28 Jul 2020 14:49:59 +1200 Subject: [PATCH 136/209] Unmark fixed tests in test_urlparse.py, and fix clippy error. --- Lib/test/test_urlparse.py | 52 +-------------------------------------- vm/src/stdlib/io.rs | 1 - 2 files changed, 1 insertion(+), 52 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 95d14fe13..61234b764 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -522,8 +522,6 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(result.url, defrag) self.assertEqual(result.fragment, frag) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') self.assertEqual(p.hostname, "fe80::822a:a8ff:fe49:470c%tESt") @@ -883,8 +881,6 @@ class UrlParseTestCase(unittest.TestCase): errors="ignore") self.assertEqual(result, [('key', '\u0141-')]) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) @@ -949,8 +945,6 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') @@ -1003,7 +997,7 @@ class UrlParseTestCase(unittest.TestCase): self.assertCountEqual(urllib.parse.__all__, expected) # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") + @unittest.expectedFailure def test_urlsplit_normalization(self): # Certain characters should never occur in the netloc, # including under normalization. @@ -1037,8 +1031,6 @@ class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" # In Python 2 this test class was in test_urllib. - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splittype(self): splittype = urllib.parse._splittype self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring')) @@ -1047,8 +1039,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splittype('type:'), ('type', '')) self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splithost(self): splithost = urllib.parse._splithost self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), @@ -1078,8 +1068,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splithost("//example.net/file#"), ('example.net', '/file#')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splituser(self): splituser = urllib.parse._splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), @@ -1093,8 +1081,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'), ('User@example.com:Pass', 'www.python.org:080')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitpasswd(self): # Some of the password examples are not sensible, but it is added to # confirming to RFC2617 and addressing issue4675. @@ -1113,8 +1099,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitpasswd('user'), ('user', None)) self.assertEqual(splitpasswd(':ab'), ('', 'ab')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitport(self): splitport = urllib.parse._splitport self.assertEqual(splitport('parrot:88'), ('parrot', '88')) @@ -1126,8 +1110,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitport('[::1]'), ('[::1]', None)) self.assertEqual(splitport(':88'), ('', '88')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitnport(self): splitnport = urllib.parse._splitnport self.assertEqual(splitnport('parrot:88'), ('parrot', 88)) @@ -1140,8 +1122,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitquery(self): # Normal cases are exercised by other tests; ensure that we also # catch cases with no port specified (testcase ensuring coverage) @@ -1154,8 +1134,6 @@ class Utility_Tests(unittest.TestCase): ('http://python.org/fake', None)) self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splittag(self): splittag = urllib.parse._splittag self.assertEqual(splittag('http://example.com?foo=bar#baz'), @@ -1168,8 +1146,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'), ('http://example.com?foo=bar#baz', 'boo')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitattr(self): splitattr = urllib.parse._splitattr self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'), @@ -1179,8 +1155,6 @@ class Utility_Tests(unittest.TestCase): ('', ['attr1=value1', 'attr2=value2'])) self.assertEqual(splitattr('/path'), ('/path', [])) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_splitvalue(self): # Normal cases are exercised by other tests; test pathological cases # with no key/value pairs. (testcase ensuring coverage) @@ -1191,8 +1165,6 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splitvalue('foobar'), ('foobar', None)) self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz')) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON") def test_to_bytes(self): result = urllib.parse._to_bytes('http://www.python.org') self.assertEqual(result, 'http://www.python.org') @@ -1208,8 +1180,6 @@ class Utility_Tests(unittest.TestCase): class DeprecationTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') @@ -1217,8 +1187,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splittype() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splithost_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splithost('') @@ -1226,8 +1194,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splithost() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splituser_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splituser('') @@ -1235,8 +1201,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splituser() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitpasswd_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitpasswd('') @@ -1244,8 +1208,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitpasswd() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitport('') @@ -1253,8 +1215,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitport() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitnport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitnport('') @@ -1262,8 +1222,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitnport() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitquery_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitquery('') @@ -1271,8 +1229,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitquery() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splittag_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittag('') @@ -1280,8 +1236,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splittag() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitattr_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitattr('') @@ -1289,8 +1243,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitattr() is deprecated as of 3.8, ' 'use urllib.parse.urlparse() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_splitvalue_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitvalue('') @@ -1298,8 +1250,6 @@ class DeprecationTest(unittest.TestCase): 'urllib.parse.splitvalue() is deprecated as of 3.8, ' 'use urllib.parse.parse_qsl() instead') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_to_bytes_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.to_bytes('') diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 587e89c89..3b33f6367 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -5,7 +5,6 @@ use std::fs; use std::io::{self, prelude::*, Cursor, SeekFrom}; use crossbeam_utils::atomic::AtomicCell; -use memchr; use num_traits::ToPrimitive; use crate::byteslike::PyBytesLike; From 30a774e143d6769c2038fb73ae61a2349cbd92b6 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Wed, 29 Jul 2020 19:37:55 +1200 Subject: [PATCH 137/209] Code review changes. --- Cargo.lock | 1 - vm/Cargo.toml | 1 - vm/src/stdlib/io.rs | 16 +++++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0fa13b5f1..da5ef537f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1412,7 +1412,6 @@ dependencies = [ "log", "maplit", "md-5", - "memchr", "mt19937", "nix", "num-bigint", diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 559685033..f7a9ac5a8 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -77,7 +77,6 @@ generational-arena = "0.2" parking_lot = "0.11" thread_local = "1.0" cfg-if = "0.1.10" -memchr = "2.1.0" ## unicode stuff unicode_names2 = "0.4" diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3b33f6367..0133faf90 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,9 +1,11 @@ /* * I/O core tools. */ +use std::convert::TryInto; use std::fs; use std::io::{self, prelude::*, Cursor, SeekFrom}; +use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; @@ -111,18 +113,22 @@ impl BufferedIO { fn readline(&mut self, size: OptionalOption, vm: &VirtualMachine) -> PyResult> { let mut buf = Vec::new(); - let mut left = size.flat_option().filter(|s| *s >= 0).unwrap_or(i64::MAX); + let mut left = size + .flatten() + .filter(|s| *s >= 0) + .and_then(|s| s.try_into().ok()) + .unwrap_or(usize::MAX); if left == 0 { return Ok(buf); } loop { let (done, used) = { let mut available = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; - if left < available.len() as i64 { - available = &available[..left as usize]; + if left < available.len() { + available = &available[..left]; } - match memchr::memchr(b'\n', available) { + match available.find_byte(b'\n') { Some(i) => { buf.extend_from_slice(&available[..=i]); (true, i + 1) @@ -134,7 +140,7 @@ impl BufferedIO { } }; self.cursor.consume(used); - left -= used as i64; + left -= used; if done || used == 0 || left == 0 { return Ok(buf); } From 6032b546668eaa2bbfb989efa2d22619eaf8372b Mon Sep 17 00:00:00 2001 From: Minho Heo Date: Thu, 30 Jul 2020 02:27:38 +0900 Subject: [PATCH 138/209] Fix negative ** fraction to returns complex value Fix #1986 which (-2) ** 0.5 yields NaN. Result: RustPython gives `float.__rpow__(0.5, -2) (0.00000000000000008659560562354934+1.4142135623730951j)` CPython 3.85 gives `>>> float.__rpow__(0.5, -2) (8.659560562354934e-17+1.4142135623730951j)` --- Lib/test/test_fractions.py | 2 -- vm/src/obj/objfloat.rs | 24 ++++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 63f66fc7c..4232d85cc 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -392,8 +392,6 @@ class FractionTest(unittest.TestCase): self.assertTypedEquals(F(-2, 10), round(F(-15, 100), 1)) self.assertTypedEquals(F(-2, 10), round(F(-25, 100), 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testArithmetic(self): self.assertEqual(F(1, 2), F(1, 10) + F(2, 5)) self.assertEqual(F(-3, 10), F(1, 10) - F(2, 5)) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 74c2925b2..894d241f4 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -1,4 +1,5 @@ use num_bigint::{BigInt, ToBigInt}; +use num_complex::Complex64; use num_rational::Ratio; use num_traits::{pow, ToPrimitive, Zero}; @@ -134,12 +135,16 @@ fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { } } -pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { +pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v1.is_zero() { let msg = format!("{} cannot be raised to a negative power", v1); Err(vm.new_zero_division_error(msg)) + } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { + let v1 = Complex64::new(v1, 0.); + let v2 = Complex64::new(v2, 0.); + v1.powc(v2).into_pyobject(vm) } else { - Ok(v1.powf(v2)) + v1.powf(v2).into_pyobject(vm) } } @@ -258,6 +263,17 @@ impl PyFloat { ) } + #[inline] + fn complex_op(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult + where + F: Fn(f64, f64) -> PyResult, + { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| op(self.value, other).into_pyobject(vm), + ) + } + #[inline] fn tuple_op(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult where @@ -331,12 +347,12 @@ impl PyFloat { #[pymethod(name = "__pow__")] fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.simple_op(other, |a, b| float_pow(a, b, vm), vm) + self.complex_op(other, |a, b| float_pow(a, b, vm), vm) } #[pymethod(name = "__rpow__")] fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.simple_op(other, |a, b| float_pow(b, a, vm), vm) + self.complex_op(other, |a, b| float_pow(b, a, vm), vm) } #[pymethod(name = "__sub__")] From 8ac29096a64d66d45ab3b4324652c3f11b3cf272 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Jul 2020 04:19:30 +0900 Subject: [PATCH 139/209] cfg and pystruct_sequence support for pymodule --- derive/src/pymodule.rs | 51 +++++++++++++++++++++++++++++++++++------- derive/src/util.rs | 1 + 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index 7a2669647..a4de5a4a9 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -15,7 +15,7 @@ fn meta_to_vec(meta: Meta) -> Result, Meta> { #[derive(Default)] struct Module { - items: HashSet, + items: HashSet<(ModuleItem, Vec)>, } #[derive(PartialEq, Eq, Hash)] @@ -25,13 +25,13 @@ enum ModuleItem { } impl Module { - fn add_item(&mut self, item: ModuleItem, span: Span) -> Result<(), Diagnostic> { + fn add_item(&mut self, item: (ModuleItem, Vec), span: Span) -> Result<(), Diagnostic> { if self.items.insert(item) { Ok(()) } else { Err(Diagnostic::span_error( span, - "Duplicate #[py*] attribute on pyimpl".to_owned(), + "Duplicate #[py*] attribute on pymodule".to_owned(), )) } } @@ -70,12 +70,35 @@ impl Module { }) } + fn extract_struct_sequence(ident: &Ident, meta: Meta) -> Result { + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pystruct_sequence = \"...\"] cannot be a name/value, you probably meant \ + #[pystruct_sequence(name = \"...\")]", + ) + })?; + + let item_meta = ItemMeta::from_nested_meta( + "pystruct_sequence", + &ident, + &nesteds, + ItemMeta::STRUCT_SEQUENCE_NAMES, + )?; + Ok(ModuleItem::Class { + item_ident: ident.clone(), + py_name: item_meta.simple_name()?, + }) + } + fn extract_item_from_syn( &mut self, attrs: &mut Vec, ident: &Ident, ) -> Result<(), Diagnostic> { let mut attr_idxs = Vec::new(); + let mut items = Vec::new(); + let mut cfgs = Vec::new(); for (i, meta) in attrs .iter() .filter_map(|attr| attr.parse_meta().ok()) @@ -86,17 +109,28 @@ impl Module { Some(name) => name, None => continue, }; - let item = match name.to_string().as_str() { + match name.to_string().as_str() { "pyfunction" => { attr_idxs.push(i); - Self::extract_function(ident, meta)? + items.push((Self::extract_function(ident, meta)?, meta_span)); + } + "pyclass" => { + items.push((Self::extract_class(ident, meta)?, meta_span)); + } + "pystruct_sequence" => { + items.push((Self::extract_struct_sequence(ident, meta)?, meta_span)); + } + "cfg" => { + cfgs.push(meta); + continue; } - "pyclass" => Self::extract_class(ident, meta)?, _ => { continue; } }; - self.add_item(item, meta_span)?; + } + for (item, meta) in items { + self.add_item((item, cfgs.clone()), meta)?; } let mut i = 0; let mut attr_idxs = &*attr_idxs; @@ -130,7 +164,7 @@ fn extract_module_items( ); } - let functions = module.items.into_iter().map(|item| match item { + let functions = module.items.into_iter().map(|(item, cfgs)| match item { ModuleItem::Function { item_ident, py_name, @@ -140,6 +174,7 @@ fn extract_module_items( #module_name.to_owned(), #py_name.to_owned())); quote! { + #( #[ #cfgs ])* vm.__module_set_attr(&module, #py_name, vm.ctx#new_func).unwrap(); } } diff --git a/derive/src/util.rs b/derive/src/util.rs index 70f56abb9..d57d25404 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -88,6 +88,7 @@ pub struct ItemMeta<'a> { impl<'a> ItemMeta<'a> { pub const SIMPLE_NAMES: &'static [&'static str] = &["name"]; + pub const STRUCT_SEQUENCE_NAMES: &'static [&'static str] = &["module", "name"]; pub const ATTRIBUTE_NAMES: &'static [&'static str] = &["name", "magic"]; pub const PROPERTY_NAMES: &'static [&'static str] = &["name", "magic", "setter"]; From 64f4f14e00baa28b10c55e0ad974584bb3619657 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Jul 2020 05:05:46 +0900 Subject: [PATCH 140/209] Reorganize os module --- derive/src/pymodule.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/os.rs | 4280 ++++++++++++++++++++-------------------- 3 files changed, 2150 insertions(+), 2134 deletions(-) diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index a4de5a4a9..b69c5fbcc 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -227,7 +227,7 @@ pub fn impl_pymodule(attr: AttributeArgs, item: Item) -> Result i64 { - use std::os::unix::io::IntoRawFd; - - i64::from(handle.into_raw_fd()) -} - -#[cfg(unix)] -pub fn rust_file(raw_fileno: i64) -> File { - use std::os::unix::io::FromRawFd; - - unsafe { File::from_raw_fd(raw_fileno as i32) } -} - -#[cfg(windows)] -pub fn raw_file_number(handle: File) -> i64 { - use std::os::windows::io::IntoRawHandle; - - handle.into_raw_handle() as i64 -} - -#[cfg(windows)] -pub fn rust_file(raw_fileno: i64) -> File { - use std::os::windows::io::{AsRawHandle, FromRawHandle}; - - let raw_fileno = match raw_fileno { - 0 => io::stdin().as_raw_handle(), - 1 => io::stdout().as_raw_handle(), - 2 => io::stderr().as_raw_handle(), - fno => fno as RawHandle, - }; - - //This seems to work as expected but further testing is required. - unsafe { File::from_raw_handle(raw_fileno) } -} - -#[cfg(target_os = "wasi")] -pub fn raw_file_number(handle: File) -> i64 { - // This should be safe, since the wasi api is pretty well defined, but once - // `wasi_ext` get's stabilized, we should use that instead. - unsafe { std::mem::transmute::<_, u32>(handle).into() } -} - -#[cfg(target_os = "wasi")] -pub fn rust_file(raw_fileno: i64) -> File { - unsafe { std::mem::transmute(raw_fileno as u32) } -} - -#[cfg(not(any(unix, windows, target_os = "wasi")))] -pub fn rust_file(raw_fileno: i64) -> File { - unimplemented!(); -} - -#[cfg(not(any(unix, windows, target_os = "wasi")))] -pub fn raw_file_number(handle: File) -> i64 { - unimplemented!(); -} - #[derive(Debug, Copy, Clone)] enum OutputMode { String, @@ -168,20 +80,6 @@ impl PyPathLike { mode: OutputMode::String, } } - #[cfg(windows)] - pub fn wide(&self) -> Vec { - use std::os::windows::ffi::OsStrExt; - self.path - .as_os_str() - .encode_wide() - .chain(std::iter::once(0)) - .collect() - } - #[cfg(unix)] - pub fn into_bytes(self) -> Vec { - use std::os::unix::ffi::OsStringExt; - self.path.into_os_string().into_vec() - } } impl TryFromObject for PyPathLike { @@ -228,78 +126,6 @@ fn make_path<'a>(_vm: &VirtualMachine, path: &'a PyPathLike, dir_fd: &DirFd) -> } } -fn os_close(fileno: i64) { - //The File type automatically closes when it goes out of scope. - //To enable us to close these file descriptors (and hence prevent leaks) - //we seek to create the relevant File and simply let it pass out of scope! - rust_file(fileno); -} - -#[cfg(unix)] -type OpenFlags = i32; -#[cfg(windows)] -type OpenFlags = u32; -#[cfg(target_os = "wasi")] -type OpenFlags = u16; - -#[cfg(any(unix, windows, target_os = "wasi"))] -pub fn os_open( - name: PyPathLike, - flags: OpenFlags, - _mode: OptionalArg, - dir_fd: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let dir_fd = DirFd { - dir_fd: dir_fd.into_option(), - }; - let fname = make_path(vm, &name, &dir_fd); - - let mut options = OpenOptions::new(); - - macro_rules! bit_contains { - ($c:expr) => { - flags & $c as OpenFlags == $c as OpenFlags - }; - } - - if bit_contains!(libc::O_WRONLY) { - options.write(true); - } else if bit_contains!(libc::O_RDWR) { - options.read(true).write(true); - } else if bit_contains!(libc::O_RDONLY) { - options.read(true); - } - - if bit_contains!(libc::O_APPEND) { - options.append(true); - } - - if bit_contains!(libc::O_CREAT) { - if bit_contains!(libc::O_EXCL) { - options.create_new(true); - } else { - options.create(true); - } - } - - #[cfg(windows)] - let flags = flags & !(libc::O_WRONLY as u32); - - #[cfg(not(target_os = "wasi"))] - options.custom_flags(flags); - let handle = options - .open(fname) - .map_err(|err| convert_io_error(vm, err))?; - - Ok(raw_file_number(handle)) -} - -#[cfg(not(any(unix, windows, target_os = "wasi")))] -pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - unimplemented!() -} - pub fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { #[allow(unreachable_patterns)] // some errors are just aliases of each other let exc_type = match err.kind() { @@ -324,336 +150,14 @@ pub fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionR os_error } -#[cfg(unix)] -pub fn convert_nix_error(vm: &VirtualMachine, err: nix::Error) -> PyBaseExceptionRef { - let nix_error = match err { - nix::Error::InvalidPath => { - let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); - vm.new_exception_msg(exc_type, err.to_string()) - } - nix::Error::InvalidUtf8 => { - let exc_type = vm.ctx.exceptions.unicode_error.clone(); - vm.new_exception_msg(exc_type, err.to_string()) - } - nix::Error::UnsupportedOperation => vm.new_runtime_error(err.to_string()), - nix::Error::Sys(errno) => { - let exc_type = convert_nix_errno(vm, errno); - vm.new_exception_msg(exc_type, err.to_string()) - } - }; - - if let nix::Error::Sys(errno) = err { - vm.set_attr(nix_error.as_object(), "errno", vm.ctx.new_int(errno as i32)) - .unwrap(); - } - - nix_error -} - -#[cfg(unix)] -fn convert_nix_errno(vm: &VirtualMachine, errno: Errno) -> PyClassRef { - match errno { - Errno::EPERM => vm.ctx.exceptions.permission_error.clone(), - _ => vm.ctx.exceptions.os_error.clone(), - } -} - /// Convert the error stored in the `errno` variable into an Exception #[inline] pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { convert_io_error(vm, io::Error::last_os_error()) } -// Flags for os_access -bitflags! { - pub struct AccessFlags: u8{ - const F_OK = 0; - const R_OK = 4; - const W_OK = 2; - const X_OK = 1; - } -} - -#[cfg(unix)] -struct Permissions { - is_readable: bool, - is_writable: bool, - is_executable: bool, -} - -#[cfg(unix)] -fn get_permissions(mode: u32) -> Permissions { - Permissions { - is_readable: mode & 4 != 0, - is_writable: mode & 2 != 0, - is_executable: mode & 1 != 0, - } -} - -#[cfg(unix)] -fn get_right_permission( - mode: u32, - file_owner: Uid, - file_group: Gid, -) -> Result { - let owner_mode = (mode & 0o700) >> 6; - let owner_permissions = get_permissions(owner_mode); - - let group_mode = (mode & 0o070) >> 3; - let group_permissions = get_permissions(group_mode); - - let others_mode = mode & 0o007; - let others_permissions = get_permissions(others_mode); - - let user_id = nix::unistd::getuid(); - let groups_ids = getgroups()?; - - if file_owner == user_id { - Ok(owner_permissions) - } else if groups_ids.contains(&file_group) { - Ok(group_permissions) - } else { - Ok(others_permissions) - } -} - -#[cfg(target_os = "macos")] -fn getgroups() -> nix::Result> { - use libc::{c_int, gid_t}; - use std::ptr; - let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; - let mut groups = Vec::::with_capacity(Errno::result(ret)? as usize); - let ret = unsafe { - libc::getgroups( - groups.capacity() as c_int, - groups.as_mut_ptr() as *mut gid_t, - ) - }; - - Errno::result(ret).map(|s| { - unsafe { groups.set_len(s as usize) }; - groups - }) -} - -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] -use nix::unistd::getgroups; - -#[cfg(target_os = "redox")] -fn getgroups() -> nix::Result> { - unimplemented!("redox getgroups") -} - -#[cfg(unix)] -fn os_access(path: PyPathLike, mode: u8, vm: &VirtualMachine) -> PyResult { - use std::os::unix::fs::MetadataExt; - - let flags = AccessFlags::from_bits(mode).ok_or_else(|| { - vm.new_value_error( - "One of the flags is wrong, there are only 4 possibilities F_OK, R_OK, W_OK and X_OK" - .to_owned(), - ) - })?; - - let metadata = fs::metadata(&path.path); - - // if it's only checking for F_OK - if flags == AccessFlags::F_OK { - return Ok(metadata.is_ok()); - } - - let metadata = metadata.map_err(|err| convert_io_error(vm, err))?; - - let user_id = metadata.uid(); - let group_id = metadata.gid(); - let mode = metadata.mode(); - - let perm = get_right_permission(mode, Uid::from_raw(user_id), Gid::from_raw(group_id)) - .map_err(|err| convert_nix_error(vm, err))?; - - let r_ok = !flags.contains(AccessFlags::R_OK) || perm.is_readable; - let w_ok = !flags.contains(AccessFlags::W_OK) || perm.is_writable; - let x_ok = !flags.contains(AccessFlags::X_OK) || perm.is_executable; - - Ok(r_ok && w_ok && x_ok) -} -#[cfg(windows)] -fn os_access(path: PyPathLike, mode: u8) -> bool { - use winapi::um::{fileapi, winnt}; - let attr = unsafe { fileapi::GetFileAttributesW(path.wide().as_ptr()) }; - attr != fileapi::INVALID_FILE_ATTRIBUTES - && (mode & 2 == 0 - || attr & winnt::FILE_ATTRIBUTE_READONLY == 0 - || attr & winnt::FILE_ATTRIBUTE_DIRECTORY != 0) -} -#[cfg(not(any(unix, windows)))] -fn os_access(path: PyStringRef, mode: u8, vm: &VirtualMachine) -> PyResult { - unimplemented!() -} - -fn os_error(message: OptionalArg, vm: &VirtualMachine) -> PyResult { - let msg = message.map_or("".to_owned(), |msg| msg.as_str().to_owned()); - - Err(vm.new_os_error(msg)) -} - -fn os_fsync(fd: i64, vm: &VirtualMachine) -> PyResult<()> { - let file = rust_file(fd); - file.sync_all().map_err(|err| convert_io_error(vm, err))?; - // Avoid closing the fd - raw_file_number(file); - Ok(()) -} - -fn os_read(fd: i64, n: usize, vm: &VirtualMachine) -> PyResult { - let mut buffer = vec![0u8; n]; - let mut file = rust_file(fd); - let n = file - .read(&mut buffer) - .map_err(|err| convert_io_error(vm, err))?; - buffer.truncate(n); - - // Avoid closing the fd - raw_file_number(file); - Ok(vm.ctx.new_bytes(buffer)) -} - -fn os_write(fd: i64, data: PyBytesLike, vm: &VirtualMachine) -> PyResult { - let mut file = rust_file(fd); - let written = data - .with_ref(|b| file.write(b)) - .map_err(|err| convert_io_error(vm, err))?; - - // Avoid closing the fd - raw_file_number(file); - Ok(vm.ctx.new_int(written)) -} - -fn os_remove(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { - let path = make_path(vm, &path, &dir_fd); - fs::remove_file(path).map_err(|err| convert_io_error(vm, err)) -} - -fn os_mkdir( - path: PyPathLike, - _mode: OptionalArg, - dir_fd: DirFd, - vm: &VirtualMachine, -) -> PyResult<()> { - let path = make_path(vm, &path, &dir_fd); - fs::create_dir(path).map_err(|err| convert_io_error(vm, err)) -} - -fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - fs::create_dir_all(path.as_str()).map_err(|err| convert_io_error(vm, err)) -} - -fn os_rmdir(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { - let path = make_path(vm, &path, &dir_fd); - fs::remove_dir(path).map_err(|err| convert_io_error(vm, err)) -} - -fn os_listdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult { - let res: PyResult> = fs::read_dir(&path.path) - .map_err(|err| convert_io_error(vm, err))? - .map(|entry| match entry { - Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), - Err(s) => Err(convert_io_error(vm, s)), - }) - .collect(); - Ok(vm.ctx.new_list(res?)) -} - -fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { - #[cfg(unix)] - { - let _ = vm; - use std::os::unix::ffi::OsStrExt; - Ok(ffi::OsStr::from_bytes(b)) - } - #[cfg(not(unix))] - { - std::str::from_utf8(b).map(|s| s.as_ref()).map_err(|_| { - vm.new_value_error("Can't convert bytes to str for env function".to_owned()) - }) - } -} - -fn os_putenv( - key: Either, - value: Either, - vm: &VirtualMachine, -) -> PyResult<()> { - let key: &ffi::OsStr = match key { - Either::A(ref s) => s.as_str().as_ref(), - Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, - }; - let value: &ffi::OsStr = match value { - Either::A(ref s) => s.as_str().as_ref(), - Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, - }; - env::set_var(key, value); - Ok(()) -} - -fn os_unsetenv(key: Either, vm: &VirtualMachine) -> PyResult<()> { - let key: &ffi::OsStr = match key { - Either::A(ref s) => s.as_str().as_ref(), - Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, - }; - env::remove_var(key); - Ok(()) -} - -fn _os_environ(vm: &VirtualMachine) -> PyDictRef { - let environ = vm.ctx.new_dict(); - #[cfg(unix)] - { - use std::os::unix::ffi::OsStringExt; - for (key, value) in env::vars_os() { - environ - .set_item( - &vm.ctx.new_bytes(key.into_vec()), - vm.ctx.new_bytes(value.into_vec()), - vm, - ) - .unwrap(); - } - } - #[cfg(windows)] - { - for (key, value) in env::vars() { - environ - .set_item(&vm.new_str(key), vm.new_str(value), vm) - .unwrap(); - } - } - environ -} - -fn os_readlink(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { - let mode = path.mode; - let path = make_path(vm, &path, &dir_fd); - let path = fs::read_link(path).map_err(|err| convert_io_error(vm, err))?; - mode.process_path(path, vm) -} - -#[derive(Debug)] -struct DirEntry { - entry: fs::DirEntry, - mode: OutputMode, -} - -type DirEntryRef = PyRef; - -impl PyValue for DirEntry { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.class(MODULE_NAME, "DirEntry") - } -} - #[derive(FromArgs, Default)] -struct DirFd { +pub struct DirFd { #[pyarg(keyword_only, default = "None")] dir_fd: Option, } @@ -664,889 +168,14 @@ struct FollowSymlinks { follow_symlinks: bool, } -impl DirEntryRef { - fn name(self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(self.entry.file_name(), vm) - } - - fn path(self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(self.entry.path(), vm) - } - - #[allow(clippy::match_bool)] - fn perform_on_metadata( - self, - follow_symlinks: FollowSymlinks, - action: fn(fs::Metadata) -> bool, - vm: &VirtualMachine, - ) -> PyResult { - let metadata = match follow_symlinks.follow_symlinks { - true => fs::metadata(self.entry.path()), - false => fs::symlink_metadata(self.entry.path()), - }; - let meta = metadata.map_err(|err| convert_io_error(vm, err))?; - Ok(action(meta)) - } - - fn is_dir(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - self.perform_on_metadata( - follow_symlinks, - |meta: fs::Metadata| -> bool { meta.is_dir() }, - vm, - ) - } - - fn is_file(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - self.perform_on_metadata( - follow_symlinks, - |meta: fs::Metadata| -> bool { meta.is_file() }, - vm, - ) - } - - fn is_symlink(self, vm: &VirtualMachine) -> PyResult { - Ok(self - .entry - .file_type() - .map_err(|err| convert_io_error(vm, err))? - .is_symlink()) - } - - fn stat(self, dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - os_stat( - Either::A(PyPathLike { - path: self.entry.path(), - mode: OutputMode::String, - }), - dir_fd, - follow_symlinks, - vm, - ) - } -} - -#[pyclass] -#[derive(Debug)] -struct ScandirIterator { - entries: PyRwLock, - exhausted: AtomicCell, - mode: OutputMode, -} - -impl PyValue for ScandirIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.class(MODULE_NAME, "ScandirIter") - } -} - -#[pyimpl] -impl ScandirIterator { - #[pymethod(name = "__next__")] - fn next(&self, vm: &VirtualMachine) -> PyResult { - if self.exhausted.load() { - return Err(objiter::new_stop_iteration(vm)); - } - - match self.entries.write().next() { - Some(entry) => match entry { - Ok(entry) => Ok(DirEntry { - entry, - mode: self.mode, - } - .into_ref(vm) - .into_object()), - Err(s) => Err(convert_io_error(vm, s)), - }, - None => { - self.exhausted.store(true); - Err(objiter::new_stop_iteration(vm)) - } - } - } - - #[pymethod] - fn close(&self) { - self.exhausted.store(true); - } - - #[pymethod(name = "__iter__")] - fn iter(zelf: PyRef) -> PyRef { - zelf - } - - #[pymethod(name = "__enter__")] - fn enter(zelf: PyRef) -> PyRef { - zelf - } - - #[pymethod(name = "__exit__")] - fn exit(zelf: PyRef, _args: PyFuncArgs) { - zelf.close() - } -} - -fn os_scandir(path: OptionalArg, vm: &VirtualMachine) -> PyResult { - let path = match path { - OptionalArg::Present(path) => path, - OptionalArg::Missing => PyPathLike::new_str(".".to_owned()), - }; - - match fs::read_dir(path.path) { - Ok(iter) => Ok(ScandirIterator { - entries: PyRwLock::new(iter), - exhausted: AtomicCell::new(false), - mode: path.mode, - } - .into_ref(vm) - .into_object()), - Err(s) => Err(convert_io_error(vm, s)), - } -} - -#[pystruct_sequence(module = "os", name = "stat_result")] -#[derive(Debug)] -struct StatResult { - st_mode: u32, - st_ino: u64, - st_dev: u64, - st_nlink: u64, - st_uid: u32, - st_gid: u32, - st_size: u64, - st_atime: f64, - st_mtime: f64, - st_ctime: f64, -} - -impl StatResult { - fn into_obj(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_struct_sequence(vm, vm.class(MODULE_NAME, "stat_result")) - .unwrap() - .into_object() - } -} - -// Copied code from Duration::as_secs_f64 as it's still unstable -fn duration_as_secs_f64(duration: Duration) -> f64 { - (duration.as_secs() as f64) + f64::from(duration.subsec_nanos()) / 1_000_000_000_f64 -} - -fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { - match sys_time.duration_since(SystemTime::UNIX_EPOCH) { - Ok(duration) => duration_as_secs_f64(duration), - Err(err) => -duration_as_secs_f64(err.duration()), - } -} - #[cfg(unix)] -fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { - let duration = Duration::new(secs as u64, nanos as u32); - duration_as_secs_f64(duration) -} +use posix::bytes_as_osstr; -#[cfg(unix)] -fn os_stat( - file: Either, - dir_fd: DirFd, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, -) -> PyResult { - #[cfg(target_os = "android")] - use std::os::android::fs::MetadataExt; - #[cfg(target_os = "linux")] - use std::os::linux::fs::MetadataExt; - #[cfg(target_os = "macos")] - use std::os::macos::fs::MetadataExt; - #[cfg(target_os = "openbsd")] - use std::os::openbsd::fs::MetadataExt; - #[cfg(target_os = "redox")] - use std::os::redox::fs::MetadataExt; - - let get_stats = move || -> io::Result { - let meta = match file { - Either::A(path) => { - let path = make_path(vm, &path, &dir_fd); - if follow_symlinks.follow_symlinks { - fs::metadata(path)? - } else { - fs::symlink_metadata(path)? - } - } - Either::B(fno) => { - let file = rust_file(fno); - let meta = file.metadata()?; - raw_file_number(file); - meta - } - }; - - Ok(StatResult { - st_mode: meta.st_mode(), - st_ino: meta.st_ino(), - st_dev: meta.st_dev(), - st_nlink: meta.st_nlink(), - st_uid: meta.st_uid(), - st_gid: meta.st_gid(), - st_size: meta.st_size(), - st_atime: to_seconds_from_unix_epoch(meta.accessed()?), - st_mtime: to_seconds_from_unix_epoch(meta.modified()?), - st_ctime: to_seconds_from_nanos(meta.st_ctime(), meta.st_ctime_nsec()), - } - .into_obj(vm)) - }; - - get_stats().map_err(|err| convert_io_error(vm, err)) -} - -// Copied from CPython fileutils.c -#[cfg(windows)] -fn attributes_to_mode(attr: u32) -> u32 { - const FILE_ATTRIBUTE_DIRECTORY: u32 = 16; - const FILE_ATTRIBUTE_READONLY: u32 = 1; - const S_IFDIR: u32 = 0o040000; - const S_IFREG: u32 = 0o100000; - let mut m: u32 = 0; - if attr & FILE_ATTRIBUTE_DIRECTORY == FILE_ATTRIBUTE_DIRECTORY { - m |= S_IFDIR | 0111; /* IFEXEC for user,group,other */ - } else { - m |= S_IFREG; - } - if attr & FILE_ATTRIBUTE_READONLY == FILE_ATTRIBUTE_READONLY { - m |= 0444; - } else { - m |= 0666; - } - m -} - -#[cfg(windows)] -fn os_stat( - file: Either, - _dir_fd: DirFd, // TODO: error - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, -) -> PyResult { - use std::os::windows::fs::MetadataExt; - - let get_stats = move || -> io::Result { - let meta = match file { - Either::A(path) => match follow_symlinks.follow_symlinks { - true => fs::metadata(path.path)?, - false => fs::symlink_metadata(path.path)?, - }, - Either::B(fno) => { - let f = rust_file(fno); - let meta = f.metadata()?; - raw_file_number(f); - meta - } - }; - - Ok(StatResult { - st_mode: attributes_to_mode(meta.file_attributes()), - st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_uid: 0, // 0 on windows - st_gid: 0, // 0 on windows - st_size: meta.file_size(), - st_atime: to_seconds_from_unix_epoch(meta.accessed()?), - st_mtime: to_seconds_from_unix_epoch(meta.modified()?), - st_ctime: to_seconds_from_unix_epoch(meta.created()?), - } - .into_obj(vm)) - }; - - get_stats().map_err(|e| convert_io_error(vm, e)) -} - -#[cfg(not(any( - target_os = "linux", - target_os = "macos", - target_os = "android", - target_os = "redox", - windows, - unix -)))] -fn os_stat( - file: Either, - _dir_fd: DirFd, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, -) -> PyResult { - unimplemented!(); -} - -fn os_lstat(file: Either, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { - os_stat( - file, - dir_fd, - FollowSymlinks { - follow_symlinks: false, - }, - vm, - ) -} - -#[cfg(unix)] -fn os_symlink( - src: PyPathLike, - dst: PyPathLike, - dir_fd: DirFd, - vm: &VirtualMachine, -) -> PyResult<()> { - use std::os::unix::fs as unix_fs; - let dst = make_path(vm, &dst, &dir_fd); - unix_fs::symlink(src.path, dst).map_err(|err| convert_io_error(vm, err)) -} - -#[cfg(windows)] -fn os_symlink( - src: PyPathLike, - dst: PyPathLike, - _dir_fd: DirFd, - vm: &VirtualMachine, -) -> PyResult<()> { - use std::os::windows::fs as win_fs; - let meta = fs::metadata(src.path.clone()).map_err(|err| convert_io_error(vm, err))?; - let ret = if meta.is_file() { - win_fs::symlink_file(src.path, dst.path) - } else if meta.is_dir() { - win_fs::symlink_dir(src.path, dst.path) - } else { - panic!("Unknown file type"); - }; - ret.map_err(|err| convert_io_error(vm, err)) -} - -#[cfg(all(not(unix), not(windows)))] -fn os_symlink( - src: PyPathLike, - dst: PyPathLike, - dir_fd: DirFd, - vm: &VirtualMachine, -) -> PyResult<()> { - unimplemented!(); -} - -fn os_getcwd(vm: &VirtualMachine) -> PyResult { - Ok(env::current_dir() - .map_err(|err| convert_io_error(vm, err))? - .as_path() - .to_str() - .unwrap() - .to_owned()) -} - -fn os_chdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - env::set_current_dir(&path.path).map_err(|err| convert_io_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_chroot(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - nix::unistd::chroot(&*path.path).map_err(|err| convert_nix_error(vm, err)) -} - -// As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) -#[cfg(all(unix, not(target_os = "redox")))] -fn os_chown( - path: Either, - uid: PyIntRef, - gid: PyIntRef, - dir_fd: DirFd, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, -) -> PyResult<()> { - let uid = isize::try_from_object(&vm, uid.as_object().clone())?; - let gid = isize::try_from_object(&vm, gid.as_object().clone())?; - - let uid = if uid >= 0 { - Some(nix::unistd::Uid::from_raw(uid as u32)) - } else if uid == -1 { - None - } else { - return Err(vm.new_os_error(String::from("Specified uid is not valid."))); - }; - - let gid = if gid >= 0 { - Some(nix::unistd::Gid::from_raw(gid as u32)) - } else if gid == -1 { - None - } else { - return Err(vm.new_os_error(String::from("Specified gid is not valid."))); - }; - - let flag = if follow_symlinks.follow_symlinks { - nix::unistd::FchownatFlags::FollowSymlink - } else { - nix::unistd::FchownatFlags::NoFollowSymlink - }; - - let dir_fd: Option = match dir_fd.dir_fd { - Some(int_ref) => Some(i32::try_from_object(&vm, int_ref.as_object().clone())?), - None => None, - }; - - match path { - Either::A(p) => nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) - .map_err(|err| convert_nix_error(vm, err)), - Either::B(fd) => { - let path = fs::read_link(format!("/proc/self/fd/{}", fd)) - .map_err(|_| vm.new_os_error(String::from("Cannot find path for specified fd")))?; - nix::unistd::fchownat(dir_fd, &path, uid, gid, flag) - .map_err(|err| convert_nix_error(vm, err)) - } - } -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_lchown(path: PyPathLike, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - os_chown( - Either::A(path), - uid, - gid, - DirFd { dir_fd: None }, - FollowSymlinks { - follow_symlinks: false, - }, - vm, - ) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_fchown(fd: i64, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - os_chown( - Either::B(fd), - uid, - gid, - DirFd { dir_fd: None }, - FollowSymlinks { - follow_symlinks: true, - }, - vm, - ) -} - -#[cfg(unix)] -fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { - use nix::fcntl::fcntl; - use nix::fcntl::FcntlArg; - let flags = fcntl(fd, FcntlArg::F_GETFD); - match flags { - Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), - Err(err) => Err(convert_nix_error(vm, err)), - } -} - -#[cfg(unix)] -pub(crate) fn set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { - use nix::fcntl; - - let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); - let mut new_flags = flags; - new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable); - if flags != new_flags { - fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?; - } - Ok(()) -} - -fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - #[cfg(not(any(unix, windows)))] - { - unimplemented!() - } - #[cfg(unix)] - { - set_inheritable(fd as RawFd, inheritable).map_err(|err| convert_nix_error(vm, err)) - } - #[cfg(windows)] - { - use winapi::um::{handleapi, winbase}; - let fd = fd as RawHandle; - let flags = if inheritable { - winbase::HANDLE_FLAG_INHERIT - } else { - 0 - }; - let ret = - unsafe { handleapi::SetHandleInformation(fd, winbase::HANDLE_FLAG_INHERIT, flags) }; - if ret == 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } -} - -#[cfg(unix)] -fn os_get_blocking(fd: RawFd, vm: &VirtualMachine) -> PyResult { - use nix::fcntl::fcntl; - use nix::fcntl::FcntlArg; - let flags = fcntl(fd, FcntlArg::F_GETFL); - match flags { - Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), - Err(err) => Err(convert_nix_error(vm, err)), - } -} - -#[cfg(unix)] -fn os_set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { - let _set_flag = || { - use nix::fcntl::fcntl; - use nix::fcntl::FcntlArg; - use nix::fcntl::OFlag; - - let flags = OFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFL)?); - let mut new_flags = flags; - new_flags.set(OFlag::from_bits_truncate(libc::O_NONBLOCK), !blocking); - if flags != new_flags { - fcntl(fd, FcntlArg::F_SETFL(new_flags))?; - } - Ok(()) - }; - _set_flag().map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(unix)] -fn os_pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { - use nix::unistd::close; - use nix::unistd::pipe; - let (rfd, wfd) = pipe().map_err(|err| convert_nix_error(vm, err))?; - os_set_inheritable(rfd.into(), false, vm) - .and_then(|_| os_set_inheritable(wfd.into(), false, vm)) - .map_err(|err| { - let _ = close(rfd); - let _ = close(wfd); - err - })?; - Ok((rfd, wfd)) -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "emscripten", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd" -))] -fn os_pipe2(flags: libc::c_int, vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { - use nix::fcntl::OFlag; - use nix::unistd::pipe2; - let oflags = OFlag::from_bits_truncate(flags); - pipe2(oflags).map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(unix)] -fn os_system(command: PyStringRef) -> PyResult { - use libc::system; - use std::ffi::CString; - - let rstr = command.as_str(); - let cstr = CString::new(rstr).unwrap(); - let x = unsafe { system(cstr.as_ptr()) }; - Ok(x) -} - -#[cfg(unix)] -fn os_chmod( - path: PyPathLike, - dir_fd: DirFd, - mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, -) -> PyResult<()> { - use std::os::unix::fs::PermissionsExt; - let path = make_path(vm, &path, &dir_fd); - let metadata = if follow_symlinks.follow_symlinks { - fs::metadata(path) - } else { - fs::symlink_metadata(path) - }; - let meta = metadata.map_err(|err| convert_io_error(vm, err))?; - let mut permissions = meta.permissions(); - permissions.set_mode(mode); - fs::set_permissions(path, permissions).map_err(|err| convert_io_error(vm, err))?; - Ok(()) -} - -#[cfg(unix)] -fn os_execv( - path: PyStringRef, - argv_list: Either, - vm: &VirtualMachine, -) -> PyResult<()> { - let path = ffi::CString::new(path.as_str()) - .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; - - let argv: Vec = match argv_list { - Either::A(list) => vm.extract_elements(list.as_object())?, - Either::B(tuple) => vm.extract_elements(tuple.as_object())?, - }; - let argv: Vec<&ffi::CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); - - if argv.is_empty() { - return Err(vm.new_value_error("execv() arg 2 must not be empty".to_owned())); - } - if argv.first().unwrap().to_bytes().is_empty() { - return Err(vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned())); - } - - unistd::execv(&path, &argv) - .map(|_ok| ()) - .map_err(|err| convert_nix_error(vm, err)) -} - -fn os_fspath(path: PyPathLike, vm: &VirtualMachine) -> PyResult { - path.mode.process_path(path.path, vm) -} - -fn os_rename(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - fs::rename(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) -} - -fn os_getpid(vm: &VirtualMachine) -> PyObjectRef { - let pid = std::process::id(); - vm.new_int(pid) -} - -fn os_cpu_count(vm: &VirtualMachine) -> PyObjectRef { - let cpu_count = num_cpus::get(); - vm.new_int(cpu_count) -} - -fn os_exit(code: i32) { - std::process::exit(code) -} - -#[cfg(unix)] -fn os_getppid(vm: &VirtualMachine) -> PyObjectRef { - let ppid = unistd::getppid().as_raw(); - vm.new_int(ppid) -} - -#[cfg(unix)] -fn os_getgid(vm: &VirtualMachine) -> PyObjectRef { - let gid = unistd::getgid().as_raw(); - vm.new_int(gid) -} - -#[cfg(unix)] -fn os_getegid(vm: &VirtualMachine) -> PyObjectRef { - let egid = unistd::getegid().as_raw(); - vm.new_int(egid) -} - -#[cfg(unix)] -fn os_getpgid(pid: u32, vm: &VirtualMachine) -> PyResult { - match unistd::getpgid(Some(Pid::from_raw(pid as i32))) { - Ok(pgid) => Ok(vm.new_int(pgid.as_raw())), - Err(err) => Err(convert_nix_error(vm, err)), - } -} - -#[cfg(unix)] -fn os_getpgrp(vm: &VirtualMachine) -> PyResult { - Ok(vm.new_int(unistd::getpgrp().as_raw())) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_getsid(pid: u32, vm: &VirtualMachine) -> PyResult { - match unistd::getsid(Some(Pid::from_raw(pid as i32))) { - Ok(sid) => Ok(vm.new_int(sid.as_raw())), - Err(err) => Err(convert_nix_error(vm, err)), - } -} - -#[cfg(unix)] -fn os_getuid(vm: &VirtualMachine) -> PyObjectRef { - let uid = unistd::getuid().as_raw(); - vm.new_int(uid) -} - -#[cfg(unix)] -fn os_geteuid(vm: &VirtualMachine) -> PyObjectRef { - let euid = unistd::geteuid().as_raw(); - vm.new_int(euid) -} - -#[cfg(unix)] -fn os_setgid(gid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setgid(Gid::from_raw(gid)).map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_setegid(egid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setegid(Gid::from_raw(egid)).map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(unix)] -fn os_setpgid(pid: u32, pgid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setpgid(Pid::from_raw(pid as i32), Pid::from_raw(pgid as i32)) - .map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_setsid(vm: &VirtualMachine) -> PyResult<()> { - unistd::setsid() - .map(|_ok| ()) - .map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(unix)] -fn os_setuid(uid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setuid(Uid::from_raw(uid)).map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_seteuid(euid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -fn os_setreuid(ruid: u32, euid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setuid(Uid::from_raw(ruid)).map_err(|err| convert_nix_error(vm, err))?; - unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_setresuid(ruid: u32, euid: u32, suid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setresuid( - Uid::from_raw(ruid), - Uid::from_raw(euid), - Uid::from_raw(suid), - ) - .map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(all(unix, not(target_os = "redox")))] -pub fn os_openpty(vm: &VirtualMachine) -> PyResult { - match openpty(None, None) { - Ok(r) => Ok(vm - .ctx - .new_tuple(vec![vm.new_int(r.master), vm.new_int(r.slave)])), - Err(err) => Err(convert_nix_error(vm, err)), - } -} - -#[cfg(unix)] -pub fn os_ttyname(fd: i32, vm: &VirtualMachine) -> PyResult { - use libc::ttyname; - let name = unsafe { ttyname(fd) }; - if name.is_null() { - Err(errno_err(vm)) - } else { - let name = unsafe { ffi::CStr::from_ptr(name) }.to_str().unwrap(); - Ok(vm.ctx.new_str(name.to_owned())) - } -} - -fn os_urandom(size: usize, vm: &VirtualMachine) -> PyResult> { - let mut buf = vec![0u8; size]; - match getrandom::getrandom(&mut buf) { - Ok(()) => Ok(buf), - Err(e) => match e.raw_os_error() { - Some(errno) => Err(convert_io_error(vm, io::Error::from_raw_os_error(errno))), - None => Err(vm.new_os_error("Getting random failed".to_owned())), - }, - } -} - -#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] -type ModeT = u32; - -#[cfg(target_os = "redox")] -type ModeT = i32; - -#[cfg(target_os = "macos")] -type ModeT = u16; - -#[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "openbsd", - target_os = "redox", - target_os = "android", -))] -fn os_umask(mask: ModeT, _vm: &VirtualMachine) -> PyResult { - let ret_mask = unsafe { libc::umask(mask) }; - Ok(ret_mask) -} - -#[pystruct_sequence(module = "os", name = "uname_result")] -#[derive(Debug)] -#[cfg(unix)] -struct UnameResult { - sysname: String, - nodename: String, - release: String, - version: String, - machine: String, -} - -#[cfg(unix)] -impl UnameResult { - fn into_obj(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_struct_sequence(vm, vm.class(MODULE_NAME, "uname_result")) - .unwrap() - .into_object() - } -} - -#[cfg(unix)] -fn os_uname(vm: &VirtualMachine) -> PyResult { - let info = uname::uname().map_err(|err| convert_io_error(vm, err))?; - Ok(UnameResult { - sysname: info.sysname, - nodename: info.nodename, - release: info.release, - version: info.version, - machine: info.machine, - } - .into_obj(vm)) -} - -// this is basically what CPython has for Py_off_t; windows uses long long -// for offsets, other platforms just use off_t -#[cfg(not(windows))] -pub type Offset = libc::off_t; -#[cfg(windows)] -pub type Offset = libc::c_longlong; - -#[cfg(all(windows, target_env = "msvc"))] -type InvalidParamHandler = extern "C" fn( - *const libc::wchar_t, - *const libc::wchar_t, - *const libc::wchar_t, - libc::c_uint, - libc::uintptr_t, -); -#[cfg(all(windows, target_env = "msvc"))] -extern "C" { - #[doc(hidden)] - pub fn _set_thread_local_invalid_parameter_handler( - pNew: InvalidParamHandler, - ) -> InvalidParamHandler; -} - -#[cfg(all(windows, target_env = "msvc"))] -#[doc(hidden)] -pub extern "C" fn silent_iph_handler( - _: *const libc::wchar_t, - _: *const libc::wchar_t, - _: *const libc::wchar_t, - _: libc::c_uint, - _: libc::uintptr_t, -) { +#[cfg(not(unix))] +fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { + std::str::from_utf8(b) + .map(|s| s.as_ref()) + .map_err(|_| vm.new_value_error("Can't convert bytes to str for env function".to_owned())) } #[macro_export] @@ -1568,672 +197,670 @@ macro_rules! suppress_iph { }}; } -fn os_isatty(fd: i32) -> bool { - unsafe { suppress_iph!(libc::isatty(fd)) != 0 } -} +#[pymodule] +mod _os { + use super::platform::OpenFlags; + use super::*; -fn os_lseek(fd: i32, position: Offset, how: i32, vm: &VirtualMachine) -> PyResult { - #[cfg(not(windows))] - let res = unsafe { suppress_iph!(libc::lseek(fd, position, how)) }; - #[cfg(windows)] - let res = unsafe { - use winapi::um::{fileapi, winnt}; - let mut li = winnt::LARGE_INTEGER::default(); - *li.QuadPart_mut() = position; - let ret = fileapi::SetFilePointer( - fd as RawHandle, - li.u().LowPart as _, - &mut li.u_mut().HighPart, - how as _, - ); - if ret == fileapi::INVALID_SET_FILE_POINTER { - -1 - } else { - li.u_mut().LowPart = ret; - *li.QuadPart() - } - }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(res) + #[pyfunction] + fn close(fileno: i64) { + //The File type automatically closes when it goes out of scope. + //To enable us to close these file descriptors (and hence prevent leaks) + //we seek to create the relevant File and simply let it pass out of scope! + rust_file(fileno); } -} -fn os_link(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - fs::hard_link(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) -} - -#[derive(FromArgs)] -struct UtimeArgs { - #[pyarg(positional_or_keyword)] - path: PyPathLike, - #[pyarg(positional_or_keyword, default = "None")] - times: Option, - #[pyarg(keyword_only, default = "None")] - ns: Option, - #[pyarg(flatten)] - _dir_fd: DirFd, - #[pyarg(flatten)] - _follow_symlinks: FollowSymlinks, -} - -#[cfg(not(target_os = "wasi"))] -fn os_utime(args: UtimeArgs, vm: &VirtualMachine) -> PyResult<()> { - let parse_tup = |tup: PyTupleRef| -> Option<(i64, i64)> { - let tup = tup.as_slice(); - if tup.len() != 2 { - return None; - } - let i = |e: &PyObjectRef| e.clone().downcast::().ok()?.as_bigint().to_i64(); - Some((i(&tup[0])?, i(&tup[1])?)) - }; - let (acc, modif) = match (args.times, args.ns) { - (Some(t), None) => parse_tup(t).ok_or_else(|| { - vm.new_type_error( - "utime: 'times' must be either a tuple of two ints or None".to_owned(), - ) - })?, - (None, Some(ns)) => { - let (a, m) = parse_tup(ns).ok_or_else(|| { - vm.new_type_error("utime: 'ns' must be a tuple of two ints".to_owned()) - })?; - // TODO: do validation to make sure this doesn't.. underflow? - (a / 1_000_000_000, m / 1_000_000_000) - } - (None, None) => { - let now = SystemTime::now(); - let now = now - .duration_since(SystemTime::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or_else(|e| -(e.duration().as_secs() as i64)); - (now, now) - } - (Some(_), Some(_)) => { - return Err(vm.new_value_error( - "utime: you may specify either 'times' or 'ns' but not both".to_owned(), - )) - } - }; - utime::set_file_times(&args.path.path, acc, modif).map_err(|e| convert_io_error(vm, e)) -} - -#[cfg(unix)] -fn os_sync(_vm: &VirtualMachine) -> PyResult<()> { - #[cfg(not(any(target_os = "redox", target_os = "android")))] - unsafe { - libc::sync(); - } - Ok(()) -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_getresuid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let mut ruid = 0; - let mut euid = 0; - let mut suid = 0; - let ret = unsafe { libc::getresuid(&mut ruid, &mut euid, &mut suid) }; - if ret == 0 { - Ok((ruid, euid, suid)) - } else { - Err(errno_err(vm)) - } -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_getresgid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let mut rgid = 0; - let mut egid = 0; - let mut sgid = 0; - let ret = unsafe { libc::getresgid(&mut rgid, &mut egid, &mut sgid) }; - if ret == 0 { - Ok((rgid, egid, sgid)) - } else { - Err(errno_err(vm)) - } -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_setresgid(rgid: u32, egid: u32, sgid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setresgid( - Gid::from_raw(rgid), - Gid::from_raw(egid), - Gid::from_raw(sgid), - ) - .map_err(|err| convert_nix_error(vm, err)) -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_setregid(rgid: u32, egid: u32, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { libc::setregid(rgid, egid) }; - if ret == 0 { - Ok(()) - } else { - Err(errno_err(vm)) - } -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_initgroups(user_name: PyStringRef, gid: u32, vm: &VirtualMachine) -> PyResult<()> { - let user = ffi::CString::new(user_name.as_str()).unwrap(); - let gid = Gid::from_raw(gid); - unistd::initgroups(&user, gid).map_err(|err| convert_nix_error(vm, err)) -} - -// cfg from nix -#[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" -))] -fn os_setgroups(group_ids: PyIterable, vm: &VirtualMachine) -> PyResult<()> { - let gids = group_ids - .iter(vm)? - .map(|entry| match entry { - Ok(id) => Ok(unistd::Gid::from_raw(id)), - Err(err) => Err(err), - }) - .collect::, _>>()?; - let ret = unistd::setgroups(&gids); - ret.map_err(|err| convert_nix_error(vm, err)) -} - -#[cfg(unix)] -fn envp_from_dict(dict: PyDictRef, vm: &VirtualMachine) -> PyResult> { - dict.into_iter() - .map(|(k, v)| { - let k = PyPathLike::try_from_object(vm, k)?.into_bytes(); - let v = PyPathLike::try_from_object(vm, v)?.into_bytes(); - if k.contains(&0) { - return Err( - vm.new_value_error("envp dict key cannot contain a nul byte".to_owned()) - ); - } - if k.contains(&b'=') { - return Err( - vm.new_value_error("envp dict key cannot contain a '=' character".to_owned()) - ); - } - if v.contains(&0) { - return Err( - vm.new_value_error("envp dict value cannot contain a nul byte".to_owned()) - ); - } - let mut env = k; - env.push(b'='); - env.extend(v); - Ok(unsafe { ffi::CString::from_vec_unchecked(env) }) - }) - .collect() -} - -#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] -#[derive(FromArgs)] -struct PosixSpawnArgs { - #[pyarg(positional_only)] - path: PyPathLike, - #[pyarg(positional_only)] - args: PyIterable, - #[pyarg(positional_only)] - env: PyMapping, - #[pyarg(keyword_only, default = "None")] - file_actions: Option>, - #[pyarg(keyword_only, default = "None")] - setsigdef: Option>, -} - -#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] -#[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] -#[repr(i32)] -enum PosixSpawnFileActionIdentifier { - Open, - Close, - Dup2, -} - -#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] -impl PosixSpawnArgs { - fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult { - let path = ffi::CString::new(self.path.into_bytes()) - .map_err(|_| vm.new_value_error("path should not have nul bytes".to_owned()))?; - - let mut file_actions = unsafe { - let mut fa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawn_file_actions_init(fa.as_mut_ptr()) == 0); - fa.assume_init() + #[pyfunction] + #[cfg(any(unix, windows, target_os = "wasi"))] + pub(crate) fn open( + name: PyPathLike, + flags: OpenFlags, + _mode: OptionalArg, + dir_fd: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let dir_fd = DirFd { + dir_fd: dir_fd.into_option(), }; - if let Some(it) = self.file_actions { - for action in it.iter(vm)? { - let action = action?; - let (id, args) = action.as_slice().split_first().ok_or_else(|| { - vm.new_type_error( - "Each file_actions element must be a non-empty tuple".to_owned(), - ) - })?; - let id = i32::try_from_object(vm, id.clone())?; - let id = PosixSpawnFileActionIdentifier::try_from(id) - .map_err(|_| vm.new_type_error("Unknown file_actions identifier".to_owned()))?; - let args = PyFuncArgs::from(args.to_vec()); - let ret = match id { - PosixSpawnFileActionIdentifier::Open => { - let (fd, path, oflag, mode): (_, PyPathLike, _, _) = args.bind(vm)?; - let path = ffi::CString::new(path.into_bytes()).map_err(|_| { - vm.new_value_error( - "POSIX_SPAWN_OPEN path should not have nul bytes".to_owned(), - ) - })?; - unsafe { - libc::posix_spawn_file_actions_addopen( - &mut file_actions, - fd, - path.as_ptr(), - oflag, - mode, - ) - } + let fname = make_path(vm, &name, &dir_fd); + + let mut options = OpenOptions::new(); + + macro_rules! bit_contains { + ($c:expr) => { + flags & $c as OpenFlags == $c as OpenFlags + }; + } + + if bit_contains!(libc::O_WRONLY) { + options.write(true); + } else if bit_contains!(libc::O_RDWR) { + options.read(true).write(true); + } else if bit_contains!(libc::O_RDONLY) { + options.read(true); + } + + if bit_contains!(libc::O_APPEND) { + options.append(true); + } + + if bit_contains!(libc::O_CREAT) { + if bit_contains!(libc::O_EXCL) { + options.create_new(true); + } else { + options.create(true); + } + } + + #[cfg(windows)] + let flags = flags & !(libc::O_WRONLY as u32); + + #[cfg(not(target_os = "wasi"))] + { + use platform::OpenOptionsExt; + options.custom_flags(flags); + } + let handle = options + .open(fname) + .map_err(|err| convert_io_error(vm, err))?; + + Ok(raw_file_number(handle)) + } + + #[pyfunction] + #[cfg(not(any(unix, windows, target_os = "wasi")))] + pub(crate) fn open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + unimplemented!() + } + + #[pyfunction] + fn error(message: OptionalArg, vm: &VirtualMachine) -> PyResult { + let msg = message.map_or("".to_owned(), |msg| msg.as_str().to_owned()); + + Err(vm.new_os_error(msg)) + } + + #[pyfunction] + fn fsync(fd: i64, vm: &VirtualMachine) -> PyResult<()> { + let file = rust_file(fd); + file.sync_all().map_err(|err| convert_io_error(vm, err))?; + // Avoid closing the fd + raw_file_number(file); + Ok(()) + } + + #[pyfunction] + fn read(fd: i64, n: usize, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; n]; + let mut file = rust_file(fd); + let n = file + .read(&mut buffer) + .map_err(|err| convert_io_error(vm, err))?; + buffer.truncate(n); + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_bytes(buffer)) + } + + #[pyfunction] + fn write(fd: i64, data: PyBytesLike, vm: &VirtualMachine) -> PyResult { + let mut file = rust_file(fd); + let written = data + .with_ref(|b| file.write(b)) + .map_err(|err| convert_io_error(vm, err))?; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_int(written)) + } + + #[pyfunction] + fn remove(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, &path, &dir_fd); + fs::remove_file(path).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn mkdir( + path: PyPathLike, + _mode: OptionalArg, + dir_fd: DirFd, + vm: &VirtualMachine, + ) -> PyResult<()> { + let path = make_path(vm, &path, &dir_fd); + fs::create_dir(path).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir_all(path.as_str()).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn rmdir(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, &path, &dir_fd); + fs::remove_dir(path).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn listdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult { + let res: PyResult> = fs::read_dir(&path.path) + .map_err(|err| convert_io_error(vm, err))? + .map(|entry| match entry { + Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), + Err(s) => Err(convert_io_error(vm, s)), + }) + .collect(); + Ok(vm.ctx.new_list(res?)) + } + + #[pyfunction] + fn putenv( + key: Either, + value: Either, + vm: &VirtualMachine, + ) -> PyResult<()> { + let key: &ffi::OsStr = match key { + Either::A(ref s) => s.as_str().as_ref(), + Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, + }; + let value: &ffi::OsStr = match value { + Either::A(ref s) => s.as_str().as_ref(), + Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, + }; + env::set_var(key, value); + Ok(()) + } + + #[pyfunction] + fn unsetenv(key: Either, vm: &VirtualMachine) -> PyResult<()> { + let key: &ffi::OsStr = match key { + Either::A(ref s) => s.as_str().as_ref(), + Either::B(ref b) => bytes_as_osstr(b.get_value(), vm)?, + }; + env::remove_var(key); + Ok(()) + } + + #[pyfunction] + fn readlink(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { + let mode = path.mode; + let path = make_path(vm, &path, &dir_fd); + let path = fs::read_link(path).map_err(|err| convert_io_error(vm, err))?; + mode.process_path(path, vm) + } + + #[pyclass] + #[derive(Debug)] + struct DirEntry { + entry: fs::DirEntry, + mode: OutputMode, + } + + impl PyValue for DirEntry { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class(super::MODULE_NAME, "DirEntry") + } + } + + #[pyimpl] + impl DirEntry { + #[pyproperty] + fn name(&self, vm: &VirtualMachine) -> PyResult { + self.mode.process_path(self.entry.file_name(), vm) + } + + #[pyproperty] + fn path(&self, vm: &VirtualMachine) -> PyResult { + self.mode.process_path(self.entry.path(), vm) + } + + #[allow(clippy::match_bool)] + fn perform_on_metadata( + &self, + follow_symlinks: FollowSymlinks, + action: fn(fs::Metadata) -> bool, + vm: &VirtualMachine, + ) -> PyResult { + let metadata = match follow_symlinks.follow_symlinks { + true => fs::metadata(self.entry.path()), + false => fs::symlink_metadata(self.entry.path()), + }; + let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + Ok(action(meta)) + } + + #[pymethod] + fn is_dir(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + |meta: fs::Metadata| -> bool { meta.is_dir() }, + vm, + ) + } + + #[pymethod] + fn is_file(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + |meta: fs::Metadata| -> bool { meta.is_file() }, + vm, + ) + } + + #[pymethod] + fn is_symlink(&self, vm: &VirtualMachine) -> PyResult { + Ok(self + .entry + .file_type() + .map_err(|err| convert_io_error(vm, err))? + .is_symlink()) + } + + #[pymethod] + fn stat( + &self, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + super::platform::stat( + Either::A(PyPathLike { + path: self.entry.path(), + mode: OutputMode::String, + }), + dir_fd, + follow_symlinks, + vm, + ) + } + } + + #[pyclass(name = "ScandirIter")] + #[derive(Debug)] + struct ScandirIterator { + entries: PyRwLock, + exhausted: AtomicCell, + mode: OutputMode, + } + + impl PyValue for ScandirIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class(super::MODULE_NAME, "ScandirIter") + } + } + + #[pyimpl] + impl ScandirIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.exhausted.load() { + return Err(objiter::new_stop_iteration(vm)); + } + + match self.entries.write().next() { + Some(entry) => match entry { + Ok(entry) => Ok(DirEntry { + entry, + mode: self.mode, } - PosixSpawnFileActionIdentifier::Close => { - let (fd,) = args.bind(vm)?; - unsafe { libc::posix_spawn_file_actions_addclose(&mut file_actions, fd) } - } - PosixSpawnFileActionIdentifier::Dup2 => { - let (fd, newfd) = args.bind(vm)?; - unsafe { - libc::posix_spawn_file_actions_adddup2(&mut file_actions, fd, newfd) - } - } - }; - if ret != 0 { - return Err(errno_err(vm)); + .into_ref(vm) + .into_object()), + Err(s) => Err(convert_io_error(vm, s)), + }, + None => { + self.exhausted.store(true); + Err(objiter::new_stop_iteration(vm)) } } } - let mut attrp = unsafe { - let mut sa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawnattr_init(sa.as_mut_ptr()) == 0); - sa.assume_init() - }; - if let Some(sigs) = self.setsigdef { - use nix::sys::signal; - let mut set = signal::SigSet::empty(); - for sig in sigs.iter(vm)? { - let sig = sig?; - let sig = signal::Signal::try_from(sig).map_err(|_| { - vm.new_value_error(format!("signal number {} out of range", sig)) - })?; - set.add(sig); - } - assert!(unsafe { libc::posix_spawnattr_setsigdefault(&mut attrp, set.as_ref()) } == 0); + #[pymethod] + fn close(&self) { + self.exhausted.store(true); } - let mut args: Vec = self - .args - .iter(vm)? - .map(|res| { - ffi::CString::new(res?.into_bytes()) - .map_err(|_| vm.new_value_error("path should not have nul bytes".to_owned())) - }) - .collect::>()?; - let argv: Vec<*mut libc::c_char> = args - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); - let mut env = envp_from_dict(self.env.into_dict(), vm)?; - let envp: Vec<*mut libc::c_char> = env - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef) -> PyRef { + zelf + } - let mut pid = 0; - let ret = unsafe { - if spawnp { - libc::posix_spawnp( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } else { - libc::posix_spawn( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } - }; + #[pymethod(name = "__enter__")] + fn enter(zelf: PyRef) -> PyRef { + zelf + } - if ret == 0 { - Ok(pid) - } else { - Err(errno_err(vm)) + #[pymethod(name = "__exit__")] + fn exit(zelf: PyRef, _args: PyFuncArgs) { + zelf.close() } } -} -#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] -fn os_posix_spawn(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult { - args.spawn(false, vm) -} -#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] -fn os_posix_spawnp(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult { - args.spawn(true, vm) -} + #[pyfunction] + fn scandir(path: OptionalArg, vm: &VirtualMachine) -> PyResult { + let path = match path { + OptionalArg::Present(path) => path, + OptionalArg::Missing => PyPathLike::new_str(".".to_owned()), + }; -#[cfg(unix)] -fn os_wifsignaled(status: i32) -> bool { - unsafe { libc::WIFSIGNALED(status) } -} -#[cfg(unix)] -fn os_wifstopped(status: i32) -> bool { - unsafe { libc::WIFSTOPPED(status) } -} -#[cfg(unix)] -fn os_wifexited(status: i32) -> bool { - unsafe { libc::WIFEXITED(status) } -} -#[cfg(unix)] -fn os_wtermsig(status: i32) -> i32 { - unsafe { libc::WTERMSIG(status) } -} -#[cfg(unix)] -fn os_wstopsig(status: i32) -> i32 { - unsafe { libc::WSTOPSIG(status) } -} -#[cfg(unix)] -fn os_wexitstatus(status: i32) -> i32 { - unsafe { libc::WEXITSTATUS(status) } -} - -// TODO: os.wait[pid] for windows -#[cfg(unix)] -fn os_waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { - let mut status = 0; - let pid = unsafe { libc::waitpid(pid, &mut status, opt) }; - let pid = Errno::result(pid).map_err(|e| convert_nix_error(vm, e))?; - Ok((pid, status)) -} -#[cfg(unix)] -fn os_wait(vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { - os_waitpid(-1, 0, vm) -} - -fn os_kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { - #[cfg(unix)] - { - let ret = unsafe { libc::kill(pid, sig as i32) }; - if ret == -1 { - Err(errno_err(vm)) - } else { - Ok(()) + match fs::read_dir(path.path) { + Ok(iter) => Ok(ScandirIterator { + entries: PyRwLock::new(iter), + exhausted: AtomicCell::new(false), + mode: path.mode, + } + .into_ref(vm) + .into_object()), + Err(s) => Err(convert_io_error(vm, s)), } } + + #[pystruct_sequence(module = "os", name = "stat_result")] + #[derive(Debug)] + pub(super) struct StatResult { + pub st_mode: u32, + pub st_ino: u64, + pub st_dev: u64, + pub st_nlink: u64, + pub st_uid: u32, + pub st_gid: u32, + pub st_size: u64, + pub st_atime: f64, + pub st_mtime: f64, + pub st_ctime: f64, + } + + impl StatResult { + pub(super) fn into_obj(self, vm: &VirtualMachine) -> PyObjectRef { + self.into_struct_sequence(vm, vm.class(super::MODULE_NAME, "stat_result")) + .unwrap() + .into_object() + } + } + + #[pyfunction] + fn lstat(file: Either, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { + super::platform::stat( + file, + dir_fd, + FollowSymlinks { + follow_symlinks: false, + }, + vm, + ) + } + + #[cfg(all(not(unix), not(windows)))] + #[pyfunction] + pub(super) fn symlink( + src: PyPathLike, + dst: PyPathLike, + dir_fd: DirFd, + vm: &VirtualMachine, + ) -> PyResult<()> { + unimplemented!(); + } + + #[pyfunction] + fn getcwd(vm: &VirtualMachine) -> PyResult { + Ok(env::current_dir() + .map_err(|err| convert_io_error(vm, err))? + .as_path() + .to_str() + .unwrap() + .to_owned()) + } + + #[pyfunction] + fn chdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { + env::set_current_dir(&path.path).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn fspath(path: PyPathLike, vm: &VirtualMachine) -> PyResult { + path.mode.process_path(path.path, vm) + } + + #[pyfunction] + fn rename(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { + fs::rename(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn getpid(vm: &VirtualMachine) -> PyObjectRef { + let pid = std::process::id(); + vm.new_int(pid) + } + + #[pyfunction] + fn cpu_count(vm: &VirtualMachine) -> PyObjectRef { + let cpu_count = num_cpus::get(); + vm.new_int(cpu_count) + } + + #[pyfunction] + fn exit(code: i32) { + std::process::exit(code) + } + + #[pyfunction] + fn urandom(size: usize, vm: &VirtualMachine) -> PyResult> { + let mut buf = vec![0u8; size]; + match getrandom::getrandom(&mut buf) { + Ok(()) => Ok(buf), + Err(e) => match e.raw_os_error() { + Some(errno) => Err(convert_io_error(vm, io::Error::from_raw_os_error(errno))), + None => Err(vm.new_os_error("Getting random failed".to_owned())), + }, + } + } + + // this is basically what CPython has for Py_off_t; windows uses long long + // for offsets, other platforms just use off_t + #[cfg(not(windows))] + pub type Offset = libc::off_t; #[cfg(windows)] - { - use winapi::um::{handleapi, processthreadsapi, wincon, winnt}; - let sig = sig as u32; - let pid = pid as u32; + pub type Offset = libc::c_longlong; - if sig == wincon::CTRL_C_EVENT || sig == wincon::CTRL_BREAK_EVENT { - let ret = unsafe { wincon::GenerateConsoleCtrlEvent(sig, pid) }; - let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; - return res; - } - - let h = unsafe { processthreadsapi::OpenProcess(winnt::PROCESS_ALL_ACCESS, 0, pid) }; - if h.is_null() { - return Err(errno_err(vm)); - } - let ret = unsafe { processthreadsapi::TerminateProcess(h, sig) }; - let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; - unsafe { handleapi::CloseHandle(h) }; - res + #[pyfunction] + fn isatty(fd: i32) -> bool { + unsafe { suppress_iph!(libc::isatty(fd)) != 0 } } - #[cfg(not(any(unix, windows)))] - { - unimplemented!() - } -} -fn os_strerror(e: i32) -> String { - unsafe { ffi::CStr::from_ptr(libc::strerror(e)) } - .to_string_lossy() - .into_owned() -} - -#[pystruct_sequence(module = "os", name = "terminal_size")] -#[allow(dead_code)] -struct PyTerminalSize { - columns: usize, - lines: usize, -} - -#[cfg(not(target_os = "wasi"))] -fn os_get_terminal_size(fd: OptionalArg, vm: &VirtualMachine) -> PyResult { - let (columns, lines) = { - #[cfg(unix)] - { - nix::ioctl_read_bad!(winsz, libc::TIOCGWINSZ, libc::winsize); - let mut w = libc::winsize { - ws_row: 0, - ws_col: 0, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe { winsz(fd.unwrap_or(libc::STDOUT_FILENO), &mut w) } - .map_err(|e| convert_nix_error(vm, e))?; - (w.ws_col.into(), w.ws_row.into()) - } + #[pyfunction] + fn lseek(fd: i32, position: Offset, how: i32, vm: &VirtualMachine) -> PyResult { + #[cfg(not(windows))] + let res = unsafe { suppress_iph!(libc::lseek(fd, position, how)) }; #[cfg(windows)] - { - use winapi::um::{handleapi, processenv, winbase, wincon}; - let stdhandle = match fd { - OptionalArg::Present(0) => winbase::STD_INPUT_HANDLE, - OptionalArg::Present(1) | OptionalArg::Missing => winbase::STD_OUTPUT_HANDLE, - OptionalArg::Present(2) => winbase::STD_ERROR_HANDLE, - _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), - }; - let h = unsafe { processenv::GetStdHandle(stdhandle) }; - if h.is_null() { - return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); + let res = unsafe { + use std::os::windows::io::RawHandle; + use winapi::um::{fileapi, winnt}; + let mut li = winnt::LARGE_INTEGER::default(); + *li.QuadPart_mut() = position; + let ret = fileapi::SetFilePointer( + fd as RawHandle, + li.u().LowPart as _, + &mut li.u_mut().HighPart, + how as _, + ); + if ret == fileapi::INVALID_SET_FILE_POINTER { + -1 + } else { + li.u_mut().LowPart = ret; + *li.QuadPart() } - if h == handleapi::INVALID_HANDLE_VALUE { - return Err(errno_err(vm)); - } - let mut csbi = wincon::CONSOLE_SCREEN_BUFFER_INFO::default(); - let ret = unsafe { wincon::GetConsoleScreenBufferInfo(h, &mut csbi) }; - if ret == 0 { - return Err(errno_err(vm)); - } - let w = csbi.srWindow; - ( - (w.Right - w.Left + 1) as usize, - (w.Bottom - w.Top + 1) as usize, - ) + }; + if res < 0 { + Err(errno_err(vm)) + } else { + Ok(res) } - }; - PyTerminalSize { columns, lines } - .into_struct_sequence(vm, vm.try_class(MODULE_NAME, "terminal_size")?) + } + + #[pyfunction] + fn link(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { + fs::hard_link(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) + } + + #[derive(FromArgs)] + struct UtimeArgs { + #[pyarg(positional_or_keyword)] + path: PyPathLike, + #[pyarg(positional_or_keyword, default = "None")] + times: Option, + #[pyarg(keyword_only, default = "None")] + ns: Option, + #[pyarg(flatten)] + _dir_fd: DirFd, + #[pyarg(flatten)] + _follow_symlinks: FollowSymlinks, + } + + #[cfg(not(target_os = "wasi"))] + #[pyfunction] + fn utime(args: UtimeArgs, vm: &VirtualMachine) -> PyResult<()> { + let parse_tup = |tup: PyTupleRef| -> Option<(i64, i64)> { + let tup = tup.as_slice(); + if tup.len() != 2 { + return None; + } + let i = |e: &PyObjectRef| e.clone().downcast::().ok()?.as_bigint().to_i64(); + Some((i(&tup[0])?, i(&tup[1])?)) + }; + let (acc, modif) = match (args.times, args.ns) { + (Some(t), None) => parse_tup(t).ok_or_else(|| { + vm.new_type_error( + "utime: 'times' must be either a tuple of two ints or None".to_owned(), + ) + })?, + (None, Some(ns)) => { + let (a, m) = parse_tup(ns).ok_or_else(|| { + vm.new_type_error("utime: 'ns' must be a tuple of two ints".to_owned()) + })?; + // TODO: do validation to make sure this doesn't.. underflow? + (a / 1_000_000_000, m / 1_000_000_000) + } + (None, None) => { + let now = SystemTime::now(); + let now = now + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| -(e.duration().as_secs() as i64)); + (now, now) + } + (Some(_), Some(_)) => { + return Err(vm.new_value_error( + "utime: you may specify either 'times' or 'ns' but not both".to_owned(), + )) + } + }; + utime::set_file_times(&args.path.path, acc, modif).map_err(|e| convert_io_error(vm, e)) + } + + #[pyfunction] + fn strerror(e: i32) -> String { + unsafe { ffi::CStr::from_ptr(libc::strerror(e)) } + .to_string_lossy() + .into_owned() + } + + #[pystruct_sequence(module = "os", name = "terminal_size")] + #[allow(dead_code)] + pub(super) struct PyTerminalSize { + pub columns: usize, + pub lines: usize, + } + + pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + let mut supports = super::platform::support_funcs(vm); + supports.extend(vec![ + SupportFunc::new(vm, "open", open, None, Some(false), None), + SupportFunc::new( + vm, + "access", + platform::access, + Some(false), + Some(false), + None, + ), + SupportFunc::new(vm, "chdir", chdir, Some(false), None, None), + // chflags Some, None Some + SupportFunc::new(vm, "listdir", listdir, Some(false), None, None), + SupportFunc::new(vm, "mkdir", mkdir, Some(false), Some(false), None), + // mkfifo Some Some None + // mknod Some Some None + // pathconf Some None None + SupportFunc::new(vm, "readlink", readlink, Some(false), Some(false), None), + SupportFunc::new(vm, "remove", remove, Some(false), Some(false), None), + SupportFunc::new(vm, "rename", rename, Some(false), Some(false), None), + SupportFunc::new(vm, "replace", rename, Some(false), Some(false), None), // TODO: Fix replace + SupportFunc::new(vm, "rmdir", rmdir, Some(false), Some(false), None), + SupportFunc::new(vm, "scandir", scandir, Some(false), None, None), + SupportFunc::new( + vm, + "stat", + platform::stat, + Some(false), + Some(false), + Some(false), + ), + SupportFunc::new( + vm, + "fstat", + platform::stat, + Some(false), + Some(false), + Some(false), + ), + SupportFunc::new(vm, "symlink", platform::symlink, None, Some(false), None), + // truncate Some None None + SupportFunc::new(vm, "unlink", remove, Some(false), Some(false), None), + #[cfg(not(target_os = "wasi"))] + SupportFunc::new(vm, "utime", utime, Some(false), Some(false), Some(false)), + ]); + supports + } } -// from libstd: -// https://github.com/rust-lang/rust/blob/daecab3a784f28082df90cebb204998051f3557d/src/libstd/sys/unix/fs.rs#L1251 -#[cfg(target_os = "macos")] -extern "C" { - fn fcopyfile( - in_fd: libc::c_int, - out_fd: libc::c_int, - state: *mut libc::c_void, // copyfile_state_t (unused) - flags: u32, // copyfile_flags_t - ) -> libc::c_int; +struct SupportFunc { + name: &'static str, + func_obj: PyObjectRef, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, } -#[cfg(target_os = "macos")] -const COPYFILE_DATA: u32 = 1 << 3; -#[cfg(target_os = "macos")] -fn os_fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; - if ret < 0 { - Err(errno_err(vm)) - } else { - Ok(()) + +impl<'a> SupportFunc { + fn new( + vm: &VirtualMachine, + name: &'static str, + func: F, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + ) -> Self + where + F: IntoPyNativeFunc, + { + let func_obj = vm.ctx.new_function(func); + Self { + name, + func_obj, + fd, + dir_fd, + follow_symlinks, + } } } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let environ = platform::_environ(vm); + let module = platform::make_module(vm); - let environ = _os_environ(vm); - - let scandir_iter = ctx.new_class("ScandirIter", ctx.object()); - ScandirIterator::extend_class(ctx, &scandir_iter); - - let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { - "name" => ctx.new_readonly_getset("name", DirEntryRef::name), - "path" => ctx.new_readonly_getset("path", DirEntryRef::path), - "is_dir" => ctx.new_method(DirEntryRef::is_dir), - "is_file" => ctx.new_method(DirEntryRef::is_file), - "is_symlink" => ctx.new_method(DirEntryRef::is_symlink), - "stat" => ctx.new_method(DirEntryRef::stat), - }); - - let stat_result = StatResult::make_class(ctx); - let terminal_size = PyTerminalSize::make_class(ctx); - - struct SupportFunc<'a> { - name: &'a str, - func_obj: PyObjectRef, - fd: Option, - dir_fd: Option, - follow_symlinks: Option, - }; - impl<'a> SupportFunc<'a> { - fn new( - vm: &VirtualMachine, - name: &'a str, - func: F, - fd: Option, - dir_fd: Option, - follow_symlinks: Option, - ) -> Self - where - F: IntoPyNativeFunc, - { - let func_obj = vm.ctx.new_function(func); - Self { - name, - func_obj, - fd, - dir_fd, - follow_symlinks, - } - } - } - #[allow(unused_mut)] - let mut support_funcs = vec![ - SupportFunc::new(vm, "open", os_open, None, Some(false), None), - SupportFunc::new(vm, "access", os_access, Some(false), Some(false), None), - SupportFunc::new(vm, "chdir", os_chdir, Some(false), None, None), - // chflags Some, None Some - SupportFunc::new(vm, "listdir", os_listdir, Some(false), None, None), - SupportFunc::new(vm, "mkdir", os_mkdir, Some(false), Some(false), None), - // mkfifo Some Some None - // mknod Some Some None - // pathconf Some None None - SupportFunc::new(vm, "readlink", os_readlink, Some(false), Some(false), None), - SupportFunc::new(vm, "remove", os_remove, Some(false), Some(false), None), - SupportFunc::new(vm, "rename", os_rename, Some(false), Some(false), None), - SupportFunc::new(vm, "replace", os_rename, Some(false), Some(false), None), // TODO: Fix replace - SupportFunc::new(vm, "rmdir", os_rmdir, Some(false), Some(false), None), - SupportFunc::new(vm, "scandir", os_scandir, Some(false), None, None), - SupportFunc::new(vm, "stat", os_stat, Some(false), Some(false), Some(false)), - SupportFunc::new(vm, "fstat", os_stat, Some(false), Some(false), Some(false)), - SupportFunc::new(vm, "symlink", os_symlink, None, Some(false), None), - // truncate Some None None - SupportFunc::new(vm, "unlink", os_remove, Some(false), Some(false), None), - #[cfg(not(target_os = "wasi"))] - SupportFunc::new(vm, "utime", os_utime, Some(false), Some(false), Some(false)), - ]; - #[cfg(unix)] - support_funcs.extend(vec![ - SupportFunc::new(vm, "chmod", os_chmod, Some(false), Some(false), Some(false)), - #[cfg(not(target_os = "redox"))] - SupportFunc::new(vm, "chroot", os_chroot, Some(false), None, None), - SupportFunc::new(vm, "chown", os_chown, Some(true), Some(true), Some(true)), - SupportFunc::new(vm, "lchown", os_lchown, None, None, None), - SupportFunc::new(vm, "fchown", os_fchown, Some(true), None, Some(true)), - SupportFunc::new(vm, "umask", os_umask, Some(false), Some(false), Some(false)), - SupportFunc::new(vm, "execv", os_execv, None, None, None), - ]); - let supports_fd = PySet::default().into_ref(vm); - let supports_dir_fd = PySet::default().into_ref(vm); - let supports_follow_symlinks = PySet::default().into_ref(vm); - - let module = py_module!(vm, MODULE_NAME, { - "close" => ctx.new_function(os_close), - "error" => ctx.new_function(os_error), - "fsync" => ctx.new_function(os_fsync), - "read" => ctx.new_function(os_read), - "write" => ctx.new_function(os_write), - "mkdirs" => ctx.new_function(os_mkdirs), - "putenv" => ctx.new_function(os_putenv), - "unsetenv" => ctx.new_function(os_unsetenv), + extend_module!(vm, module, { "environ" => environ, - "ScandirIter" => scandir_iter, - "DirEntry" => dir_entry, - "stat_result" => stat_result, - "terminal_size" => terminal_size, - "lstat" => ctx.new_function(os_lstat), - "getcwd" => ctx.new_function(os_getcwd), - "chdir" => ctx.new_function(os_chdir), - "fspath" => ctx.new_function(os_fspath), - "getpid" => ctx.new_function(os_getpid), - "cpu_count" => ctx.new_function(os_cpu_count), - "_exit" => ctx.new_function(os_exit), - "urandom" => ctx.new_function(os_urandom), - "isatty" => ctx.new_function(os_isatty), - "lseek" => ctx.new_function(os_lseek), - "set_inheritable" => ctx.new_function(os_set_inheritable), - "link" => ctx.new_function(os_link), - "kill" => ctx.new_function(os_kill), - "strerror" => ctx.new_function(os_strerror), "O_RDONLY" => ctx.new_int(libc::O_RDONLY), "O_WRONLY" => ctx.new_int(libc::O_WRONLY), @@ -2251,6 +878,13 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "SEEK_END" => ctx.new_int(libc::SEEK_END), }); + _os::extend_module(&vm, &module); + platform::extend_module_platform_specific(&vm, &module); + + let support_funcs = _os::support_funcs(vm); + let supports_fd = PySet::default().into_ref(vm); + let supports_dir_fd = PySet::default().into_ref(vm); + let supports_follow_symlinks = PySet::default().into_ref(vm); for support in support_funcs { if support.fd.unwrap_or(false) { supports_fd @@ -2280,120 +914,457 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "supports_follow_symlinks" => supports_follow_symlinks.into_object(), }); - #[cfg(not(target_os = "wasi"))] - extend_module!(vm, module, { - "get_terminal_size" => ctx.new_function(os_get_terminal_size), - }); - - extend_module_platform_specific(&vm, &module); - module } +pub(crate) use _os::open; + +// Copied code from Duration::as_secs_f64 as it's still unstable +fn duration_as_secs_f64(duration: Duration) -> f64 { + (duration.as_secs() as f64) + f64::from(duration.subsec_nanos()) / 1_000_000_000_f64 +} + +fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { + match sys_time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(duration) => duration_as_secs_f64(duration), + Err(err) => -duration_as_secs_f64(err.duration()), + } +} #[cfg(unix)] -fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { - let ctx = &vm.ctx; +#[pymodule(name = "posix")] +mod posix { + use super::*; - let uname_result = UnameResult::make_class(ctx); + use crate::obj::objdict::PyMapping; + use crate::obj::objlist::PyListRef; + use crate::pyobject::PyIterable; + use bitflags::bitflags; + use nix::errno::Errno; + use nix::unistd::{self, Gid, Pid, Uid}; + use std::convert::TryFrom; + pub(super) use std::os::unix::fs::OpenOptionsExt; + use std::os::unix::io::RawFd; - extend_module!(vm, module, { - "chmod" => ctx.new_function(os_chmod), - "execv" => ctx.new_function(os_execv), // TODO: windows - "get_inheritable" => ctx.new_function(os_get_inheritable), // TODO: windows - "get_blocking" => ctx.new_function(os_get_blocking), - "getppid" => ctx.new_function(os_getppid), - "getgid" => ctx.new_function(os_getgid), - "getegid" => ctx.new_function(os_getegid), - "getpgid" => ctx.new_function(os_getpgid), - "getuid" => ctx.new_function(os_getuid), - "getpgrp" => ctx.new_function(os_getpgrp), - "geteuid" => ctx.new_function(os_geteuid), - "pipe" => ctx.new_function(os_pipe), //TODO: windows - "set_blocking" => ctx.new_function(os_set_blocking), - "setgid" => ctx.new_function(os_setgid), - "setpgid" => ctx.new_function(os_setpgid), - "setuid" => ctx.new_function(os_setuid), - "sync" => ctx.new_function(os_sync), - "system" => ctx.new_function(os_system), - "ttyname" => ctx.new_function(os_ttyname), - "uname" => ctx.new_function(os_uname), - "uname_result" => uname_result, - "wait" => ctx.new_function(os_wait), - "waitpid" => ctx.new_function(os_waitpid), - "WIFSIGNALED" => ctx.new_function(os_wifsignaled), - "WIFSTOPPED" => ctx.new_function(os_wifstopped), - "WIFEXITED" => ctx.new_function(os_wifexited), - "WTERMSIG" => ctx.new_function(os_wtermsig), - "WSTOPSIG" => ctx.new_function(os_wstopsig), - "WEXITSTATUS" => ctx.new_function(os_wexitstatus), - "WNOHANG" => ctx.new_int(libc::WNOHANG), - "EX_OK" => ctx.new_int(exitcode::OK as i8), - "EX_USAGE" => ctx.new_int(exitcode::USAGE as i8), - "EX_DATAERR" => ctx.new_int(exitcode::DATAERR as i8), - "EX_NOINPUT" => ctx.new_int(exitcode::NOINPUT as i8), - "EX_NOUSER" => ctx.new_int(exitcode::NOUSER as i8), - "EX_NOHOST" => ctx.new_int(exitcode::NOHOST as i8), - "EX_UNAVAILABLE" => ctx.new_int(exitcode::UNAVAILABLE as i8), - "EX_SOFTWARE" => ctx.new_int(exitcode::SOFTWARE as i8), - "EX_OSERR" => ctx.new_int(exitcode::OSERR as i8), - "EX_OSFILE" => ctx.new_int(exitcode::OSFILE as i8), - "EX_CANTCREAT" => ctx.new_int(exitcode::CANTCREAT as i8), - "EX_IOERR" => ctx.new_int(exitcode::IOERR as i8), - "EX_TEMPFAIL" => ctx.new_int(exitcode::TEMPFAIL as i8), - "EX_PROTOCOL" => ctx.new_int(exitcode::PROTOCOL as i8), - "EX_NOPERM" => ctx.new_int(exitcode::NOPERM as i8), - "EX_CONFIG" => ctx.new_int(exitcode::CONFIG as i8), - "O_NONBLOCK" => ctx.new_int(libc::O_NONBLOCK), - "O_CLOEXEC" => ctx.new_int(libc::O_CLOEXEC), - }); + pub(super) type OpenFlags = i32; + + impl PyPathLike { + pub fn into_bytes(self) -> Vec { + use std::os::unix::ffi::OsStringExt; + self.path.into_os_string().into_vec() + } + } + + pub(crate) fn raw_file_number(handle: File) -> i64 { + use std::os::unix::io::IntoRawFd; + + i64::from(handle.into_raw_fd()) + } + + pub(crate) fn rust_file(raw_fileno: i64) -> File { + use std::os::unix::io::FromRawFd; + + unsafe { File::from_raw_fd(raw_fileno as i32) } + } + + pub fn convert_nix_error(vm: &VirtualMachine, err: nix::Error) -> PyBaseExceptionRef { + let nix_error = match err { + nix::Error::InvalidPath => { + let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); + vm.new_exception_msg(exc_type, err.to_string()) + } + nix::Error::InvalidUtf8 => { + let exc_type = vm.ctx.exceptions.unicode_error.clone(); + vm.new_exception_msg(exc_type, err.to_string()) + } + nix::Error::UnsupportedOperation => vm.new_runtime_error(err.to_string()), + nix::Error::Sys(errno) => { + let exc_type = convert_nix_errno(vm, errno); + vm.new_exception_msg(exc_type, err.to_string()) + } + }; + + if let nix::Error::Sys(errno) = err { + vm.set_attr(nix_error.as_object(), "errno", vm.ctx.new_int(errno as i32)) + .unwrap(); + } + + nix_error + } + + fn convert_nix_errno(vm: &VirtualMachine, errno: Errno) -> PyClassRef { + match errno { + Errno::EPERM => vm.ctx.exceptions.permission_error.clone(), + _ => vm.ctx.exceptions.os_error.clone(), + } + } + + // Flags for os_access + bitflags! { + pub struct AccessFlags: u8{ + const F_OK = 0; + const R_OK = 4; + const W_OK = 2; + const X_OK = 1; + } + } + + struct Permissions { + is_readable: bool, + is_writable: bool, + is_executable: bool, + } + + fn get_permissions(mode: u32) -> Permissions { + Permissions { + is_readable: mode & 4 != 0, + is_writable: mode & 2 != 0, + is_executable: mode & 1 != 0, + } + } + + fn get_right_permission( + mode: u32, + file_owner: Uid, + file_group: Gid, + ) -> Result { + let owner_mode = (mode & 0o700) >> 6; + let owner_permissions = get_permissions(owner_mode); + + let group_mode = (mode & 0o070) >> 3; + let group_permissions = get_permissions(group_mode); + + let others_mode = mode & 0o007; + let others_permissions = get_permissions(others_mode); + + let user_id = nix::unistd::getuid(); + let groups_ids = getgroups()?; + + if file_owner == user_id { + Ok(owner_permissions) + } else if groups_ids.contains(&file_group) { + Ok(group_permissions) + } else { + Ok(others_permissions) + } + } + + #[cfg(target_os = "macos")] + fn getgroups() -> nix::Result> { + use libc::{c_int, gid_t}; + use std::ptr; + let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; + let mut groups = Vec::::with_capacity(Errno::result(ret)? as usize); + let ret = unsafe { + libc::getgroups( + groups.capacity() as c_int, + groups.as_mut_ptr() as *mut gid_t, + ) + }; + + Errno::result(ret).map(|s| { + unsafe { groups.set_len(s as usize) }; + groups + }) + } + + #[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] + use nix::unistd::getgroups; + + #[cfg(target_os = "redox")] + fn getgroups() -> nix::Result> { + unimplemented!("redox getgroups") + } + + #[pyfunction] + pub(super) fn access(path: PyPathLike, mode: u8, vm: &VirtualMachine) -> PyResult { + use std::os::unix::fs::MetadataExt; + + let flags = AccessFlags::from_bits(mode).ok_or_else(|| { + vm.new_value_error( + "One of the flags is wrong, there are only 4 possibilities F_OK, R_OK, W_OK and X_OK" + .to_owned(), + ) + })?; + + let metadata = fs::metadata(&path.path); + + // if it's only checking for F_OK + if flags == AccessFlags::F_OK { + return Ok(metadata.is_ok()); + } + + let metadata = metadata.map_err(|err| convert_io_error(vm, err))?; + + let user_id = metadata.uid(); + let group_id = metadata.gid(); + let mode = metadata.mode(); + + let perm = get_right_permission(mode, Uid::from_raw(user_id), Gid::from_raw(group_id)) + .map_err(|err| convert_nix_error(vm, err))?; + + let r_ok = !flags.contains(AccessFlags::R_OK) || perm.is_readable; + let w_ok = !flags.contains(AccessFlags::W_OK) || perm.is_writable; + let x_ok = !flags.contains(AccessFlags::X_OK) || perm.is_executable; + + Ok(r_ok && w_ok && x_ok) + } + + pub(super) fn bytes_as_osstr<'a>( + b: &'a [u8], + _vm: &VirtualMachine, + ) -> PyResult<&'a ffi::OsStr> { + use std::os::unix::ffi::OsStrExt; + Ok(ffi::OsStr::from_bytes(b)) + } + + pub(super) fn _environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + use std::os::unix::ffi::OsStringExt; + for (key, value) in env::vars_os() { + environ + .set_item( + &vm.ctx.new_bytes(key.into_vec()), + vm.ctx.new_bytes(value.into_vec()), + vm, + ) + .unwrap(); + } + + environ + } + + fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { + let duration = Duration::new(secs as u64, nanos as u32); + duration_as_secs_f64(duration) + } + + #[pyfunction] + pub(super) fn stat( + file: Either, + dir_fd: super::DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + #[cfg(target_os = "android")] + use std::os::android::fs::MetadataExt; + #[cfg(target_os = "linux")] + use std::os::linux::fs::MetadataExt; + #[cfg(target_os = "macos")] + use std::os::macos::fs::MetadataExt; + #[cfg(target_os = "openbsd")] + use std::os::openbsd::fs::MetadataExt; + #[cfg(target_os = "redox")] + use std::os::redox::fs::MetadataExt; + + let get_stats = move || -> io::Result { + let meta = match file { + Either::A(path) => { + let path = make_path(vm, &path, &dir_fd); + if follow_symlinks.follow_symlinks { + fs::metadata(path)? + } else { + fs::symlink_metadata(path)? + } + } + Either::B(fno) => { + let file = rust_file(fno); + let meta = file.metadata()?; + raw_file_number(file); + meta + } + }; + + Ok(super::_os::StatResult { + st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), + st_size: meta.st_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_nanos(meta.st_ctime(), meta.st_ctime_nsec()), + } + .into_obj(vm)) + }; + + get_stats().map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + pub(super) fn symlink( + src: PyPathLike, + dst: PyPathLike, + dir_fd: DirFd, + vm: &VirtualMachine, + ) -> PyResult<()> { + use std::os::unix::fs as unix_fs; + let dst = make_path(vm, &dst, &dir_fd); + unix_fs::symlink(src.path, dst).map_err(|err| convert_io_error(vm, err)) + } #[cfg(not(target_os = "redox"))] - extend_module!(vm, module, { - "chown" => ctx.new_function(os_chown), - "lchown" => ctx.new_function(os_lchown), - "fchown" => ctx.new_function(os_fchown), - "chroot" => ctx.new_function(os_chroot), - "getsid" => ctx.new_function(os_getsid), - "setsid" => ctx.new_function(os_setsid), - "setegid" => ctx.new_function(os_setegid), - "seteuid" => ctx.new_function(os_seteuid), - "setreuid" => ctx.new_function(os_setreuid), - "openpty" => ctx.new_function(os_openpty), - "O_DSYNC" => ctx.new_int(libc::O_DSYNC), - "O_NDELAY" => ctx.new_int(libc::O_NDELAY), - "O_NOCTTY" => ctx.new_int(libc::O_NOCTTY), - }); + #[pyfunction] + fn chroot(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { + nix::unistd::chroot(&*path.path).map_err(|err| convert_nix_error(vm, err)) + } - // cfg taken from nix - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - extend_module!(vm, module, { - "setresuid" => ctx.new_function(os_setresuid), - "getresuid" => ctx.new_function(os_getresuid), - "getresgid" => ctx.new_function(os_getresgid), - "setresgid" => ctx.new_function(os_setresgid), - "setregid" => ctx.new_function(os_setregid), - "initgroups" => ctx.new_function(os_initgroups), - "setgroups" => ctx.new_function(os_setgroups), - }); + // As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn chown( + path: Either, + uid: PyIntRef, + gid: PyIntRef, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + let uid = isize::try_from_object(&vm, uid.as_object().clone())?; + let gid = isize::try_from_object(&vm, gid.as_object().clone())?; - // cfg taken from nix - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - all( - target_os = "linux", - not(any(target_env = "musl", target_arch = "mips", target_arch = "mips64")) + let uid = if uid >= 0 { + Some(nix::unistd::Uid::from_raw(uid as u32)) + } else if uid == -1 { + None + } else { + return Err(vm.new_os_error(String::from("Specified uid is not valid."))); + }; + + let gid = if gid >= 0 { + Some(nix::unistd::Gid::from_raw(gid as u32)) + } else if gid == -1 { + None + } else { + return Err(vm.new_os_error(String::from("Specified gid is not valid."))); + }; + + let flag = if follow_symlinks.follow_symlinks { + nix::unistd::FchownatFlags::FollowSymlink + } else { + nix::unistd::FchownatFlags::NoFollowSymlink + }; + + let dir_fd: Option = match dir_fd.dir_fd { + Some(int_ref) => Some(i32::try_from_object(&vm, int_ref.as_object().clone())?), + None => None, + }; + + match path { + Either::A(p) => nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) + .map_err(|err| convert_nix_error(vm, err)), + Either::B(fd) => { + let path = fs::read_link(format!("/proc/self/fd/{}", fd)).map_err(|_| { + vm.new_os_error(String::from("Cannot find path for specified fd")) + })?; + nix::unistd::fchownat(dir_fd, &path, uid, gid, flag) + .map_err(|err| convert_nix_error(vm, err)) + } + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn lchown(path: PyPathLike, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + chown( + Either::A(path), + uid, + gid, + DirFd { dir_fd: None }, + FollowSymlinks { + follow_symlinks: false, + }, + vm, ) - ))] - extend_module!(vm, module, { - "SEEK_DATA" => ctx.new_int(unistd::Whence::SeekData as i8), - "SEEK_HOLE" => ctx.new_int(unistd::Whence::SeekHole as i8) - }); + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn fchown(fd: i64, uid: PyIntRef, gid: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + chown( + Either::B(fd), + uid, + gid, + DirFd { dir_fd: None }, + FollowSymlinks { + follow_symlinks: true, + }, + vm, + ) + } + + #[pyfunction] + fn get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { + use nix::fcntl::fcntl; + use nix::fcntl::FcntlArg; + let flags = fcntl(fd, FcntlArg::F_GETFD); + match flags { + Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), + Err(err) => Err(convert_nix_error(vm, err)), + } + } + + pub(crate) fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { + use nix::fcntl; + let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); + let mut new_flags = flags; + new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable); + if flags != new_flags { + fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?; + } + Ok(()) + } + + #[pyfunction] + fn set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { + raw_set_inheritable(fd as RawFd, inheritable).map_err(|err| convert_nix_error(vm, err)) + } + + #[pyfunction] + fn get_blocking(fd: RawFd, vm: &VirtualMachine) -> PyResult { + use nix::fcntl::fcntl; + use nix::fcntl::FcntlArg; + let flags = fcntl(fd, FcntlArg::F_GETFL); + match flags { + Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), + Err(err) => Err(convert_nix_error(vm, err)), + } + } + + #[pyfunction] + fn set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { + let _set_flag = || { + use nix::fcntl::fcntl; + use nix::fcntl::FcntlArg; + use nix::fcntl::OFlag; + + let flags = OFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFL)?); + let mut new_flags = flags; + new_flags.set(OFlag::from_bits_truncate(libc::O_NONBLOCK), !blocking); + if flags != new_flags { + fcntl(fd, FcntlArg::F_SETFL(new_flags))?; + } + Ok(()) + }; + _set_flag().map_err(|err| convert_nix_error(vm, err)) + } + + #[pyfunction] + fn pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { + use nix::unistd::close; + use nix::unistd::pipe; + let (rfd, wfd) = pipe().map_err(|err| convert_nix_error(vm, err))?; + set_inheritable(rfd.into(), false, vm) + .and_then(|_| set_inheritable(wfd.into(), false, vm)) + .map_err(|err| { + let _ = close(rfd); + let _ = close(wfd); + err + })?; + Ok((rfd, wfd)) + } + // cfg from nix #[cfg(any( target_os = "android", @@ -2404,33 +1375,1078 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { target_os = "netbsd", target_os = "openbsd" ))] - extend_module!(vm, module, { - "pipe2" => ctx.new_function(os_pipe2), - }); + #[pyfunction] + fn pipe2(flags: libc::c_int, vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { + use nix::fcntl::OFlag; + use nix::unistd::pipe2; + let oflags = OFlag::from_bits_truncate(flags); + pipe2(oflags).map_err(|err| convert_nix_error(vm, err)) + } - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - extend_module!(vm, module, { - "posix_spawn" => ctx.new_function(os_posix_spawn), - "posix_spawnp" => ctx.new_function(os_posix_spawnp), - "POSIX_SPAWN_OPEN" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Open)), - "POSIX_SPAWN_CLOSE" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Close)), - "POSIX_SPAWN_DUP2" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Dup2)), - }); + #[pyfunction] + fn system(command: PyStringRef) -> PyResult { + use std::ffi::CString; + + let rstr = command.as_str(); + let cstr = CString::new(rstr).unwrap(); + let x = unsafe { libc::system(cstr.as_ptr()) }; + Ok(x) + } + + #[pyfunction] + fn chmod( + path: PyPathLike, + dir_fd: DirFd, + mode: u32, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + use std::os::unix::fs::PermissionsExt; + + let path = make_path(vm, &path, &dir_fd); + let metadata = if follow_symlinks.follow_symlinks { + fs::metadata(path) + } else { + fs::symlink_metadata(path) + }; + let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + let mut permissions = meta.permissions(); + permissions.set_mode(mode); + fs::set_permissions(path, permissions).map_err(|err| convert_io_error(vm, err))?; + Ok(()) + } + + #[pyfunction] + fn execv( + path: PyStringRef, + argv_list: Either, + vm: &VirtualMachine, + ) -> PyResult<()> { + let path = ffi::CString::new(path.as_str()) + .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; + + let argv: Vec = match argv_list { + Either::A(list) => vm.extract_elements(list.as_object())?, + Either::B(tuple) => vm.extract_elements(tuple.as_object())?, + }; + let argv: Vec<&ffi::CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); + + if argv.is_empty() { + return Err(vm.new_value_error("execv() arg 2 must not be empty".to_owned())); + } + if argv.first().unwrap().to_bytes().is_empty() { + return Err( + vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned()) + ); + } + + unistd::execv(&path, &argv) + .map(|_ok| ()) + .map_err(|err| convert_nix_error(vm, err)) + } + + #[pyfunction] + fn getppid(vm: &VirtualMachine) -> PyObjectRef { + let ppid = unistd::getppid().as_raw(); + vm.new_int(ppid) + } + + #[pyfunction] + fn getgid(vm: &VirtualMachine) -> PyObjectRef { + let gid = unistd::getgid().as_raw(); + vm.new_int(gid) + } + + #[pyfunction] + fn getegid(vm: &VirtualMachine) -> PyObjectRef { + let egid = unistd::getegid().as_raw(); + vm.new_int(egid) + } + + #[pyfunction] + fn getpgid(pid: u32, vm: &VirtualMachine) -> PyResult { + match unistd::getpgid(Some(Pid::from_raw(pid as i32))) { + Ok(pgid) => Ok(vm.new_int(pgid.as_raw())), + Err(err) => Err(convert_nix_error(vm, err)), + } + } + + #[pyfunction] + fn getpgrp(vm: &VirtualMachine) -> PyResult { + Ok(vm.new_int(unistd::getpgrp().as_raw())) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn getsid(pid: u32, vm: &VirtualMachine) -> PyResult { + match unistd::getsid(Some(Pid::from_raw(pid as i32))) { + Ok(sid) => Ok(vm.new_int(sid.as_raw())), + Err(err) => Err(convert_nix_error(vm, err)), + } + } + + #[pyfunction] + fn getuid(vm: &VirtualMachine) -> PyObjectRef { + let uid = unistd::getuid().as_raw(); + vm.new_int(uid) + } + + #[pyfunction] + fn geteuid(vm: &VirtualMachine) -> PyObjectRef { + let euid = unistd::geteuid().as_raw(); + vm.new_int(euid) + } + + #[pyfunction] + fn setgid(gid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setgid(Gid::from_raw(gid)).map_err(|err| convert_nix_error(vm, err)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn setegid(egid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setegid(Gid::from_raw(egid)).map_err(|err| convert_nix_error(vm, err)) + } + + #[pyfunction] + fn setpgid(pid: u32, pgid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setpgid(Pid::from_raw(pid as i32), Pid::from_raw(pgid as i32)) + .map_err(|err| convert_nix_error(vm, err)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn setsid(vm: &VirtualMachine) -> PyResult<()> { + unistd::setsid() + .map(|_ok| ()) + .map_err(|err| convert_nix_error(vm, err)) + } + + #[pyfunction] + fn setuid(uid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setuid(Uid::from_raw(uid)).map_err(|err| convert_nix_error(vm, err)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn seteuid(euid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn setreuid(ruid: u32, euid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setuid(Uid::from_raw(ruid)).map_err(|err| convert_nix_error(vm, err))?; + unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn setresuid(ruid: u32, euid: u32, suid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setresuid( + Uid::from_raw(ruid), + Uid::from_raw(euid), + Uid::from_raw(suid), + ) + .map_err(|err| convert_nix_error(vm, err)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn openpty(vm: &VirtualMachine) -> PyResult { + match nix::pty::openpty(None, None) { + Ok(r) => Ok(vm + .ctx + .new_tuple(vec![vm.new_int(r.master), vm.new_int(r.slave)])), + Err(err) => Err(convert_nix_error(vm, err)), + } + } + + #[pyfunction] + fn ttyname(fd: i32, vm: &VirtualMachine) -> PyResult { + use libc::ttyname; + let name = unsafe { ttyname(fd) }; + if name.is_null() { + Err(errno_err(vm)) + } else { + let name = unsafe { ffi::CStr::from_ptr(name) }.to_str().unwrap(); + Ok(vm.ctx.new_str(name.to_owned())) + } + } + + #[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))] + type ModeT = u32; + + #[cfg(target_os = "redox")] + type ModeT = i32; #[cfg(target_os = "macos")] - extend_module!(vm, module, { - "_COPYFILE_DATA" => ctx.new_int(COPYFILE_DATA), - "_fcopyfile" => ctx.new_function(os_fcopyfile), - }); + type ModeT = u16; + + #[cfg(any( + target_os = "macos", + target_os = "linux", + target_os = "openbsd", + target_os = "redox", + target_os = "android", + ))] + #[pyfunction] + fn umask(mask: ModeT, _vm: &VirtualMachine) -> PyResult { + let ret_mask = unsafe { libc::umask(mask) }; + Ok(ret_mask) + } + + #[pystruct_sequence(module = "os", name = "uname_result")] + #[derive(Debug)] + struct UnameResult { + sysname: String, + nodename: String, + release: String, + version: String, + machine: String, + } + + impl UnameResult { + fn into_obj(self, vm: &VirtualMachine) -> PyObjectRef { + self.into_struct_sequence(vm, vm.class(super::MODULE_NAME, "uname_result")) + .unwrap() + .into_object() + } + } + + #[pyfunction] + fn uname(vm: &VirtualMachine) -> PyResult { + let info = uname::uname().map_err(|err| convert_io_error(vm, err))?; + Ok(UnameResult { + sysname: info.sysname, + nodename: info.nodename, + release: info.release, + version: info.version, + machine: info.machine, + } + .into_obj(vm)) + } + + #[pyfunction] + fn sync(_vm: &VirtualMachine) -> PyResult<()> { + #[cfg(not(any(target_os = "redox", target_os = "android")))] + unsafe { + libc::sync(); + } + Ok(()) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn getresuid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { + let mut ruid = 0; + let mut euid = 0; + let mut suid = 0; + let ret = unsafe { libc::getresuid(&mut ruid, &mut euid, &mut suid) }; + if ret == 0 { + Ok((ruid, euid, suid)) + } else { + Err(errno_err(vm)) + } + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn getresgid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { + let mut rgid = 0; + let mut egid = 0; + let mut sgid = 0; + let ret = unsafe { libc::getresgid(&mut rgid, &mut egid, &mut sgid) }; + if ret == 0 { + Ok((rgid, egid, sgid)) + } else { + Err(errno_err(vm)) + } + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn setresgid(rgid: u32, egid: u32, sgid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setresgid( + Gid::from_raw(rgid), + Gid::from_raw(egid), + Gid::from_raw(sgid), + ) + .map_err(|err| convert_nix_error(vm, err)) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn setregid(rgid: u32, egid: u32, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { libc::setregid(rgid, egid) }; + if ret == 0 { + Ok(()) + } else { + Err(errno_err(vm)) + } + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn initgroups(user_name: PyStringRef, gid: u32, vm: &VirtualMachine) -> PyResult<()> { + let user = ffi::CString::new(user_name.as_str()).unwrap(); + let gid = Gid::from_raw(gid); + unistd::initgroups(&user, gid).map_err(|err| convert_nix_error(vm, err)) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn setgroups(group_ids: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + let gids = group_ids + .iter(vm)? + .map(|entry| match entry { + Ok(id) => Ok(unistd::Gid::from_raw(id)), + Err(err) => Err(err), + }) + .collect::, _>>()?; + let ret = unistd::setgroups(&gids); + ret.map_err(|err| convert_nix_error(vm, err)) + } + + fn envp_from_dict(dict: PyDictRef, vm: &VirtualMachine) -> PyResult> { + dict.into_iter() + .map(|(k, v)| { + let k = PyPathLike::try_from_object(vm, k)?.into_bytes(); + let v = PyPathLike::try_from_object(vm, v)?.into_bytes(); + if k.contains(&0) { + return Err( + vm.new_value_error("envp dict key cannot contain a nul byte".to_owned()) + ); + } + if k.contains(&b'=') { + return Err(vm.new_value_error( + "envp dict key cannot contain a '=' character".to_owned(), + )); + } + if v.contains(&0) { + return Err( + vm.new_value_error("envp dict value cannot contain a nul byte".to_owned()) + ); + } + let mut env = k; + env.push(b'='); + env.extend(v); + Ok(unsafe { ffi::CString::from_vec_unchecked(env) }) + }) + .collect() + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[derive(FromArgs)] + pub(super) struct PosixSpawnArgs { + #[pyarg(positional_only)] + path: PyPathLike, + #[pyarg(positional_only)] + args: PyIterable, + #[pyarg(positional_only)] + env: PyMapping, + #[pyarg(keyword_only, default = "None")] + file_actions: Option>, + #[pyarg(keyword_only, default = "None")] + setsigdef: Option>, + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] + #[repr(i32)] + enum PosixSpawnFileActionIdentifier { + Open, + Close, + Dup2, + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + impl PosixSpawnArgs { + fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult { + let path = ffi::CString::new(self.path.into_bytes()) + .map_err(|_| vm.new_value_error("path should not have nul bytes".to_owned()))?; + + let mut file_actions = unsafe { + let mut fa = std::mem::MaybeUninit::uninit(); + assert!(libc::posix_spawn_file_actions_init(fa.as_mut_ptr()) == 0); + fa.assume_init() + }; + if let Some(it) = self.file_actions { + for action in it.iter(vm)? { + let action = action?; + let (id, args) = action.as_slice().split_first().ok_or_else(|| { + vm.new_type_error( + "Each file_actions element must be a non-empty tuple".to_owned(), + ) + })?; + let id = i32::try_from_object(vm, id.clone())?; + let id = PosixSpawnFileActionIdentifier::try_from(id).map_err(|_| { + vm.new_type_error("Unknown file_actions identifier".to_owned()) + })?; + let args = PyFuncArgs::from(args.to_vec()); + let ret = match id { + PosixSpawnFileActionIdentifier::Open => { + let (fd, path, oflag, mode): (_, PyPathLike, _, _) = args.bind(vm)?; + let path = ffi::CString::new(path.into_bytes()).map_err(|_| { + vm.new_value_error( + "POSIX_SPAWN_OPEN path should not have nul bytes".to_owned(), + ) + })?; + unsafe { + libc::posix_spawn_file_actions_addopen( + &mut file_actions, + fd, + path.as_ptr(), + oflag, + mode, + ) + } + } + PosixSpawnFileActionIdentifier::Close => { + let (fd,) = args.bind(vm)?; + unsafe { + libc::posix_spawn_file_actions_addclose(&mut file_actions, fd) + } + } + PosixSpawnFileActionIdentifier::Dup2 => { + let (fd, newfd) = args.bind(vm)?; + unsafe { + libc::posix_spawn_file_actions_adddup2(&mut file_actions, fd, newfd) + } + } + }; + if ret != 0 { + return Err(errno_err(vm)); + } + } + } + + let mut attrp = unsafe { + let mut sa = std::mem::MaybeUninit::uninit(); + assert!(libc::posix_spawnattr_init(sa.as_mut_ptr()) == 0); + sa.assume_init() + }; + if let Some(sigs) = self.setsigdef { + use nix::sys::signal; + let mut set = signal::SigSet::empty(); + for sig in sigs.iter(vm)? { + let sig = sig?; + let sig = signal::Signal::try_from(sig).map_err(|_| { + vm.new_value_error(format!("signal number {} out of range", sig)) + })?; + set.add(sig); + } + assert!( + unsafe { libc::posix_spawnattr_setsigdefault(&mut attrp, set.as_ref()) } == 0 + ); + } + + let mut args: Vec = self + .args + .iter(vm)? + .map(|res| { + ffi::CString::new(res?.into_bytes()).map_err(|_| { + vm.new_value_error("path should not have nul bytes".to_owned()) + }) + }) + .collect::>()?; + let argv: Vec<*mut libc::c_char> = args + .iter_mut() + .map(|s| s.as_ptr() as _) + .chain(std::iter::once(std::ptr::null_mut())) + .collect(); + let mut env = envp_from_dict(self.env.into_dict(), vm)?; + let envp: Vec<*mut libc::c_char> = env + .iter_mut() + .map(|s| s.as_ptr() as _) + .chain(std::iter::once(std::ptr::null_mut())) + .collect(); + + let mut pid = 0; + let ret = unsafe { + if spawnp { + libc::posix_spawnp( + &mut pid, + path.as_ptr(), + &file_actions, + &attrp, + argv.as_ptr(), + envp.as_ptr(), + ) + } else { + libc::posix_spawn( + &mut pid, + path.as_ptr(), + &file_actions, + &attrp, + argv.as_ptr(), + envp.as_ptr(), + ) + } + }; + + if ret == 0 { + Ok(pid) + } else { + Err(errno_err(vm)) + } + } + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyfunction] + fn posix_spawn(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult { + args.spawn(false, vm) + } + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyfunction] + fn posix_spawnp(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult { + args.spawn(true, vm) + } + + #[pyfunction(name = "WIFSIGNALED")] + fn wifsignaled(status: i32) -> bool { + unsafe { libc::WIFSIGNALED(status) } + } + #[pyfunction(name = "WIFSTOPPED")] + fn wifstopped(status: i32) -> bool { + unsafe { libc::WIFSTOPPED(status) } + } + #[pyfunction(name = "WIFEXITED")] + fn wifexited(status: i32) -> bool { + unsafe { libc::WIFEXITED(status) } + } + #[pyfunction(name = "WTERMSIG")] + fn wtermsig(status: i32) -> i32 { + unsafe { libc::WTERMSIG(status) } + } + #[pyfunction(name = "WSTOPSIG")] + fn wstopsig(status: i32) -> i32 { + unsafe { libc::WSTOPSIG(status) } + } + #[pyfunction(name = "WEXITSTATUS")] + fn wexitstatus(status: i32) -> i32 { + unsafe { libc::WEXITSTATUS(status) } + } + + // TODO: os.wait[pid] for windows + #[pyfunction] + fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { + let mut status = 0; + let pid = unsafe { libc::waitpid(pid, &mut status, opt) }; + let pid = Errno::result(pid).map_err(|e| convert_nix_error(vm, e))?; + Ok((pid, status)) + } + #[pyfunction] + fn wait(vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { + waitpid(-1, 0, vm) + } + + #[pyfunction] + fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { + { + let ret = unsafe { libc::kill(pid, sig as i32) }; + if ret == -1 { + Err(errno_err(vm)) + } else { + Ok(()) + } + } + } + + #[pyfunction] + fn get_terminal_size(fd: OptionalArg, vm: &VirtualMachine) -> PyResult { + let (columns, lines) = { + #[cfg(unix)] + { + nix::ioctl_read_bad!(winsz, libc::TIOCGWINSZ, libc::winsize); + let mut w = libc::winsize { + ws_row: 0, + ws_col: 0, + ws_xpixel: 0, + ws_ypixel: 0, + }; + unsafe { winsz(fd.unwrap_or(libc::STDOUT_FILENO), &mut w) } + .map_err(|e| convert_nix_error(vm, e))?; + (w.ws_col.into(), w.ws_row.into()) + } + }; + super::_os::PyTerminalSize { columns, lines } + .into_struct_sequence(vm, vm.try_class(super::MODULE_NAME, "terminal_size")?) + } + + // from libstd: + // https://github.com/rust-lang/rust/blob/daecab3a784f28082df90cebb204998051f3557d/src/libstd/sys/unix/fs.rs#L1251 + #[cfg(target_os = "macos")] + extern "C" { + fn fcopyfile( + in_fd: libc::c_int, + out_fd: libc::c_int, + state: *mut libc::c_void, // copyfile_state_t (unused) + flags: u32, // copyfile_flags_t + ) -> libc::c_int; + } + #[cfg(target_os = "macos")] + const COPYFILE_DATA: u32 = 1 << 3; + #[cfg(target_os = "macos")] + #[pyfunction] + fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; + if ret < 0 { + Err(errno_err(vm)) + } else { + Ok(()) + } + } + + pub(super) fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { + let ctx = &vm.ctx; + + extend_module!(vm, module, { + "WNOHANG" => ctx.new_int(libc::WNOHANG), + "EX_OK" => ctx.new_int(exitcode::OK as i8), + "EX_USAGE" => ctx.new_int(exitcode::USAGE as i8), + "EX_DATAERR" => ctx.new_int(exitcode::DATAERR as i8), + "EX_NOINPUT" => ctx.new_int(exitcode::NOINPUT as i8), + "EX_NOUSER" => ctx.new_int(exitcode::NOUSER as i8), + "EX_NOHOST" => ctx.new_int(exitcode::NOHOST as i8), + "EX_UNAVAILABLE" => ctx.new_int(exitcode::UNAVAILABLE as i8), + "EX_SOFTWARE" => ctx.new_int(exitcode::SOFTWARE as i8), + "EX_OSERR" => ctx.new_int(exitcode::OSERR as i8), + "EX_OSFILE" => ctx.new_int(exitcode::OSFILE as i8), + "EX_CANTCREAT" => ctx.new_int(exitcode::CANTCREAT as i8), + "EX_IOERR" => ctx.new_int(exitcode::IOERR as i8), + "EX_TEMPFAIL" => ctx.new_int(exitcode::TEMPFAIL as i8), + "EX_PROTOCOL" => ctx.new_int(exitcode::PROTOCOL as i8), + "EX_NOPERM" => ctx.new_int(exitcode::NOPERM as i8), + "EX_CONFIG" => ctx.new_int(exitcode::CONFIG as i8), + "O_NONBLOCK" => ctx.new_int(libc::O_NONBLOCK), + "O_CLOEXEC" => ctx.new_int(libc::O_CLOEXEC), + }); + + #[cfg(not(target_os = "redox"))] + extend_module!(vm, module, { + + "O_DSYNC" => ctx.new_int(libc::O_DSYNC), + "O_NDELAY" => ctx.new_int(libc::O_NDELAY), + "O_NOCTTY" => ctx.new_int(libc::O_NOCTTY), + }); + + // cfg taken from nix + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + all( + target_os = "linux", + not(any(target_env = "musl", target_arch = "mips", target_arch = "mips64")) + ) + ))] + extend_module!(vm, module, { + "SEEK_DATA" => ctx.new_int(unistd::Whence::SeekData as i8), + "SEEK_HOLE" => ctx.new_int(unistd::Whence::SeekHole as i8) + }); + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "emscripten", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + ))] + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + extend_module!(vm, module, { + "POSIX_SPAWN_OPEN" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Open)), + "POSIX_SPAWN_CLOSE" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Close)), + "POSIX_SPAWN_DUP2" => ctx.new_int(i32::from(PosixSpawnFileActionIdentifier::Dup2)), + }); + + #[cfg(target_os = "macos")] + extend_module!(vm, module, { + "_COPYFILE_DATA" => ctx.new_int(COPYFILE_DATA), + }); + } + + pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + vec![ + SupportFunc::new(vm, "chmod", chmod, Some(false), Some(false), Some(false)), + #[cfg(not(target_os = "redox"))] + SupportFunc::new(vm, "chroot", chroot, Some(false), None, None), + SupportFunc::new(vm, "chown", chown, Some(true), Some(true), Some(true)), + SupportFunc::new(vm, "lchown", lchown, None, None, None), + SupportFunc::new(vm, "fchown", fchown, Some(true), None, Some(true)), + SupportFunc::new(vm, "umask", umask, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "execv", execv, None, None, None), + ] + } } +#[cfg(unix)] +use posix as platform; +#[cfg(unix)] +pub(crate) use posix::{convert_nix_error, raw_set_inheritable as set_inheritable}; #[cfg(windows)] -fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { - let ctx = &vm.ctx; - extend_module!(vm, module, { - "O_BINARY" => ctx.new_int(libc::O_BINARY), - }); +#[pymodule] +mod nt { + use super::*; + pub(super) use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::RawHandle; + + pub(super) type OpenFlags = u32; + + pub fn raw_file_number(handle: File) -> i64 { + use std::os::windows::io::IntoRawHandle; + + handle.into_raw_handle() as i64 + } + + pub fn rust_file(raw_fileno: i64) -> File { + use std::os::windows::io::{AsRawHandle, FromRawHandle}; + + let raw_fileno = match raw_fileno { + 0 => io::stdin().as_raw_handle(), + 1 => io::stdout().as_raw_handle(), + 2 => io::stderr().as_raw_handle(), + fno => fno as RawHandle, + }; + + //This seems to work as expected but further testing is required. + unsafe { File::from_raw_handle(raw_fileno) } + } + + impl PyPathLike { + pub fn wide(&self) -> Vec { + use std::os::windows::ffi::OsStrExt; + self.path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect() + } + } + + #[pyfunction] + pub(super) fn access(path: PyPathLike, mode: u8) -> bool { + use winapi::um::{fileapi, winnt}; + let attr = unsafe { fileapi::GetFileAttributesW(path.wide().as_ptr()) }; + attr != fileapi::INVALID_FILE_ATTRIBUTES + && (mode & 2 == 0 + || attr & winnt::FILE_ATTRIBUTE_READONLY == 0 + || attr & winnt::FILE_ATTRIBUTE_DIRECTORY != 0) + } + + #[pyfunction] + pub(super) fn symlink( + src: PyPathLike, + dst: PyPathLike, + _dir_fd: DirFd, + vm: &VirtualMachine, + ) -> PyResult<()> { + use std::os::windows::fs as win_fs; + let meta = fs::metadata(src.path.clone()).map_err(|err| convert_io_error(vm, err))?; + let ret = if meta.is_file() { + win_fs::symlink_file(src.path, dst.path) + } else if meta.is_dir() { + win_fs::symlink_dir(src.path, dst.path) + } else { + panic!("Unknown file type"); + }; + ret.map_err(|err| convert_io_error(vm, err)) + } + + #[pyfunction] + fn set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(windows)] + { + use winapi::um::{handleapi, winbase}; + let fd = fd as RawHandle; + let flags = if inheritable { + winbase::HANDLE_FLAG_INHERIT + } else { + 0 + }; + let ret = + unsafe { handleapi::SetHandleInformation(fd, winbase::HANDLE_FLAG_INHERIT, flags) }; + if ret == 0 { + Err(errno_err(vm)) + } else { + Ok(()) + } + } + } + + // Copied from CPython fileutils.c + fn attributes_to_mode(attr: u32) -> u32 { + const FILE_ATTRIBUTE_DIRECTORY: u32 = 16; + const FILE_ATTRIBUTE_READONLY: u32 = 1; + const S_IFDIR: u32 = 0o040000; + const S_IFREG: u32 = 0o100000; + let mut m: u32 = 0; + if attr & FILE_ATTRIBUTE_DIRECTORY == FILE_ATTRIBUTE_DIRECTORY { + m |= S_IFDIR | 0111; /* IFEXEC for user,group,other */ + } else { + m |= S_IFREG; + } + if attr & FILE_ATTRIBUTE_READONLY == FILE_ATTRIBUTE_READONLY { + m |= 0444; + } else { + m |= 0666; + } + m + } + + pub(super) fn _environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + + for (key, value) in env::vars() { + environ + .set_item(&vm.new_str(key), vm.new_str(value), vm) + .unwrap(); + } + environ + } + + #[pyfunction] + pub(super) fn stat( + file: Either, + _dir_fd: DirFd, // TODO: error + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + use std::os::windows::fs::MetadataExt; + + let get_stats = move || -> io::Result { + let meta = match file { + Either::A(path) => match follow_symlinks.follow_symlinks { + true => fs::metadata(path.path)?, + false => fs::symlink_metadata(path.path)?, + }, + Either::B(fno) => { + let f = rust_file(fno); + let meta = f.metadata()?; + raw_file_number(f); + meta + } + }; + + Ok(super::_os::StatResult { + st_mode: attributes_to_mode(meta.file_attributes()), + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_uid: 0, // 0 on windows + st_gid: 0, // 0 on windows + st_size: meta.file_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_unix_epoch(meta.created()?), + } + .into_obj(vm)) + }; + + get_stats().map_err(|e| convert_io_error(vm, e)) + } + + #[pyfunction] + fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { + { + use winapi::um::{handleapi, processthreadsapi, wincon, winnt}; + let sig = sig as u32; + let pid = pid as u32; + + if sig == wincon::CTRL_C_EVENT || sig == wincon::CTRL_BREAK_EVENT { + let ret = unsafe { wincon::GenerateConsoleCtrlEvent(sig, pid) }; + let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; + return res; + } + + let h = unsafe { processthreadsapi::OpenProcess(winnt::PROCESS_ALL_ACCESS, 0, pid) }; + if h.is_null() { + return Err(errno_err(vm)); + } + let ret = unsafe { processthreadsapi::TerminateProcess(h, sig) }; + let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; + unsafe { handleapi::CloseHandle(h) }; + res + } + } + + #[pyfunction] + fn get_terminal_size(fd: OptionalArg, vm: &VirtualMachine) -> PyResult { + let (columns, lines) = { + { + use winapi::um::{handleapi, processenv, winbase, wincon}; + let stdhandle = match fd { + OptionalArg::Present(0) => winbase::STD_INPUT_HANDLE, + OptionalArg::Present(1) | OptionalArg::Missing => winbase::STD_OUTPUT_HANDLE, + OptionalArg::Present(2) => winbase::STD_ERROR_HANDLE, + _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), + }; + let h = unsafe { processenv::GetStdHandle(stdhandle) }; + if h.is_null() { + return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); + } + if h == handleapi::INVALID_HANDLE_VALUE { + return Err(errno_err(vm)); + } + let mut csbi = wincon::CONSOLE_SCREEN_BUFFER_INFO::default(); + let ret = unsafe { wincon::GetConsoleScreenBufferInfo(h, &mut csbi) }; + if ret == 0 { + return Err(errno_err(vm)); + } + let w = csbi.srWindow; + ( + (w.Right - w.Left + 1) as usize, + (w.Bottom - w.Top + 1) as usize, + ) + } + }; + super::_os::PyTerminalSize { columns, lines } + .into_struct_sequence(vm, vm.try_class(super::MODULE_NAME, "terminal_size")?) + } + + #[cfg(target_env = "msvc")] + type InvalidParamHandler = extern "C" fn( + *const libc::wchar_t, + *const libc::wchar_t, + *const libc::wchar_t, + libc::c_uint, + libc::uintptr_t, + ); + #[cfg(target_env = "msvc")] + extern "C" { + #[doc(hidden)] + pub fn _set_thread_local_invalid_parameter_handler( + pNew: InvalidParamHandler, + ) -> InvalidParamHandler; + } + + #[cfg(target_env = "msvc")] + #[doc(hidden)] + pub extern "C" fn silent_iph_handler( + _: *const libc::wchar_t, + _: *const libc::wchar_t, + _: *const libc::wchar_t, + _: libc::c_uint, + _: libc::uintptr_t, + ) { + } + + pub(super) fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { + let ctx = &vm.ctx; + extend_module!(vm, module, { + "O_BINARY" => ctx.new_int(libc::O_BINARY), + }); + } + + pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + Vec::new() + } } +#[cfg(windows)] +use nt as platform; +#[cfg(windows)] +pub use nt::{_set_thread_local_invalid_parameter_handler, silent_iph_handler}; #[cfg(not(any(unix, windows)))] -fn extend_module_platform_specific(_vm: &VirtualMachine, _module: &PyObjectRef) {} +mod minor { + use super::*; + + pub const MODULE_NAME: &str = "posix"; + + #[cfg(target_os = "wasi")] + pub(super) type OpenFlags = u16; + + #[cfg(target_os = "wasi")] + pub(crate) fn raw_file_number(handle: File) -> i64 { + // This should be safe, since the wasi api is pretty well defined, but once + // `wasi_ext` get's stabilized, we should use that instead. + unsafe { std::mem::transmute::<_, u32>(handle).into() } + } + #[cfg(not(target_os = "wasi"))] + pub(crate) fn raw_file_number(handle: File) -> i64 { + unimplemented!(); + } + + #[cfg(target_os = "wasi")] + pub(crate) fn rust_file(raw_fileno: i64) -> File { + unsafe { std::mem::transmute(raw_fileno as u32) } + } + + #[cfg(not(target_os = "wasi"))] + pub(crate) fn rust_file(raw_fileno: i64) -> File { + unimplemented!(); + } + + #[pyfunction] + pub(super) fn access(path: PyStringRef, mode: u8, vm: &VirtualMachine) -> PyResult { + unimplemented!() + } + + #[pyfunction] + fn stat( + file: Either, + _dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + unimplemented!(); + } + + pub(super) fn _environ(vm: &VirtualMachine) -> PyDictRef { + vm.ctx.new_dict() + } + + fn extend_module_platform_specific(_vm: &VirtualMachine, _module: &PyObjectRef) {} + + pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + Vec::new() + } +} +#[cfg(not(any(unix, windows)))] +use minor as platform; + +pub(crate) use platform::{raw_file_number, rust_file, MODULE_NAME}; From 123eefd87f4d869403fc9e9e430202810072774f Mon Sep 17 00:00:00 2001 From: clemado1 Date: Wed, 29 Jul 2020 22:52:38 +0900 Subject: [PATCH 141/209] Allow None parameter for remove, index, count method of array --- Lib/test/test_array.py | 6 ------ vm/src/stdlib/array.rs | 24 +++++++++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 7cca83d78..d1b2f03f4 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -918,8 +918,6 @@ class BaseTest: del a[start:stop:step] self.assertEqual(a, array.array(self.typecode, L)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_index(self): example = 2*self.example a = array.array(self.typecode, example) @@ -929,8 +927,6 @@ class BaseTest: self.assertRaises(ValueError, a.index, None) self.assertRaises(ValueError, a.index, self.outside) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_count(self): example = 2*self.example a = array.array(self.typecode, example) @@ -940,8 +936,6 @@ class BaseTest: self.assertEqual(a.count(self.outside), 0) self.assertEqual(a.count(None), 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_remove(self): for x in self.example: example = 2*self.example diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 87c2afaa2..4e397d1f4 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -99,8 +99,9 @@ macro_rules! def_array_enum { fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self { $(ArrayContentType::$n(v) => { - let val = $t::try_from_object(vm, obj)?; - Ok(v.iter().filter(|&&a| a == val).count()) + Ok(>::try_from_object(vm, obj)?.map_or(0, |val| { + v.iter().filter(|&&a| a == val).count() + })) })* } } @@ -108,11 +109,15 @@ macro_rules! def_array_enum { fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ match self { $(ArrayContentType::$n(v) => { - let val = $t::try_from_object(vm, obj)?; - if let Some(pos) = v.iter().position(|&a| a == val) { - v.remove(pos); - } else { - return Err(vm.new_value_error("array.remove(x): x not in array".to_owned())); + let pos = >::try_from_object(vm, obj)?.map_or(None, |val| { + v.iter().position(|&a| a == val) + }); + + match pos { + Some(x) => { + v.remove(x); + }, + None => return Err(vm.new_value_error("array.remove(x): x not in array".to_owned())) } })* } @@ -147,8 +152,9 @@ macro_rules! def_array_enum { fn index(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult> { match self { $(ArrayContentType::$n(v) => { - let val = $t::try_from_object(vm, x)?; - Ok(v.iter().position(|&a| a == val)) + Ok(>::try_from_object(vm, x)?.map_or(None, |val| { + v.iter().position(|&a| a == val) + })) })* } } From f6a08b8a0186438d14d70a48e9f7b926b95c4507 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 30 Jul 2020 11:37:31 -0500 Subject: [PATCH 142/209] Add a note about `scripts\symlinks-to-hardlinks.ps1` in the README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a50600b4a..f5a45c601 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ To test RustPython, do the following: $ git clone https://github.com/RustPython/RustPython $ cd RustPython + # if you're on windows: + $ powershell scripts\symlinks-to-hardlinks.ps1 $ cargo run demo.py Hello, RustPython! From 8a6ac64f03fcb9bd49ff65b018685ea3c6779645 Mon Sep 17 00:00:00 2001 From: lynskylate Date: Fri, 31 Jul 2020 02:11:00 +0800 Subject: [PATCH 143/209] Introduce symtable test and add is_annotation for symbol --- Lib/test/test_symtable.py | 251 ++++++++++++++++++++++++++++++++++++ compiler/src/symboltable.rs | 42 +++++- vm/src/stdlib/symtable.rs | 81 ++++++++++-- 3 files changed, 361 insertions(+), 13 deletions(-) create mode 100644 Lib/test/test_symtable.py diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py new file mode 100644 index 000000000..8e83bc72f --- /dev/null +++ b/Lib/test/test_symtable.py @@ -0,0 +1,251 @@ +""" +Test the API of the symtable module. +""" +import symtable +import unittest + + + +TEST_CODE = """ +import sys + +glob = 42 +some_var = 12 + +class Mine: + instance_var = 24 + def a_method(p1, p2): + pass + +def spam(a, b, *var, **kw): + global bar + bar = 47 + some_var = 10 + x = 23 + glob + def internal(): + return x + def other_internal(): + nonlocal some_var + some_var = 3 + return some_var + return internal + +def foo(): + pass + +def namespace_test(): pass +def namespace_test(): pass +""" + + +def find_block(block, name): + for ch in block.get_children(): + if ch.get_name() == name: + return ch + + +class SymtableTest(unittest.TestCase): + + top = symtable.symtable(TEST_CODE, "?", "exec") + # These correspond to scopes in TEST_CODE + Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") + spam = find_block(top, "spam") + internal = find_block(spam, "internal") + other_internal = find_block(spam, "other_internal") + foo = find_block(top, "foo") + + def test_type(self): + self.assertEqual(self.top.get_type(), "module") + self.assertEqual(self.Mine.get_type(), "class") + self.assertEqual(self.a_method.get_type(), "function") + self.assertEqual(self.spam.get_type(), "function") + self.assertEqual(self.internal.get_type(), "function") + + @unittest.skip("TODO: RUSTPYTHON") + def test_id(self): + self.assertGreater(self.top.get_id(), 0) + self.assertGreater(self.Mine.get_id(), 0) + self.assertGreater(self.a_method.get_id(), 0) + self.assertGreater(self.spam.get_id(), 0) + self.assertGreater(self.internal.get_id(), 0) + + @unittest.skip("TODO: RUSTPYTHON") + def test_optimized(self): + self.assertFalse(self.top.is_optimized()) + + self.assertTrue(self.spam.is_optimized()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_nested(self): + self.assertFalse(self.top.is_nested()) + self.assertFalse(self.Mine.is_nested()) + self.assertFalse(self.spam.is_nested()) + self.assertTrue(self.internal.is_nested()) + + def test_children(self): + self.assertTrue(self.top.has_children()) + self.assertTrue(self.Mine.has_children()) + self.assertFalse(self.foo.has_children()) + + def test_lineno(self): + self.assertEqual(self.top.get_lineno(), 0) + self.assertEqual(self.spam.get_lineno(), 12) + + @unittest.skip("TODO: RUSTPYTHON") + def test_function_info(self): + func = self.spam + self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"]) + expected = ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x'] + self.assertEqual(sorted(func.get_locals()), expected) + self.assertEqual(sorted(func.get_globals()), ["bar", "glob"]) + self.assertEqual(self.internal.get_frees(), ("x",)) + + @unittest.skip("TODO: RUSTPYTHON") + def test_globals(self): + self.assertTrue(self.spam.lookup("glob").is_global()) + self.assertFalse(self.spam.lookup("glob").is_declared_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) + self.assertTrue(self.spam.lookup("bar").is_declared_global()) + self.assertFalse(self.internal.lookup("x").is_global()) + self.assertFalse(self.Mine.lookup("instance_var").is_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_nonlocal(self): + self.assertFalse(self.spam.lookup("some_var").is_nonlocal()) + self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal()) + expected = ("some_var",) + self.assertEqual(self.other_internal.get_nonlocals(), expected) + + def test_local(self): + self.assertTrue(self.spam.lookup("x").is_local()) + self.assertFalse(self.spam.lookup("bar").is_local()) + + def test_free(self): + self.assertTrue(self.internal.lookup("x").is_free()) + + def test_referenced(self): + self.assertTrue(self.internal.lookup("x").is_referenced()) + self.assertTrue(self.spam.lookup("internal").is_referenced()) + self.assertFalse(self.spam.lookup("x").is_referenced()) + + def test_parameters(self): + for sym in ("a", "var", "kw"): + self.assertTrue(self.spam.lookup(sym).is_parameter()) + self.assertFalse(self.spam.lookup("x").is_parameter()) + + def test_symbol_lookup(self): + self.assertEqual(len(self.top.get_identifiers()), + len(self.top.get_symbols())) + + self.assertRaises(KeyError, self.top.lookup, "not_here") + + def test_namespaces(self): + self.assertTrue(self.top.lookup("Mine").is_namespace()) + self.assertTrue(self.Mine.lookup("a_method").is_namespace()) + self.assertTrue(self.top.lookup("spam").is_namespace()) + self.assertTrue(self.spam.lookup("internal").is_namespace()) + self.assertTrue(self.top.lookup("namespace_test").is_namespace()) + self.assertFalse(self.spam.lookup("x").is_namespace()) + + # TODO(RUSTPYTHON): lookup should return same pythonref + # self.assertTrue(self.top.lookup("spam").get_namespace() is self.spam) + ns_test = self.top.lookup("namespace_test") + self.assertEqual(len(ns_test.get_namespaces()), 2) + self.assertRaises(ValueError, ns_test.get_namespace) + + def test_assigned(self): + self.assertTrue(self.spam.lookup("x").is_assigned()) + self.assertTrue(self.spam.lookup("bar").is_assigned()) + self.assertTrue(self.top.lookup("spam").is_assigned()) + self.assertTrue(self.Mine.lookup("a_method").is_assigned()) + self.assertFalse(self.internal.lookup("x").is_assigned()) + + def test_annotated(self): + st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') + st2 = st1.get_children()[0] + self.assertTrue(st2.lookup('x').is_local()) + self.assertTrue(st2.lookup('x').is_annotated()) + self.assertFalse(st2.lookup('x').is_global()) + st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') + st4 = st3.get_children()[0] + self.assertTrue(st4.lookup('x').is_local()) + self.assertFalse(st4.lookup('x').is_annotated()) + + # Test that annotations in the global scope are valid after the + # variable is declared as nonlocal. + st5 = symtable.symtable('global x\nx: int', 'test', 'exec') + self.assertTrue(st5.lookup("x").is_global()) + + # Test that annotations for nonlocals are valid after the + # variable is declared as nonlocal. + st6 = symtable.symtable('def g():\n' + ' x = 2\n' + ' def f():\n' + ' nonlocal x\n' + ' x: int', + 'test', 'exec') + + def test_imported(self): + self.assertTrue(self.top.lookup("sys").is_imported()) + + def test_name(self): + self.assertEqual(self.top.get_name(), "top") + self.assertEqual(self.spam.get_name(), "spam") + self.assertEqual(self.spam.lookup("x").get_name(), "x") + self.assertEqual(self.Mine.get_name(), "Mine") + + @unittest.skip("TODO: RUSTPYTHON") + def test_class_info(self): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + @unittest.skip("TODO: RUSTPYTHON") + def test_filename_correct(self): + ### Bug tickler: SyntaxError file name correct whether error raised + ### while parsing or building symbol table. + def checkfilename(brokencode, offset): + try: + symtable.symtable(brokencode, "spam", "exec") + except SyntaxError as e: + self.assertEqual(e.filename, "spam") + self.assertEqual(e.lineno, 1) + self.assertEqual(e.offset, offset) + else: + self.fail("no SyntaxError for %r" % (brokencode,)) + # TODO: RUSTPYTHON, now offset get 15 + checkfilename("def f(x): foo)(", 14) # parse-time + checkfilename("def f(x): global x", 11) # symtable-build-time + symtable.symtable("pass", b"spam", "exec") + with self.assertWarns(DeprecationWarning), \ + self.assertRaises(TypeError): + symtable.symtable("pass", bytearray(b"spam"), "exec") + with self.assertWarns(DeprecationWarning): + symtable.symtable("pass", memoryview(b"spam"), "exec") + with self.assertRaises(TypeError): + symtable.symtable("pass", list(b"spam"), "exec") + + def test_eval(self): + symbols = symtable.symtable("42", "?", "eval") + + @unittest.skip("TODO: RUSTPYTHON") + def test_single(self): + symbols = symtable.symtable("42", "?", "single") + + def test_exec(self): + symbols = symtable.symtable("def f(x): return x", "?", "exec") + + @unittest.skip("TODO: RUSTPYTHON") + def test_bytes(self): + top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") + self.assertIsNotNone(find_block(top, "Mine")) + + code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n' + + top = symtable.symtable(code, "?", "exec") + self.assertIsNotNone(find_block(top, "\u017d")) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 5714367bb..f2f0aabd1 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -97,10 +97,13 @@ pub struct Symbol { pub name: String, // pub table: SymbolTableRef, pub scope: SymbolScope, + // TODO(lynskylate@gmail.com): Use bitflags replace pub is_referenced: bool, pub is_assigned: bool, pub is_parameter: bool, pub is_free: bool, + pub is_annotated: bool, + pub is_imported: bool, // indicates if the symbol gets a value assigned by a named expression in a comprehension // this is required to correct the scope in the analysis. @@ -121,6 +124,8 @@ impl Symbol { is_assigned: false, is_parameter: false, is_free: false, + is_annotated: false, + is_imported: false, is_assign_namedexpr_in_comprehension: false, is_iter: false, } @@ -276,6 +281,11 @@ impl<'a> SymbolTableAnalyzer<'a> { fn analyze_unknown_symbol(&self, symbol: &mut Symbol) { if symbol.is_assigned || symbol.is_parameter { symbol.scope = SymbolScope::Local; + } else if !symbol.is_assigned && symbol.is_referenced { + // if symbol is referenced in its block, but not assigned to. + // https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.Symbol.is_free + symbol.scope = SymbolScope::Unknown; + symbol.is_free = true; } else { // Interesting stuff about the __class__ variable: // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object @@ -394,7 +404,10 @@ enum SymbolUsage { Nonlocal, Used, Assigned, + Imported, + AnnotationAssigned, Parameter, + AnnotationParameter, AssignedNamedExprInCompr, Iter, } @@ -460,7 +473,11 @@ impl SymbolTableBuilder { } fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { - self.register_name(¶meter.arg, SymbolUsage::Parameter) + if let Some(_annotation) = ¶meter.annotation { + self.register_name(¶meter.arg, SymbolUsage::AnnotationParameter) + } else { + self.register_name(¶meter.arg, SymbolUsage::Parameter) + } } fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { @@ -564,12 +581,12 @@ impl SymbolTableBuilder { for name in names { if let Some(alias) = &name.alias { // `import mymodule as myalias` - self.register_name(alias, SymbolUsage::Assigned)?; + self.register_name(alias, SymbolUsage::Imported)?; } else { // `import module` self.register_name( name.symbol.split('.').next().unwrap(), - SymbolUsage::Assigned, + SymbolUsage::Imported, )?; } } @@ -601,7 +618,12 @@ impl SymbolTableBuilder { annotation, value, } => { - self.scan_expression(target, &ExpressionContext::Store)?; + // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233 + if let ast::ExpressionType::Identifier { ref name } = target.node { + self.register_name(name, SymbolUsage::AnnotationAssigned)?; + } else { + self.scan_expression(target, &ExpressionContext::Store)?; + } self.scan_expression(annotation, &ExpressionContext::Load)?; if let Some(value) = value { self.scan_expression(value, &ExpressionContext::Load)?; @@ -984,9 +1006,21 @@ impl SymbolTableBuilder { }); } } + SymbolUsage::Imported => { + symbol.is_assigned = true; + symbol.is_imported = true; + } SymbolUsage::Parameter => { symbol.is_parameter = true; } + SymbolUsage::AnnotationParameter => { + symbol.is_parameter = true; + symbol.is_annotated = true; + } + SymbolUsage::AnnotationAssigned => { + symbol.is_assigned = true; + symbol.is_annotated = true; + } SymbolUsage::Assigned => { symbol.is_assigned = true; } diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 0d90b0bdb..f52a3888d 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -109,10 +109,17 @@ impl PySymbolTable { if let Some(symbol) = self.symtable.symbols.get(name) { Ok(PySymbol { symbol: symbol.clone(), + namespaces: self + .symtable + .sub_tables + .iter() + .filter(|table| table.name == name) + .cloned() + .collect(), } .into_ref(vm)) } else { - Err(vm.new_lookup_error(name.to_owned())) + Err(vm.new_key_error(vm.new_str(format!("lookup {} failed", name)))) } } @@ -129,12 +136,23 @@ impl PySymbolTable { #[pymethod(name = "get_symbols")] fn get_symbols(&self, vm: &VirtualMachine) -> PyResult { - let symbols = self - .symtable - .symbols - .values() - .map(|s| (PySymbol { symbol: s.clone() }).into_ref(vm).into_object()) - .collect(); + let mut symbols: Vec = vec![]; + for symbol in self.symtable.symbols.values() { + symbols.push( + PySymbol { + symbol: symbol.clone(), + namespaces: self + .symtable + .sub_tables + .iter() + .filter(|&table| table.name == symbol.name) + .cloned() + .collect(), + } + .into_ref(vm) + .into_object(), + ) + } Ok(vm.ctx.new_list(symbols)) } @@ -158,6 +176,7 @@ impl PySymbolTable { #[pyclass(name = "Symbol")] struct PySymbol { symbol: symboltable::Symbol, + namespaces: Vec, } impl fmt::Debug for PySymbol { @@ -189,6 +208,25 @@ impl PySymbol { self.symbol.is_local() } + #[pymethod(name = "is_imported")] + fn is_imported(&self) -> bool { + self.symbol.is_imported + } + + #[pymethod(name = "is_nested")] + fn is_nested(&self) -> bool { + // TODO + false + } + + #[pymethod(name = "is_nonlocal")] + fn is_nonlocal(&self) -> bool { + match self.symbol.scope { + symboltable::SymbolScope::Nonlocal => true, + _ => false, + } + } + #[pymethod(name = "is_referenced")] fn is_referenced(&self) -> bool { self.symbol.is_referenced @@ -211,7 +249,32 @@ impl PySymbol { #[pymethod(name = "is_namespace")] fn is_namespace(&self) -> bool { - // TODO - false + self.namespaces.len() > 0 + } + + #[pymethod(name = "is_annotated")] + fn is_annotated(&self) -> bool { + self.symbol.is_annotated + } + + #[pymethod(name = "get_namespaces")] + fn get_namespaces(&self, vm: &VirtualMachine) -> PyResult { + let namespaces = self + .namespaces + .iter() + .map(|table| to_py_symbol_table(table.clone()).into_ref(vm).into_object()) + .collect(); + Ok(vm.ctx.new_list(namespaces)) + } + + #[pymethod(name = "get_namespace")] + fn get_namespace(&self, vm: &VirtualMachine) -> PyResult { + if self.namespaces.len() != 1 { + Err(vm.new_value_error("namespace is bound to multiple namespaces".to_owned())) + } else { + Ok(to_py_symbol_table(self.namespaces.first().unwrap().clone()) + .into_ref(vm) + .into_object()) + } } } From 2dbab05837e4b0fc98bf983eb1ee9739e8799d8c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 31 Jul 2020 03:55:42 +0900 Subject: [PATCH 144/209] Windows build note for failing part --- vm/src/frozen.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index 2629ee449..379e90b8e 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -18,6 +18,7 @@ pub fn get_module_inits() -> HashMap { // Python modules that the vm calls into, but are not actually part of the stdlib. They could // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. // Includes _importlib_bootstrap and _importlib_bootstrap_external + // For Windows: did you forget to run `powershell scripts\symlinks-to-hardlinks.ps1`? ext_modules!(dir = "Lib/python_builtins/"); #[cfg(not(feature = "freeze-stdlib"))] From f754a4077e5cffde5e38810316f27a0d96da9034 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Thu, 30 Jul 2020 14:02:21 -0700 Subject: [PATCH 145/209] Added tests for sys.executable --- Lib/test/test_sys_executable.py | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Lib/test/test_sys_executable.py diff --git a/Lib/test/test_sys_executable.py b/Lib/test/test_sys_executable.py new file mode 100644 index 000000000..7aacdb026 --- /dev/null +++ b/Lib/test/test_sys_executable.py @@ -0,0 +1,62 @@ +import unittest +import subprocess +import sys +import os + +# These should go into a test_sys.py file, but given these are the only +# ones being tested currently that seems unnecessary. + + +class SysExecutableTest(unittest.TestCase): + + # This is a copy of test.test_sys.SysModuleTest.test_executable from cpython. + def cpython_tests(self): + # sys.executable should be absolute + self.assertEqual(os.path.abspath(sys.executable), sys.executable) + + # Issue #7774: Ensure that sys.executable is an empty string if argv[0] + # has been set to a non existent program name and Python is unable to + # retrieve the real program name + + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + python_dir = os.path.dirname(os.path.realpath(sys.executable)) + p = subprocess.Popen( + [ + "nonexistent", + "-c", + 'import sys; print(sys.executable.encode("ascii", "backslashreplace"))', + ], + executable=sys.executable, + stdout=subprocess.PIPE, + cwd=python_dir, + ) + stdout = p.communicate()[0] + executable = stdout.strip().decode("ASCII") + p.wait() + self.assertIn( + executable, + ["b''", repr(sys.executable.encode("ascii", "backslashreplace"))], + ) + + def test_no_follow_symlink(self): + paths = [os.path.abspath("test_symlink"), "./test_symlink"] + for path in paths: + with self.subTest(path=path): + os.symlink(sys.executable, path) + command = [ + path, + "-c", + "import sys; print(sys.executable, end='')", + ] + try: + process = subprocess.run(command, capture_output=True) + finally: + os.remove(path) + self.assertEqual( + os.path.abspath(path), process.stdout.decode("utf-8") + ) + + +if __name__ == "__main__": + unittest.main() From 141eba5be2fac539e629431dd619120fb574c413 Mon Sep 17 00:00:00 2001 From: Basix Date: Sat, 1 Aug 2020 00:14:37 +0900 Subject: [PATCH 146/209] Implement msvcrt.SetErrorMode --- vm/src/stdlib/msvcrt.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 3a5e4568d..9d73063e0 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -4,6 +4,9 @@ use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; +use winapi::shared::minwindef::UINT; +use winapi::um::errhandlingapi::SetErrorMode; + use itertools::Itertools; extern "C" { @@ -72,7 +75,16 @@ fn msvcrt_open_osfhandle(handle: isize, flags: i32, vm: &VirtualMachine) -> PyRe } } +fn msvcrt_seterrormode(mode: UINT, _: &VirtualMachine) -> UINT { + unsafe { suppress_iph!(SetErrorMode(mode)) } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + use winapi::um::winbase::{ + SEM_FAILCRITICALERRORS, SEM_NOALIGNMENTFAULTEXCEPT, SEM_NOGPFAULTERRORBOX, + SEM_NOOPENFILEERRORBOX, + }; + let ctx = &vm.ctx; py_module!(vm, "msvcrt", { "getch" => ctx.new_function(msvcrt_getch), @@ -83,5 +95,10 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "putwch" => ctx.new_function(msvcrt_putwch), "setmode" => ctx.new_function(msvcrt_setmode), "open_osfhandle" => ctx.new_function(msvcrt_open_osfhandle), + "SetErrorMode" => ctx.new_function(msvcrt_seterrormode), + "SEM_FAILCRITICALERRORS" => ctx.new_int(SEM_FAILCRITICALERRORS), + "SEM_NOALIGNMENTFAULTEXCEPT" => ctx.new_int(SEM_NOALIGNMENTFAULTEXCEPT), + "SEM_NOGPFAULTERRORBOX" => ctx.new_int(SEM_NOGPFAULTERRORBOX), + "SEM_NOOPENFILEERRORBOX" => ctx.new_int(SEM_NOOPENFILEERRORBOX), }) } From 12df480cc9bef173b0e269cd63006e3c530cfd81 Mon Sep 17 00:00:00 2001 From: Yiqun Ling Date: Sat, 1 Aug 2020 01:29:11 +0800 Subject: [PATCH 147/209] Apply suggestions from code review Co-authored-by: Jeong YunWon --- compiler/src/symboltable.rs | 9 +++++---- vm/src/stdlib/symtable.rs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index f2f0aabd1..811fd48b6 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -473,11 +473,12 @@ impl SymbolTableBuilder { } fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { - if let Some(_annotation) = ¶meter.annotation { - self.register_name(¶meter.arg, SymbolUsage::AnnotationParameter) + let usage = if parameter.annotation.is_some() { + SymbolUsage::AnnotationParamater } else { - self.register_name(¶meter.arg, SymbolUsage::Parameter) - } + SymbolUsage::Parameter + }; + self.regsiter_name(¶meter.arg, usage) } fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index f52a3888d..47359fa23 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -249,7 +249,7 @@ impl PySymbol { #[pymethod(name = "is_namespace")] fn is_namespace(&self) -> bool { - self.namespaces.len() > 0 + self.namespaces.is_empty() } #[pymethod(name = "is_annotated")] From 0d83cab37a237d0680263942fe151e6abfd1b01f Mon Sep 17 00:00:00 2001 From: lynskylate Date: Sat, 1 Aug 2020 02:05:44 +0800 Subject: [PATCH 148/209] Optimize code --- compiler/src/symboltable.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 811fd48b6..79e0b2bed 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -97,7 +97,7 @@ pub struct Symbol { pub name: String, // pub table: SymbolTableRef, pub scope: SymbolScope, - // TODO(lynskylate@gmail.com): Use bitflags replace + // TODO: Use bitflags replace pub is_referenced: bool, pub is_assigned: bool, pub is_parameter: bool, @@ -474,11 +474,11 @@ impl SymbolTableBuilder { fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { let usage = if parameter.annotation.is_some() { - SymbolUsage::AnnotationParamater + SymbolUsage::AnnotationParameter } else { SymbolUsage::Parameter }; - self.regsiter_name(¶meter.arg, usage) + self.register_name(¶meter.arg, usage) } fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { From 58990cbbaf6510ac1ff89e0956de344f371c7ca3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 1 Aug 2020 04:39:26 +0900 Subject: [PATCH 149/209] Apply suggestions from code review --- vm/src/stdlib/msvcrt.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 9d73063e0..5f05bef97 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -6,7 +6,6 @@ use crate::VirtualMachine; use winapi::shared::minwindef::UINT; use winapi::um::errhandlingapi::SetErrorMode; - use itertools::Itertools; extern "C" { From 5c9de57ed99fc41037f12f818e7f2cf17c814e2d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 1 Aug 2020 04:40:12 +0900 Subject: [PATCH 150/209] Apply suggestions from code review --- vm/src/stdlib/msvcrt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 5f05bef97..0695bbb69 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -4,9 +4,9 @@ use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; +use itertools::Itertools; use winapi::shared::minwindef::UINT; use winapi::um::errhandlingapi::SetErrorMode; -use itertools::Itertools; extern "C" { fn _getch() -> i32; From 24eb1185ddc62d6c97bf7e47e3f9c4af92b04c39 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Fri, 31 Jul 2020 16:36:11 -0700 Subject: [PATCH 151/209] Made exceptions write to stderr instead of stdout --- vm/src/exceptions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 7f5d4d242..be9833224 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -148,9 +148,9 @@ impl PyBaseException { /// Print exception chain pub fn print_exception(vm: &VirtualMachine, exc: &PyBaseExceptionRef) { - let stdout = io::stdout(); - let mut stdout = stdout.lock(); - let _ = write_exception(&mut stdout, vm, exc); + let stderr = io::stderr(); + let mut stderr = stderr.lock(); + let _ = write_exception(&mut stderr, vm, exc); } pub fn write_exception( From 9a9a24d5f0c69cfde3614e26ea64f75286c1bf59 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Sun, 26 Jul 2020 11:10:52 -0700 Subject: [PATCH 152/209] venv module and tests The venv module was taken from CPython 3.9.0 Beta 5. --- Lib/test/test_venv.py | 541 +++++++++++++++++++++++++++ Lib/venv/__init__.py | 499 ++++++++++++++++++++++++ Lib/venv/__main__.py | 10 + Lib/venv/scripts/common/Activate.ps1 | 241 ++++++++++++ Lib/venv/scripts/common/activate | 66 ++++ Lib/venv/scripts/nt/activate.bat | 33 ++ Lib/venv/scripts/nt/deactivate.bat | 21 ++ Lib/venv/scripts/posix/activate.csh | 25 ++ Lib/venv/scripts/posix/activate.fish | 64 ++++ 9 files changed, 1500 insertions(+) create mode 100644 Lib/test/test_venv.py create mode 100644 Lib/venv/__init__.py create mode 100644 Lib/venv/__main__.py create mode 100644 Lib/venv/scripts/common/Activate.ps1 create mode 100644 Lib/venv/scripts/common/activate create mode 100644 Lib/venv/scripts/nt/activate.bat create mode 100644 Lib/venv/scripts/nt/deactivate.bat create mode 100644 Lib/venv/scripts/posix/activate.csh create mode 100644 Lib/venv/scripts/posix/activate.fish diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py new file mode 100644 index 000000000..ca003d55d --- /dev/null +++ b/Lib/test/test_venv.py @@ -0,0 +1,541 @@ +""" +Test harness for the venv module. + +Copyright (C) 2011-2012 Vinay Sajip. +Licensed to the PSF under a contributor agreement. +""" + +import ensurepip +import os +import os.path +import re +import shutil +import struct +import subprocess +import sys +import tempfile +from test.support import (captured_stdout, captured_stderr, requires_zlib, + can_symlink, EnvironmentVarGuard, rmtree, + import_module, + skip_if_broken_multiprocessing_synchronize) +import unittest +import venv +from unittest.mock import patch + +try: + import ctypes +except ImportError: + ctypes = None + +# Platforms that set sys._base_executable can create venvs from within +# another venv, so no need to skip tests that require venv.create(). +requireVenvCreate = unittest.skipUnless( + sys.prefix == sys.base_prefix + or sys._base_executable != sys.executable, + 'cannot run venv.create from within a venv on this platform') + +def check_output(cmd, encoding=None): + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding=encoding) + out, err = p.communicate() + if p.returncode: + raise subprocess.CalledProcessError( + p.returncode, cmd, out, err) + return out, err + +class BaseTest(unittest.TestCase): + """Base class for venv tests.""" + maxDiff = 80 * 50 + + def setUp(self): + self.env_dir = os.path.realpath(tempfile.mkdtemp()) + if os.name == 'nt': + self.bindir = 'Scripts' + self.lib = ('Lib',) + self.include = 'Include' + else: + self.bindir = 'bin' + self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.include = 'include' + executable = sys._base_executable + self.exe = os.path.split(executable)[-1] + if (sys.platform == 'win32' + and os.path.lexists(executable) + and not os.path.exists(executable)): + self.cannot_link_exe = True + else: + self.cannot_link_exe = False + + def tearDown(self): + rmtree(self.env_dir) + + def run_with_capture(self, func, *args, **kwargs): + with captured_stdout() as output: + with captured_stderr() as error: + func(*args, **kwargs) + return output.getvalue(), error.getvalue() + + def get_env_file(self, *args): + return os.path.join(self.env_dir, *args) + + def get_text_file_contents(self, *args, encoding='utf-8'): + with open(self.get_env_file(*args), 'r', encoding=encoding) as f: + result = f.read() + return result + +class BasicTest(BaseTest): + """Test venv module functionality.""" + + def isdir(self, *args): + fn = self.get_env_file(*args) + self.assertTrue(os.path.isdir(fn)) + + def test_defaults(self): + """ + Test the create function with default arguments. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + self.isdir(self.bindir) + self.isdir(self.include) + self.isdir(*self.lib) + # Issue 21197 + p = self.get_env_file('lib64') + conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and + (sys.platform != 'darwin')) + if conditions: + self.assertTrue(os.path.islink(p)) + else: + self.assertFalse(os.path.exists(p)) + data = self.get_text_file_contents('pyvenv.cfg') + executable = sys._base_executable + path = os.path.dirname(executable) + self.assertIn('home = %s' % path, data) + fn = self.get_env_file(self.bindir, self.exe) + if not os.path.exists(fn): # diagnostics for Windows buildbot failures + bd = self.get_env_file(self.bindir) + print('Contents of %r:' % bd) + print(' %r' % os.listdir(bd)) + self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + + def test_prompt(self): + env_name = os.path.split(self.env_dir)[1] + + rmtree(self.env_dir) + builder = venv.EnvBuilder() + self.run_with_capture(builder.create, self.env_dir) + context = builder.ensure_directories(self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertNotIn("prompt = ", data) + + rmtree(self.env_dir) + builder = venv.EnvBuilder(prompt='My prompt') + self.run_with_capture(builder.create, self.env_dir) + context = builder.ensure_directories(self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + self.assertEqual(context.prompt, '(My prompt) ') + self.assertIn("prompt = 'My prompt'\n", data) + + rmtree(self.env_dir) + builder = venv.EnvBuilder(prompt='.') + cwd = os.path.basename(os.getcwd()) + self.run_with_capture(builder.create, self.env_dir) + context = builder.ensure_directories(self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertIn("prompt = '%s'\n" % cwd, data) + + def test_upgrade_dependencies(self): + builder = venv.EnvBuilder() + bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + python_exe = 'python.exe' if sys.platform == 'win32' else 'python' + with tempfile.TemporaryDirectory() as fake_env_dir: + + def pip_cmd_checker(cmd): + self.assertEqual( + cmd, + [ + os.path.join(fake_env_dir, bin_path, python_exe), + '-m', + 'pip', + 'install', + '--upgrade', + 'pip', + 'setuptools' + ] + ) + + fake_context = builder.ensure_directories(fake_env_dir) + with patch('venv.subprocess.check_call', pip_cmd_checker): + builder.upgrade_dependencies(fake_context) + + @requireVenvCreate + def test_prefixes(self): + """ + Test that the prefix values are as expected. + """ + # check a venv's prefixes + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + cmd = [envpy, '-c', None] + for prefix, expected in ( + ('prefix', self.env_dir), + ('exec_prefix', self.env_dir), + ('base_prefix', sys.base_prefix), + ('base_exec_prefix', sys.base_exec_prefix)): + cmd[2] = 'import sys; print(sys.%s)' % prefix + out, err = check_output(cmd) + self.assertEqual(out.strip(), expected.encode()) + + if sys.platform == 'win32': + ENV_SUBDIRS = ( + ('Scripts',), + ('Include',), + ('Lib',), + ('Lib', 'site-packages'), + ) + else: + ENV_SUBDIRS = ( + ('bin',), + ('include',), + ('lib',), + ('lib', 'python%d.%d' % sys.version_info[:2]), + ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'), + ) + + def create_contents(self, paths, filename): + """ + Create some files in the environment which are unrelated + to the virtual environment. + """ + for subdirs in paths: + d = os.path.join(self.env_dir, *subdirs) + os.mkdir(d) + fn = os.path.join(d, filename) + with open(fn, 'wb') as f: + f.write(b'Still here?') + + def test_overwrite_existing(self): + """ + Test creating environment in an existing directory. + """ + self.create_contents(self.ENV_SUBDIRS, 'foo') + venv.create(self.env_dir) + for subdirs in self.ENV_SUBDIRS: + fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) + self.assertTrue(os.path.exists(fn)) + with open(fn, 'rb') as f: + self.assertEqual(f.read(), b'Still here?') + + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + for subdirs in self.ENV_SUBDIRS: + fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) + self.assertFalse(os.path.exists(fn)) + + def clear_directory(self, path): + for fn in os.listdir(path): + fn = os.path.join(path, fn) + if os.path.islink(fn) or os.path.isfile(fn): + os.remove(fn) + elif os.path.isdir(fn): + rmtree(fn) + + def test_unoverwritable_fails(self): + #create a file clashing with directories in the env dir + for paths in self.ENV_SUBDIRS[:3]: + fn = os.path.join(self.env_dir, *paths) + with open(fn, 'wb') as f: + f.write(b'') + self.assertRaises((ValueError, OSError), venv.create, self.env_dir) + self.clear_directory(self.env_dir) + + def test_upgrade(self): + """ + Test upgrading an existing environment directory. + """ + # See Issue #21643: the loop needs to run twice to ensure + # that everything works on the upgrade (the first run just creates + # the venv). + for upgrade in (False, True): + builder = venv.EnvBuilder(upgrade=upgrade) + self.run_with_capture(builder.create, self.env_dir) + self.isdir(self.bindir) + self.isdir(self.include) + self.isdir(*self.lib) + fn = self.get_env_file(self.bindir, self.exe) + if not os.path.exists(fn): + # diagnostics for Windows buildbot failures + bd = self.get_env_file(self.bindir) + print('Contents of %r:' % bd) + print(' %r' % os.listdir(bd)) + self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + + def test_isolation(self): + """ + Test isolation from system site-packages + """ + for ssp, s in ((True, 'true'), (False, 'false')): + builder = venv.EnvBuilder(clear=True, system_site_packages=ssp) + builder.create(self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + self.assertIn('include-system-site-packages = %s\n' % s, data) + + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_symlinking(self): + """ + Test symlinking works as expected + """ + for usl in (False, True): + builder = venv.EnvBuilder(clear=True, symlinks=usl) + builder.create(self.env_dir) + fn = self.get_env_file(self.bindir, self.exe) + # Don't test when False, because e.g. 'python' is always + # symlinked to 'python3.3' in the env, even when symlinking in + # general isn't wanted. + if usl: + if self.cannot_link_exe: + # Symlinking is skipped when our executable is already a + # special app symlink + self.assertFalse(os.path.islink(fn)) + else: + self.assertTrue(os.path.islink(fn)) + + # If a venv is created from a source build and that venv is used to + # run the test, the pyvenv.cfg in the venv created in the test will + # point to the venv being used to run the test, and we lose the link + # to the source build - so Python can't initialise properly. + @requireVenvCreate + def test_executable(self): + """ + Test that the sys.executable value is as expected. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'import sys; print(sys.executable)']) + self.assertEqual(out.strip(), envpy.encode()) + + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_executable_symlinks(self): + """ + Test that the sys.executable value is as expected. + """ + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True, symlinks=True) + builder.create(self.env_dir) + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'import sys; print(sys.executable)']) + self.assertEqual(out.strip(), envpy.encode()) + + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_unicode_in_batch_file(self): + """ + Test handling of Unicode paths + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ') + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + envpy = os.path.join(env_dir, self.bindir, self.exe) + out, err = check_output( + [activate, '&', self.exe, '-c', 'print(0)'], + encoding='oem', + ) + self.assertEqual(out.strip(), '0') + + @requireVenvCreate + def test_multiprocessing(self): + """ + Test that the multiprocessing is able to spawn. + """ + # bpo-36342: Instantiation of a Pool object imports the + # multiprocessing.synchronize module. Skip the test if this module + # cannot be imported. + skip_if_broken_multiprocessing_synchronize() + + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'from multiprocessing import Pool; ' + 'pool = Pool(1); ' + 'print(pool.apply_async("Python".lower).get(3)); ' + 'pool.terminate()']) + self.assertEqual(out.strip(), "python".encode()) + + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_strict.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + f"source {activate}\n" + "deactivate\n") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + + + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') + def test_macos_env(self): + rmtree(self.env_dir) + builder = venv.EnvBuilder() + builder.create(self.env_dir) + + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) + self.assertEqual(out.strip(), 'False'.encode()) + +@requireVenvCreate +class EnsurePipTest(BaseTest): + """Test venv module installation of pip.""" + def assert_pip_not_installed(self): + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'try:\n import pip\nexcept ImportError:\n print("OK")']) + # We force everything to text, so unittest gives the detailed diff + # if we get unexpected results + err = err.decode("latin-1") # Force to text, prevent decoding errors + self.assertEqual(err, "") + out = out.decode("latin-1") # Force to text, prevent decoding errors + self.assertEqual(out.strip(), "OK") + + + def test_no_pip_by_default(self): + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + self.assert_pip_not_installed() + + def test_explicit_no_pip(self): + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + self.assert_pip_not_installed() + + def test_devnull(self): + # Fix for issue #20053 uses os.devnull to force a config file to + # appear empty. However http://bugs.python.org/issue20541 means + # that doesn't currently work properly on Windows. Once that is + # fixed, the "win_location" part of test_with_pip should be restored + with open(os.devnull, "rb") as f: + self.assertEqual(f.read(), b"") + + self.assertTrue(os.path.exists(os.devnull)) + + def do_test_with_pip(self, system_site_packages): + rmtree(self.env_dir) + with EnvironmentVarGuard() as envvars: + # pip's cross-version compatibility may trigger deprecation + # warnings in current versions of Python. Ensure related + # environment settings don't cause venv to fail. + envvars["PYTHONWARNINGS"] = "e" + # ensurepip is different enough from a normal pip invocation + # that we want to ensure it ignores the normal pip environment + # variable settings. We set PIP_NO_INSTALL here specifically + # to check that ensurepip (and hence venv) ignores it. + # See http://bugs.python.org/issue19734 + envvars["PIP_NO_INSTALL"] = "1" + # Also check that we ignore the pip configuration file + # See http://bugs.python.org/issue20053 + with tempfile.TemporaryDirectory() as home_dir: + envvars["HOME"] = home_dir + bad_config = "[global]\nno-install=1" + # Write to both config file names on all platforms to reduce + # cross-platform variation in test code behaviour + win_location = ("pip", "pip.ini") + posix_location = (".pip", "pip.conf") + # Skips win_location due to http://bugs.python.org/issue20541 + for dirname, fname in (posix_location,): + dirpath = os.path.join(home_dir, dirname) + os.mkdir(dirpath) + fpath = os.path.join(dirpath, fname) + with open(fpath, 'w') as f: + f.write(bad_config) + + # Actually run the create command with all that unhelpful + # config in place to ensure we ignore it + try: + self.run_with_capture(venv.create, self.env_dir, + system_site_packages=system_site_packages, + with_pip=True) + except subprocess.CalledProcessError as exc: + # The output this produces can be a little hard to read, + # but at least it has all the details + details = exc.output.decode(errors="replace") + msg = "{}\n\n**Subprocess Output**\n{}" + self.fail(msg.format(exc, details)) + # Ensure pip is available in the virtual environment + envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) + # Ignore DeprecationWarning since pip code is not part of Python + out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I', + '-m', 'pip', '--version']) + # We force everything to text, so unittest gives the detailed diff + # if we get unexpected results + err = err.decode("latin-1") # Force to text, prevent decoding errors + self.assertEqual(err, "") + out = out.decode("latin-1") # Force to text, prevent decoding errors + expected_version = "pip {}".format(ensurepip.version()) + self.assertEqual(out[:len(expected_version)], expected_version) + env_dir = os.fsencode(self.env_dir).decode("latin-1") + self.assertIn(env_dir, out) + + # http://bugs.python.org/issue19728 + # Check the private uninstall command provided for the Windows + # installers works (at least in a virtual environment) + with EnvironmentVarGuard() as envvars: + out, err = check_output([envpy, + '-W', 'ignore::DeprecationWarning', '-I', + '-m', 'ensurepip._uninstall']) + # We force everything to text, so unittest gives the detailed diff + # if we get unexpected results + err = err.decode("latin-1") # Force to text, prevent decoding errors + # Ignore the warning: + # "The directory '$HOME/.cache/pip/http' or its parent directory + # is not owned by the current user and the cache has been disabled. + # Please check the permissions and owner of that directory. If + # executing pip with sudo, you may want sudo's -H flag." + # where $HOME is replaced by the HOME environment variable. + err = re.sub("^(WARNING: )?The directory .* or its parent directory " + "is not owned or is not writable by the current user.*$", "", + err, flags=re.MULTILINE) + self.assertEqual(err.rstrip(), "") + # Being fairly specific regarding the expected behaviour for the + # initial bundling phase in Python 3.4. If the output changes in + # future pip versions, this test can likely be relaxed further. + out = out.decode("latin-1") # Force to text, prevent decoding errors + self.assertIn("Successfully uninstalled pip", out) + self.assertIn("Successfully uninstalled setuptools", out) + # Check pip is now gone from the virtual environment. This only + # applies in the system_site_packages=False case, because in the + # other case, pip may still be available in the system site-packages + if not system_site_packages: + self.assert_pip_not_installed() + + # Issue #26610: pip/pep425tags.py requires ctypes + @unittest.skipUnless(ctypes, 'pip requires ctypes') + @requires_zlib() + def test_with_pip(self): + self.do_test_with_pip(False) + self.do_test_with_pip(True) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py new file mode 100644 index 000000000..8009deb3e --- /dev/null +++ b/Lib/venv/__init__.py @@ -0,0 +1,499 @@ +""" +Virtual environment (venv) package for Python. Based on PEP 405. + +Copyright (C) 2011-2014 Vinay Sajip. +Licensed to the PSF under a contributor agreement. +""" +import logging +import os +import shutil +import subprocess +import sys +import sysconfig +import types + + +CORE_VENV_DEPS = ('pip', 'setuptools') +logger = logging.getLogger(__name__) + + +class EnvBuilder: + """ + This class exists to allow virtual environment creation to be + customized. The constructor parameters determine the builder's + behaviour when called upon to create a virtual environment. + + By default, the builder makes the system (global) site-packages dir + *un*available to the created environment. + + If invoked using the Python -m option, the default is to use copying + on Windows platforms but symlinks elsewhere. If instantiated some + other way, the default is to *not* use symlinks. + + :param system_site_packages: If True, the system (global) site-packages + dir is available to created environments. + :param clear: If True, delete the contents of the environment directory if + it already exists, before environment creation. + :param symlinks: If True, attempt to symlink rather than copy files into + virtual environment. + :param upgrade: If True, upgrade an existing virtual environment. + :param with_pip: If True, ensure pip is installed in the virtual + environment + :param prompt: Alternative terminal prefix for the environment. + :param upgrade_deps: Update the base venv modules to the latest on PyPI + """ + + def __init__(self, system_site_packages=False, clear=False, + symlinks=False, upgrade=False, with_pip=False, prompt=None, + upgrade_deps=False): + self.system_site_packages = system_site_packages + self.clear = clear + self.symlinks = symlinks + self.upgrade = upgrade + self.with_pip = with_pip + if prompt == '.': # see bpo-38901 + prompt = os.path.basename(os.getcwd()) + self.prompt = prompt + self.upgrade_deps = upgrade_deps + + def create(self, env_dir): + """ + Create a virtual environment in a directory. + + :param env_dir: The target directory to create an environment in. + + """ + env_dir = os.path.abspath(env_dir) + context = self.ensure_directories(env_dir) + # See issue 24875. We need system_site_packages to be False + # until after pip is installed. + true_system_site_packages = self.system_site_packages + self.system_site_packages = False + self.create_configuration(context) + self.setup_python(context) + if self.with_pip: + self._setup_pip(context) + if not self.upgrade: + self.setup_scripts(context) + self.post_setup(context) + if true_system_site_packages: + # We had set it to False before, now + # restore it and rewrite the configuration + self.system_site_packages = True + self.create_configuration(context) + if self.upgrade_deps: + self.upgrade_dependencies(context) + + def clear_directory(self, path): + for fn in os.listdir(path): + fn = os.path.join(path, fn) + if os.path.islink(fn) or os.path.isfile(fn): + os.remove(fn) + elif os.path.isdir(fn): + shutil.rmtree(fn) + + def ensure_directories(self, env_dir): + """ + Create the directories for the environment. + + Returns a context object which holds paths in the environment, + for use by subsequent logic. + """ + + def create_if_needed(d): + if not os.path.exists(d): + os.makedirs(d) + elif os.path.islink(d) or os.path.isfile(d): + raise ValueError('Unable to create directory %r' % d) + + if os.path.exists(env_dir) and self.clear: + self.clear_directory(env_dir) + context = types.SimpleNamespace() + context.env_dir = env_dir + context.env_name = os.path.split(env_dir)[1] + prompt = self.prompt if self.prompt is not None else context.env_name + context.prompt = '(%s) ' % prompt + create_if_needed(env_dir) + executable = sys._base_executable + dirname, exename = os.path.split(os.path.abspath(executable)) + context.executable = executable + context.python_dir = dirname + context.python_exe = exename + if sys.platform == 'win32': + binname = 'Scripts' + incpath = 'Include' + libpath = os.path.join(env_dir, 'Lib', 'site-packages') + else: + binname = 'bin' + incpath = 'include' + libpath = os.path.join(env_dir, 'lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + context.inc_path = path = os.path.join(env_dir, incpath) + create_if_needed(path) + create_if_needed(libpath) + # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX + if ((sys.maxsize > 2**32) and (os.name == 'posix') and + (sys.platform != 'darwin')): + link_path = os.path.join(env_dir, 'lib64') + if not os.path.exists(link_path): # Issue #21643 + os.symlink('lib', link_path) + context.bin_path = binpath = os.path.join(env_dir, binname) + context.bin_name = binname + context.env_exe = os.path.join(binpath, exename) + create_if_needed(binpath) + return context + + def create_configuration(self, context): + """ + Create a configuration file indicating where the environment's Python + was copied from, and whether the system site-packages should be made + available in the environment. + + :param context: The information for the environment creation request + being processed. + """ + context.cfg_path = path = os.path.join(context.env_dir, 'pyvenv.cfg') + with open(path, 'w', encoding='utf-8') as f: + f.write('home = %s\n' % context.python_dir) + if self.system_site_packages: + incl = 'true' + else: + incl = 'false' + f.write('include-system-site-packages = %s\n' % incl) + f.write('version = %d.%d.%d\n' % sys.version_info[:3]) + if self.prompt is not None: + f.write(f'prompt = {self.prompt!r}\n') + + if os.name != 'nt': + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + """ + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! + if relative_symlinks_ok: + assert os.path.dirname(src) == os.path.dirname(dst) + os.symlink(os.path.basename(src), dst) + else: + os.symlink(src, dst) + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: + shutil.copyfile(src, dst) + else: + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + """ + bad_src = os.path.lexists(src) and not os.path.exists(src) + if self.symlinks and not bad_src and not os.path.islink(dst): + try: + if relative_symlinks_ok: + assert os.path.dirname(src) == os.path.dirname(dst) + os.symlink(os.path.basename(src), dst) + else: + os.symlink(src, dst) + return + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + + # On Windows, we rewrite symlinks to our base python.exe into + # copies of venvlauncher.exe + basename, ext = os.path.splitext(os.path.basename(src)) + srcfn = os.path.join(os.path.dirname(__file__), + "scripts", + "nt", + basename + ext) + # Builds or venv's from builds need to remap source file + # locations, as we do not put them into Lib/venv/scripts + if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): + if basename.endswith('_d'): + ext = '_d' + ext + basename = basename[:-2] + if basename == 'python': + basename = 'venvlauncher' + elif basename == 'pythonw': + basename = 'venvwlauncher' + src = os.path.join(os.path.dirname(src), basename + ext) + else: + src = srcfn + if not os.path.exists(src): + if not bad_src: + logger.warning('Unable to copy %r', src) + return + + shutil.copyfile(src, dst) + + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir + if os.name != 'nt': + copier(context.executable, path) + if not os.path.islink(path): + os.chmod(path, 0o755) + for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + path = os.path.join(binpath, suffix) + if not os.path.exists(path): + # Issue 18807: make copies if + # symlinks are not wanted + copier(context.env_exe, path, relative_symlinks_ok=True) + if not os.path.islink(path): + os.chmod(path, 0o755) + else: + if self.symlinks: + # For symlinking, we need a complete copy of the root directory + # If symlinks fail, you'll get unnecessary copies of files, but + # we assume that if you've opted into symlinks on Windows then + # you know what you're doing. + suffixes = [ + f for f in os.listdir(dirname) if + os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') + ] + if sysconfig.is_python_build(True): + suffixes = [ + f for f in suffixes if + os.path.normcase(f).startswith(('python', 'vcruntime')) + ] + else: + suffixes = ['python.exe', 'python_d.exe', 'pythonw.exe', + 'pythonw_d.exe'] + + for suffix in suffixes: + src = os.path.join(dirname, suffix) + if os.path.lexists(src): + copier(src, os.path.join(binpath, suffix)) + + if sysconfig.is_python_build(True): + # copy init.tcl + for root, dirs, files in os.walk(context.python_dir): + if 'init.tcl' in files: + tcldir = os.path.basename(root) + tcldir = os.path.join(context.env_dir, 'Lib', tcldir) + if not os.path.exists(tcldir): + os.makedirs(tcldir) + src = os.path.join(root, 'init.tcl') + dst = os.path.join(tcldir, 'init.tcl') + shutil.copyfile(src, dst) + break + + def _setup_pip(self, context): + """Installs or upgrades pip in a virtual environment""" + # We run ensurepip in isolated mode to avoid side effects from + # environment vars, the current directory and anything else + # intended for the global Python environment + cmd = [context.env_exe, '-Im', 'ensurepip', '--upgrade', + '--default-pip'] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + + def setup_scripts(self, context): + """ + Set up scripts into the created environment from a directory. + + This method installs the default scripts into the environment + being created. You can prevent the default installation by overriding + this method if you really need to, or if you need to specify + a different location for the scripts to install. By default, the + 'scripts' directory in the venv package is used as the source of + scripts to install. + """ + path = os.path.abspath(os.path.dirname(__file__)) + path = os.path.join(path, 'scripts') + self.install_scripts(context, path) + + def post_setup(self, context): + """ + Hook for post-setup modification of the venv. Subclasses may install + additional packages or scripts here, add activation shell scripts, etc. + + :param context: The information for the environment creation request + being processed. + """ + pass + + def replace_variables(self, text, context): + """ + Replace variable placeholders in script text with context-specific + variables. + + Return the text passed in , but with variables replaced. + + :param text: The text in which to replace placeholder variables. + :param context: The information for the environment creation request + being processed. + """ + text = text.replace('__VENV_DIR__', context.env_dir) + text = text.replace('__VENV_NAME__', context.env_name) + text = text.replace('__VENV_PROMPT__', context.prompt) + text = text.replace('__VENV_BIN_NAME__', context.bin_name) + text = text.replace('__VENV_PYTHON__', context.env_exe) + return text + + def install_scripts(self, context, path): + """ + Install scripts into the created environment from a directory. + + :param context: The information for the environment creation request + being processed. + :param path: Absolute pathname of a directory containing script. + Scripts in the 'common' subdirectory of this directory, + and those in the directory named for the platform + being run on, are installed in the created environment. + Placeholder variables are replaced with environment- + specific values. + """ + binpath = context.bin_path + plen = len(path) + for root, dirs, files in os.walk(path): + if root == path: # at top-level, remove irrelevant dirs + for d in dirs[:]: + if d not in ('common', os.name): + dirs.remove(d) + continue # ignore files in top level + for f in files: + if (os.name == 'nt' and f.startswith('python') + and f.endswith(('.exe', '.pdb'))): + continue + srcfile = os.path.join(root, f) + suffix = root[plen:].split(os.sep)[2:] + if not suffix: + dstdir = binpath + else: + dstdir = os.path.join(binpath, *suffix) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + dstfile = os.path.join(dstdir, f) + with open(srcfile, 'rb') as f: + data = f.read() + if not srcfile.endswith(('.exe', '.pdb')): + try: + data = data.decode('utf-8') + data = self.replace_variables(data, context) + data = data.encode('utf-8') + except UnicodeError as e: + data = None + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + if data is not None: + with open(dstfile, 'wb') as f: + f.write(data) + shutil.copymode(srcfile, dstfile) + + def upgrade_dependencies(self, context): + logger.debug( + f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' + ) + if sys.platform == 'win32': + python_exe = os.path.join(context.bin_path, 'python.exe') + else: + python_exe = os.path.join(context.bin_path, 'python') + cmd = [python_exe, '-m', 'pip', 'install', '--upgrade'] + cmd.extend(CORE_VENV_DEPS) + subprocess.check_call(cmd) + + +def create(env_dir, system_site_packages=False, clear=False, + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): + """Create a virtual environment in a directory.""" + builder = EnvBuilder(system_site_packages=system_site_packages, + clear=clear, symlinks=symlinks, with_pip=with_pip, + prompt=prompt, upgrade_deps=upgrade_deps) + builder.create(env_dir) + +def main(args=None): + compatible = True + if sys.version_info < (3, 3): + compatible = False + elif not hasattr(sys, 'base_prefix'): + compatible = False + if not compatible: + raise ValueError('This script is only for use with Python >= 3.3') + else: + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False + else: + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help='Upgrade core dependencies: {} to the latest ' + 'version in PyPI'.format( + ' '.join(CORE_VENV_DEPS))) + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps) + for d in options.dirs: + builder.create(d) + +if __name__ == '__main__': + rc = 1 + try: + main() + rc = 0 + except Exception as e: + print('Error: %s' % e, file=sys.stderr) + sys.exit(rc) diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py new file mode 100644 index 000000000..912423e4a --- /dev/null +++ b/Lib/venv/__main__.py @@ -0,0 +1,10 @@ +import sys +from . import main + +rc = 1 +try: + main() + rc = 0 +except Exception as e: + print('Error: %s' % e, file=sys.stderr) +sys.exit(rc) diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 new file mode 100644 index 000000000..2fb3852c3 --- /dev/null +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -0,0 +1,241 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate new file mode 100644 index 000000000..45af3536a --- /dev/null +++ b/Lib/venv/scripts/common/activate @@ -0,0 +1,66 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="__VENV_DIR__" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="__VENV_PROMPT__${PS1:-}" + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat new file mode 100644 index 000000000..f61413e23 --- /dev/null +++ b/Lib/venv/scripts/nt/activate.bat @@ -0,0 +1,33 @@ +@echo off + +rem This file is UTF-8 encoded, so we need to update the current code page while executing it +for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do ( + set _OLD_CODEPAGE=%%a +) +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" 65001 > nul +) + +set VIRTUAL_ENV=__VENV_DIR__ + +if not defined PROMPT set PROMPT=$P$G + +if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% +if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% + +set _OLD_VIRTUAL_PROMPT=%PROMPT% +set PROMPT=__VENV_PROMPT__%PROMPT% + +if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% +set PYTHONHOME= + +if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% +if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% + +set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% + +:END +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul + set _OLD_CODEPAGE= +) diff --git a/Lib/venv/scripts/nt/deactivate.bat b/Lib/venv/scripts/nt/deactivate.bat new file mode 100644 index 000000000..313c07911 --- /dev/null +++ b/Lib/venv/scripts/nt/deactivate.bat @@ -0,0 +1,21 @@ +@echo off + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) +set _OLD_VIRTUAL_PROMPT= + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" + set _OLD_VIRTUAL_PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) + +set _OLD_VIRTUAL_PATH= + +set VIRTUAL_ENV= + +:END diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh new file mode 100644 index 000000000..68a0dc74e --- /dev/null +++ b/Lib/venv/scripts/posix/activate.csh @@ -0,0 +1,25 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "__VENV_DIR__" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "__VENV_PROMPT__$prompt" +endif + +alias pydoc python -m pydoc + +rehash diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish new file mode 100644 index 000000000..54b9ea567 --- /dev/null +++ b/Lib/venv/scripts/posix/activate.fish @@ -0,0 +1,64 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "__VENV_DIR__" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end From 0161b9808298099ba769f2774fb97e8dcaa20fbe Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Mon, 27 Jul 2020 09:39:37 -0700 Subject: [PATCH 153/209] Mark expected failures for the venv tests --- Lib/test/support/__init__.py | 25 +++++++++++++++++++++++++ Lib/test/test_venv.py | 7 +++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 175f34945..544d88620 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2568,6 +2568,31 @@ def skip_if_buggy_ucrt_strfptime(test): _buggy_ucrt = False return unittest.skip("buggy MSVC UCRT strptime/strftime")(test) if _buggy_ucrt else test + +def skip_if_broken_multiprocessing_synchronize(): + """ + Skip tests if the multiprocessing.synchronize module is missing, if there + is no available semaphore implementation, or if creating a lock raises an + OSError (on Linux only). + """ + + # Skip tests if the _multiprocessing extension is missing. + import_module('_multiprocessing') + + # Skip tests if there is no available semaphore implementation: + # multiprocessing.synchronize requires _multiprocessing.SemLock. + synchronize = import_module('multiprocessing.synchronize') + + if sys.platform == "linux": + try: + # bpo-38377: On Linux, creating a semaphore fails with OSError + # if the current user does not have the permission to create + # a file in /dev/shm/ directory. + synchronize.Lock(ctx=None) + except OSError as exc: + raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") + + class PythonSymlink: """Creates a symlink for the current Python executable""" def __init__(self, link=None): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index ca003d55d..98343b6bc 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -5,7 +5,8 @@ Copyright (C) 2011-2012 Vinay Sajip. Licensed to the PSF under a contributor agreement. """ -import ensurepip +# pip isn't working yet +# import ensurepip import os import os.path import re @@ -531,8 +532,10 @@ class EnsurePipTest(BaseTest): self.assert_pip_not_installed() # Issue #26610: pip/pep425tags.py requires ctypes + # TODO: RUSTPYTHON @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @requires_zlib + @unittest.expectedFailure def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) From 22dc98091b60b70309c92d7c20dd797a74b4a702 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Fri, 31 Jul 2020 17:26:47 -0700 Subject: [PATCH 154/209] Added graceful handling of creating a venv with pip Currently, RustPython doesn't support the pip module. Normally, when creating a virtual environment, pip is automatically installed. This adds an specific error message when pip is attempted to be installed. --- Lib/venv/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 8009deb3e..d80463762 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -290,6 +290,10 @@ class EnvBuilder: def _setup_pip(self, context): """Installs or upgrades pip in a virtual environment""" + # TODO: RustPython + msg = ("Pip isn't supported yet. To create a virtual environment" + "without pip, call venv with the --without-pip flag.") + raise NotImplementedError(msg) # We run ensurepip in isolated mode to avoid side effects from # environment vars, the current directory and anything else # intended for the global Python environment From 33808e623c9d81172d77568737c81c7bc7fc9c7a Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Fri, 31 Jul 2020 11:33:10 -0700 Subject: [PATCH 155/209] Added support for strings to dis.dis Currently, the dis module's dis function only accepts functions tp decompile. This adds support for disassembling strings. It does this by compiling the string, then disassembling compiled output. --- vm/src/stdlib/dis.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index adc31ff17..eec0ca7bd 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -1,7 +1,9 @@ use crate::bytecode::CodeFlags; use crate::obj::objcode::PyCodeRef; +use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; +use rustpython_compiler::compile; fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { // Method or function: @@ -9,6 +11,15 @@ fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { return dis_disassemble(co, vm); } + // String: + if let Ok(co_str) = PyStringRef::try_from_object(vm, obj.clone()) { + let code = vm + .compile(co_str.as_str(), compile::Mode::Exec, "".to_owned()) + .map_err(|err| vm.new_syntax_error(&err))? + .into_object(); + return dis_disassemble(code, vm); + } + dis_disassemble(obj, vm) } From b066b94196e91eda8628128dd0d1f2f4665eb8e4 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Fri, 31 Jul 2020 13:13:48 -0700 Subject: [PATCH 156/209] Tests for dis module. As of this commit, these tests fail. --- Lib/test/test_dis.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Lib/test/test_dis.py diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py new file mode 100644 index 000000000..7b611ded9 --- /dev/null +++ b/Lib/test/test_dis.py @@ -0,0 +1,30 @@ +import unittest +import dis +from test.support import captured_stdout + + +# This only tests that it prints something in order +# to avoid changing this test if the bytecode changes +class TestDis(unittest.TestCase): + def test_dis(self): + test_cases = (self.test_dis, "x = 2; print(x)") + + for case in test_cases: + with self.subTest(case=case): + with captured_stdout() as stdout: + dis.dis(case) + self.assertNotEqual("", stdout.getvalue()) + + def test_disassemble(self): + # In CPython this would raise an AttributeError, not a + # TypeError because dis is implemented in python in CPython and + # as such the type mismatch wouldn't be caught immeadiately + with self.assertRaises(TypeError): + dis.disassemble(self.test_disassemble) + with captured_stdout() as stdout: + dis.dis(self.test_disassemble.__code__) + self.assertNotEqual("", stdout.getvalue()) + + +if __name__ == "__main__": + unittest.main() From 5ecb288819e08b22ad0dda4a71c4a52f99ff9793 Mon Sep 17 00:00:00 2001 From: Bolun Thompson Date: Fri, 31 Jul 2020 16:21:51 -0700 Subject: [PATCH 157/209] Rewrote dis tests to start a new process to capture stdout (and stderr) This was chaned because stdout is being written to by rust code, which currently can't be redirected by reassigning sys.stdout. This caused the tests to fail because they couldn't see the output of what was being tested. --- Lib/test/test_dis.py | 62 ++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 7b611ded9..8bbba86a4 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,29 +1,59 @@ +import subprocess +import sys import unittest -import dis -from test.support import captured_stdout - # This only tests that it prints something in order # to avoid changing this test if the bytecode changes -class TestDis(unittest.TestCase): - def test_dis(self): - test_cases = (self.test_dis, "x = 2; print(x)") - for case in test_cases: - with self.subTest(case=case): - with captured_stdout() as stdout: - dis.dis(case) - self.assertNotEqual("", stdout.getvalue()) +# These tests start a new process instead of redirecting stdout because +# stdout is being written to by rust code, which currently can't be +# redirected by reassigning sys.stdout + + +class TestDis(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.setup = """ +import dis +def tested_func(): pass +""" + cls.command = (sys.executable, "-c") + + def test_dis(self): + test_code = f""" +{self.setup} +dis.dis(tested_func) +dis.dis("x = 2; print(x)") +""" + + result = subprocess.run( + self.command + (test_code,), capture_output=True + ) + self.assertNotEqual("", result.stdout.decode()) + self.assertEqual("", result.stderr.decode()) def test_disassemble(self): + test_code = f""" +{self.setup} +dis.disassemble(tested_func) +""" + result = subprocess.run( + self.command + (test_code,), capture_output=True + ) # In CPython this would raise an AttributeError, not a # TypeError because dis is implemented in python in CPython and # as such the type mismatch wouldn't be caught immeadiately - with self.assertRaises(TypeError): - dis.disassemble(self.test_disassemble) - with captured_stdout() as stdout: - dis.dis(self.test_disassemble.__code__) - self.assertNotEqual("", stdout.getvalue()) + self.assertIn("TypeError", result.stderr.decode()) + + test_code = f""" +{self.setup} +dis.disassemble(tested_func.__code__) +""" + result = subprocess.run( + self.command + (test_code,), capture_output=True + ) + self.assertNotEqual("", result.stdout.decode()) + self.assertEqual("", result.stderr.decode()) if __name__ == "__main__": From cf9370a240529ee8dfb115f8b23040afd4923f7d Mon Sep 17 00:00:00 2001 From: Basix Date: Fri, 31 Jul 2020 21:56:07 +0900 Subject: [PATCH 158/209] Implement os.waitpid on Windows --- vm/src/stdlib/os.rs | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 02a4dd4aa..3bb7465ba 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1971,7 +1971,6 @@ mod posix { unsafe { libc::WEXITSTATUS(status) } } - // TODO: os.wait[pid] for windows #[pyfunction] fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { let mut status = 0; @@ -2134,6 +2133,8 @@ mod nt { use super::*; pub(super) use std::os::windows::fs::OpenOptionsExt; use std::os::windows::io::RawHandle; + #[cfg(target_env = "msvc")] + use winapi::vc::vcruntime::intptr_t; pub(super) type OpenFlags = u32; @@ -2290,6 +2291,46 @@ mod nt { get_stats().map_err(|e| convert_io_error(vm, e)) } + // cwait is available on MSVC only (according to CPython) + #[cfg(target_env = "msvc")] + extern "C" { + fn _cwait(termstat: *mut i32, procHandle: intptr_t, action: i32) -> intptr_t; + fn _get_errno(pValue: *mut i32) -> i32; + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn waitpid(pid: intptr_t, opt: i32, vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> { + const ECHILD: i32 = 10; + const EINVAL: i32 = 22; + + let mut status = 0; + let pid = unsafe { suppress_iph!(_cwait(&mut status, pid, opt)) }; + if pid == -1 { + let mut errno = 0; + unsafe { _get_errno(&mut errno) }; + match errno { + ECHILD => Err(vm.new_exception_msg( + vm.ctx.exceptions.os_error.clone(), + "ECHILD: No spawned processes".to_owned(), + )), + EINVAL => Err(vm.new_exception_msg( + vm.ctx.exceptions.os_error.clone(), + "EINVAL: Invalid argument".to_owned(), + )), + _ => unreachable!(), + } + } else { + Ok((pid, status << 8)) + } + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn wait(vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> { + waitpid(-1, 0, vm) + } + #[pyfunction] fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { { From 07476a24fc1d57a9b1985c51b1a98545d53897e3 Mon Sep 17 00:00:00 2001 From: lynskylate Date: Sun, 2 Aug 2020 02:52:07 +0800 Subject: [PATCH 159/209] Fix is namespace --- vm/src/stdlib/symtable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 47359fa23..bab151f00 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -249,7 +249,7 @@ impl PySymbol { #[pymethod(name = "is_namespace")] fn is_namespace(&self) -> bool { - self.namespaces.is_empty() + !self.namespaces.is_empty() } #[pymethod(name = "is_annotated")] From c4ddff84cef3f53e35ba5685f8a9a716940250cc Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Thu, 30 Jul 2020 07:36:32 +1200 Subject: [PATCH 160/209] Implement "{x.attr}".format(...). --- vm/src/format.rs | 309 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 73 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 7a8945e1a..f7d3721ee 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -2,8 +2,9 @@ use crate::function::PyFuncArgs; use crate::obj::objint::PyInt; use crate::obj::objstr::PyString; use crate::obj::{objstr, objtype}; -use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use itertools::{Itertools, PeekingNext}; use num_bigint::{BigInt, Sign}; use num_traits::cast::ToPrimitive; use num_traits::Signed; @@ -557,6 +558,9 @@ pub(crate) enum FormatParseError { MissingStartBracket, UnescapedStartBracketInLiteral, InvalidFormatSpecifier, + EmptyAttribute, + MissingRightBracket, + InvalidCharacterAfterRightBracket, } impl FromStr for FormatSpec { @@ -566,25 +570,116 @@ impl FromStr for FormatSpec { } } +#[derive(Debug, PartialEq)] +pub(crate) enum FieldNamePart { + Attribute(String), + Index(usize), + StringIndex(String), +} + +impl FieldNamePart { + fn parse_part( + chars: &mut impl PeekingNext, + ) -> Result { + let ch = chars.next().unwrap(); + if ch == '.' { + let mut attribute = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + attribute.push(ch); + } + if attribute.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else { + Ok(FieldNamePart::Attribute(attribute)) + } + } else if ch == '[' { + let mut index = String::new(); + for ch in chars { + if ch == ']' { + return if index.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else if let Ok(index) = index.parse::() { + Ok(FieldNamePart::Index(index)) + } else { + Ok(FieldNamePart::StringIndex(index)) + }; + } + index.push(ch); + } + Err(FormatParseError::MissingRightBracket) + } else { + Err(FormatParseError::InvalidCharacterAfterRightBracket) + } + } +} + +#[derive(Debug, PartialEq)] +pub(crate) enum FieldType { + AutoSpec, + IndexSpec(usize), + KeywordSpec(String), +} + +#[derive(Debug, PartialEq)] +pub(crate) struct FieldName { + field_type: FieldType, + parts: Vec, +} + +impl FieldName { + fn parse(text: &str) -> Result { + let mut chars = text.chars().peekable(); + let mut first = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + first.push(ch); + } + + let field_type = if first.is_empty() { + FieldType::AutoSpec + } else if let Ok(index) = first.parse::() { + FieldType::IndexSpec(index) + } else { + FieldType::KeywordSpec(first) + }; + + let mut parts = Vec::new(); + while chars.peek().is_some() { + parts.push(FieldNamePart::parse_part(&mut chars)?) + } + + Ok(FieldName { field_type, parts }) + } +} + #[derive(Debug, PartialEq)] enum FormatPart { - AutoSpec(String), - IndexSpec(usize, String), - KeywordSpec(String, String), + Field(FieldName, String), Literal(String), } impl FormatPart { fn is_auto(&self) -> bool { match self { - FormatPart::AutoSpec(_) => true, + FormatPart::Field( + FieldName { + field_type: FieldType::AutoSpec, + .. + }, + _, + ) => true, _ => false, } } fn is_index(&self) -> bool { match self { - FormatPart::IndexSpec(_, _) => true, + FormatPart::Field( + FieldName { + field_type: FieldType::IndexSpec(_), + .. + }, + _, + ) => true, _ => false, } } @@ -655,16 +750,7 @@ impl FormatString { String::new() }; let format_spec = preconversor_spec + &format_spec; - - if arg_part.is_empty() { - return Ok(FormatPart::AutoSpec(format_spec)); - } - - if let Ok(index) = arg_part.parse::() { - Ok(FormatPart::IndexSpec(index, format_spec)) - } else { - Ok(FormatPart::KeywordSpec(arg_part.to_owned(), format_spec)) - } + Ok(FormatPart::Field(FieldName::parse(arg_part)?, format_spec)) } fn parse_spec<'a>( @@ -724,46 +810,34 @@ impl FormatString { } } - pub(crate) fn format(&self, arguments: &PyFuncArgs, vm: &VirtualMachine) -> PyResult { + fn format_internal( + &self, + vm: &VirtualMachine, + mut field_func: impl FnMut(&FieldType) -> PyResult, + ) -> PyResult { let mut final_string = String::new(); - if self.format_parts.iter().any(FormatPart::is_auto) - && self.format_parts.iter().any(FormatPart::is_index) - { - return Err(vm.new_value_error( - "cannot switch from automatic field numbering to manual field specification" - .to_owned(), - )); - } - let mut auto_argument_index: usize = 1; for part in &self.format_parts { let result_string: String = match part { - FormatPart::AutoSpec(format_spec) => { - let result = match arguments.args.get(auto_argument_index) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_index_error("tuple index out of range".to_owned())); + FormatPart::Field(FieldName { field_type, parts }, format_spec) => { + let mut argument = field_func(field_type)?; + + for name_part in parts { + match name_part { + FieldNamePart::Attribute(attribute) => { + argument = vm.get_attribute(argument, attribute.as_str())?; + } + FieldNamePart::Index(index) => { + // TODO Implement DictKey for usize so we can pass index directly + argument = argument.get_item(&index.into_pyobject(vm)?, vm)?; + } + FieldNamePart::StringIndex(index) => { + argument = argument.get_item(index, vm)?; + } } - }; - auto_argument_index += 1; - objstr::clone_value(&result) - } - FormatPart::IndexSpec(index, format_spec) => { - let result = match arguments.args.get(*index + 1) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_index_error("tuple index out of range".to_owned())); - } - }; - objstr::clone_value(&result) - } - FormatPart::KeywordSpec(keyword, format_spec) => { - let result = match arguments.get_optional_kwarg(&keyword) { - Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, - None => { - return Err(vm.new_key_error(vm.new_str(keyword.to_owned()))); - } - }; - objstr::clone_value(&result) + } + + let value = call_object_format(vm, argument, &format_spec)?; + objstr::clone_value(&value) } FormatPart::Literal(literal) => literal.clone(), }; @@ -772,25 +846,44 @@ impl FormatString { Ok(vm.ctx.new_str(final_string)) } - pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut final_string = String::new(); - for part in &self.format_parts { - let result_string: String = match part { - FormatPart::AutoSpec(_) | FormatPart::IndexSpec(_, _) => { - return Err( - vm.new_value_error("Format string contains positional fields".to_owned()) - ); - } - FormatPart::KeywordSpec(keyword, format_spec) => { - let argument = dict.get_item(keyword, &vm)?; - let result = call_object_format(vm, argument.clone(), &format_spec)?; - objstr::clone_value(&result) - } - FormatPart::Literal(literal) => literal.clone(), - }; - final_string.push_str(&result_string); + pub(crate) fn format(&self, arguments: &PyFuncArgs, vm: &VirtualMachine) -> PyResult { + if self.format_parts.iter().any(FormatPart::is_auto) + && self.format_parts.iter().any(FormatPart::is_index) + { + return Err(vm.new_value_error( + "cannot switch from automatic field numbering to manual field specification" + .to_owned(), + )); } - Ok(vm.ctx.new_str(final_string)) + + let mut auto_argument_index: usize = 1; + self.format_internal(vm, |field_type| match field_type { + FieldType::AutoSpec => { + auto_argument_index += 1; + arguments + .args + .get(auto_argument_index - 1) + .cloned() + .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) + } + FieldType::IndexSpec(index) => arguments + .args + .get(*index + 1) + .cloned() + .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())), + FieldType::KeywordSpec(keyword) => arguments + .get_optional_kwarg(&keyword) + .ok_or_else(|| vm.new_key_error(vm.new_str(keyword.to_owned()))), + }) + } + + pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.format_internal(vm, |field_type| match field_type { + FieldType::AutoSpec | FieldType::IndexSpec(_) => { + Err(vm.new_value_error("Format string contains positional fields".to_owned())) + } + FieldType::KeywordSpec(keyword) => dict.get_item(keyword, &vm), + }) } } @@ -968,9 +1061,21 @@ mod tests { let expected = Ok(FormatString { format_parts: vec![ FormatPart::Literal("abcd".to_owned()), - FormatPart::IndexSpec(1, String::new()), + FormatPart::Field( + FieldName { + field_type: FieldType::IndexSpec(1), + parts: Vec::new(), + }, + String::new(), + ), FormatPart::Literal(":".to_owned()), - FormatPart::KeywordSpec("key".to_owned(), String::new()), + FormatPart::Field( + FieldName { + field_type: FieldType::KeywordSpec("key".to_owned()), + parts: Vec::new(), + }, + String::new(), + ), ], }); @@ -993,7 +1098,13 @@ mod tests { let expected = Ok(FormatString { format_parts: vec![ FormatPart::Literal("{".to_owned()), - FormatPart::KeywordSpec("key".to_owned(), String::new()), + FormatPart::Field( + FieldName { + field_type: FieldType::KeywordSpec("key".to_owned()), + parts: Vec::new(), + }, + String::new(), + ), FormatPart::Literal("}ddfe".to_owned()), ], }); @@ -1014,4 +1125,56 @@ mod tests { assert_eq!(parse_format_spec("o!"), Err("Invalid format specifier")); assert_eq!(parse_format_spec("d "), Err("Invalid format specifier")); } + + #[test] + fn test_parse_field_name() { + assert_eq!( + FieldName::parse(""), + Ok(FieldName { + field_type: FieldType::AutoSpec, + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("0"), + Ok(FieldName { + field_type: FieldType::IndexSpec(0), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key"), + Ok(FieldName { + field_type: FieldType::KeywordSpec("key".to_owned()), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key.attr[0][string]"), + Ok(FieldName { + field_type: FieldType::KeywordSpec("key".to_owned()), + parts: vec![ + FieldNamePart::Attribute("attr".to_owned()), + FieldNamePart::Index(0), + FieldNamePart::StringIndex("string".to_owned()) + ], + }) + ); + assert_eq!( + FieldName::parse("key.."), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key[]"), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key["), + Err(FormatParseError::MissingRightBracket) + ); + assert_eq!( + FieldName::parse("key[0]after"), + Err(FormatParseError::InvalidCharacterAfterRightBracket) + ); + } } From e90e2bfc6e1bd4a2b84d3ddabf1499c1e3c13ebf Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 11:30:06 +1200 Subject: [PATCH 161/209] Implement string module. --- vm/src/format.rs | 231 ++++++++++++++++++++-------------------- vm/src/obj/objlist.rs | 7 ++ vm/src/obj/objstr.rs | 51 ++------- vm/src/stdlib/string.rs | 96 +++++++++++++++-- 4 files changed, 221 insertions(+), 164 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index f7d3721ee..c217911ae 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,6 +1,5 @@ +use crate::exceptions::PyBaseExceptionRef; use crate::function::PyFuncArgs; -use crate::obj::objint::PyInt; -use crate::obj::objstr::PyString; use crate::obj::{objstr, objtype}; use crate::pyobject::{IntoPyObject, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -558,11 +557,23 @@ pub(crate) enum FormatParseError { MissingStartBracket, UnescapedStartBracketInLiteral, InvalidFormatSpecifier, + UnknownConversion, EmptyAttribute, MissingRightBracket, InvalidCharacterAfterRightBracket, } +impl FormatParseError { + pub fn into_pyobject(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + FormatParseError::UnmatchedBracket => { + vm.new_value_error("expected '}' before end of string".to_owned()) + } + _ => vm.new_value_error("Unexpected error parsing format string".to_owned()), + } + } +} + impl FromStr for FormatSpec { type Err = &'static str; fn from_str(s: &str) -> Result { @@ -622,12 +633,12 @@ pub(crate) enum FieldType { #[derive(Debug, PartialEq)] pub(crate) struct FieldName { - field_type: FieldType, - parts: Vec, + pub(crate) field_type: FieldType, + pub(crate) parts: Vec, } impl FieldName { - fn parse(text: &str) -> Result { + pub(crate) fn parse(text: &str) -> Result { let mut chars = text.chars().peekable(); let mut first = String::new(); for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { @@ -652,42 +663,18 @@ impl FieldName { } #[derive(Debug, PartialEq)] -enum FormatPart { - Field(FieldName, String), +pub(crate) enum FormatPart { + Field { + field_name: String, + preconversion_spec: Option, + format_spec: String, + }, Literal(String), } -impl FormatPart { - fn is_auto(&self) -> bool { - match self { - FormatPart::Field( - FieldName { - field_type: FieldType::AutoSpec, - .. - }, - _, - ) => true, - _ => false, - } - } - - fn is_index(&self) -> bool { - match self { - FormatPart::Field( - FieldName { - field_type: FieldType::IndexSpec(_), - .. - }, - _, - ) => true, - _ => false, - } - } -} - #[derive(Debug, PartialEq)] pub(crate) struct FormatString { - format_parts: Vec, + pub(crate) format_parts: Vec, } impl FormatString { @@ -744,22 +731,31 @@ impl FormatString { // before the bang is a keyword or arg index, after the comma is maybe a conversor spec. let arg_part = parts[0]; - let preconversor_spec = if parts.len() > 1 { - "!".to_owned() + parts[1] + let preconversion_spec = if let Some(conversion) = parts.get(1) { + let mut chars = conversion.chars(); + if let Some(ch) = chars.next() { + // conversions are only every one character + if chars.next().is_some() { + return Err(FormatParseError::UnknownConversion); + } + Some(ch) + } else { + return Err(FormatParseError::UnknownConversion); + } } else { - String::new() + None }; - let format_spec = preconversor_spec + &format_spec; - Ok(FormatPart::Field(FieldName::parse(arg_part)?, format_spec)) + + Ok(FormatPart::Field { + field_name: arg_part.to_owned(), + preconversion_spec, + format_spec, + }) } - fn parse_spec<'a>( - text: &'a str, - args: &'a PyFuncArgs, - ) -> Result<(FormatPart, &'a str), FormatParseError> { + fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> { let mut nested = false; let mut end_bracket_pos = None; - let mut spec_template = String::new(); let mut left = String::new(); // There may be one layer nesting brackets in spec @@ -773,30 +769,18 @@ impl FormatString { return Err(FormatParseError::InvalidFormatSpecifier); } else { nested = true; + left.push(c); continue; } } else if c == '}' { if nested { nested = false; - if let Some(obj) = args.kwargs.get(&spec_template) { - if let Some(s) = (*obj).clone().payload::() { - left.push_str(s.as_str()); - } else if let Some(i) = (*obj).clone().payload::() { - left.push_str(&PyInt::repr(i)); - } else { - return Err(FormatParseError::InvalidFormatSpecifier); - } - } else { - // CPython return KeyError here - return Err(FormatParseError::InvalidFormatSpecifier); - } + left.push(c); continue; } else { end_bracket_pos = Some(idx); break; } - } else if nested { - spec_template.push(c); } else { left.push(c); } @@ -813,13 +797,20 @@ impl FormatString { fn format_internal( &self, vm: &VirtualMachine, - mut field_func: impl FnMut(&FieldType) -> PyResult, - ) -> PyResult { + field_func: &mut impl FnMut(&FieldType) -> PyResult, + ) -> PyResult { let mut final_string = String::new(); for part in &self.format_parts { let result_string: String = match part { - FormatPart::Field(FieldName { field_type, parts }, format_spec) => { - let mut argument = field_func(field_type)?; + FormatPart::Field { + field_name, + preconversion_spec, + format_spec, + } => { + let FieldName { field_type, parts } = + FieldName::parse(field_name.as_str()).map_err(|e| e.into_pyobject(vm))?; + + let mut argument = field_func(&field_type)?; for name_part in parts { match name_part { @@ -831,54 +822,66 @@ impl FormatString { argument = argument.get_item(&index.into_pyobject(vm)?, vm)?; } FieldNamePart::StringIndex(index) => { - argument = argument.get_item(index, vm)?; + argument = argument.get_item(&index, vm)?; } } } - let value = call_object_format(vm, argument, &format_spec)?; + let nested_format = + FormatString::from_str(&format_spec).map_err(|e| e.into_pyobject(vm))?; + let format_spec = nested_format.format_internal(vm, field_func)?; + + let value = + call_object_format(vm, argument, *preconversion_spec, &format_spec)?; objstr::clone_value(&value) } FormatPart::Literal(literal) => literal.clone(), }; final_string.push_str(&result_string); } - Ok(vm.ctx.new_str(final_string)) + Ok(final_string) } - pub(crate) fn format(&self, arguments: &PyFuncArgs, vm: &VirtualMachine) -> PyResult { - if self.format_parts.iter().any(FormatPart::is_auto) - && self.format_parts.iter().any(FormatPart::is_index) - { - return Err(vm.new_value_error( - "cannot switch from automatic field numbering to manual field specification" - .to_owned(), - )); - } - - let mut auto_argument_index: usize = 1; - self.format_internal(vm, |field_type| match field_type { + pub(crate) fn format(&self, arguments: &PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let mut auto_argument_index: usize = 0; + let mut seen_index = false; + self.format_internal(vm, &mut |field_type| match field_type { FieldType::AutoSpec => { + if seen_index { + return Err(vm.new_value_error( + "cannot switch from manual field specification to automatic field numbering" + .to_owned(), + )); + } auto_argument_index += 1; arguments .args - .get(auto_argument_index - 1) + .get(auto_argument_index) + .cloned() + .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) + } + FieldType::IndexSpec(index) => { + if auto_argument_index != 0 { + return Err(vm.new_value_error( + "cannot switch from automatic field numbering to manual field specification" + .to_owned(), + )); + } + seen_index = true; + arguments + .args + .get(*index) .cloned() .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) } - FieldType::IndexSpec(index) => arguments - .args - .get(*index + 1) - .cloned() - .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())), FieldType::KeywordSpec(keyword) => arguments .get_optional_kwarg(&keyword) .ok_or_else(|| vm.new_key_error(vm.new_str(keyword.to_owned()))), }) } - pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.format_internal(vm, |field_type| match field_type { + pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.format_internal(vm, &mut |field_type| match field_type { FieldType::AutoSpec | FieldType::IndexSpec(_) => { Err(vm.new_value_error("Format string contains positional fields".to_owned())) } @@ -887,16 +890,20 @@ impl FormatString { } } -fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { - let (preconversor, new_format_spec) = FormatPreconversor::parse_and_consume(format_spec); - let argument = match preconversor { +fn call_object_format( + vm: &VirtualMachine, + argument: PyObjectRef, + preconversion_spec: Option, + format_spec: &str, +) -> PyResult { + let argument = match preconversion_spec.and_then(|c| FormatPreconversor::from_char(c)) { Some(FormatPreconversor::Str) => vm.call_method(&argument, "__str__", vec![])?, Some(FormatPreconversor::Repr) => vm.call_method(&argument, "__repr__", vec![])?, Some(FormatPreconversor::Ascii) => vm.call_method(&argument, "__repr__", vec![])?, Some(FormatPreconversor::Bytes) => vm.call_method(&argument, "decode", vec![])?, None => argument, }; - let returned_type = vm.ctx.new_str(new_format_spec.to_owned()); + let returned_type = vm.ctx.new_str(format_spec.to_owned()); let result = vm.call_method(&argument, "__format__", vec![returned_type])?; if !objtype::isinstance(&result, &vm.ctx.types.str_type) { @@ -909,20 +916,20 @@ fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: & pub(crate) trait FromTemplate<'a>: Sized { type Err; - fn from_str(s: &'a str, arg: &'a PyFuncArgs) -> Result; + fn from_str(s: &'a str) -> Result; } impl<'a> FromTemplate<'a> for FormatString { type Err = FormatParseError; - fn from_str(text: &'a str, args: &'a PyFuncArgs) -> Result { + fn from_str(text: &'a str) -> Result { let mut cur_text: &str = text; let mut parts: Vec = Vec::new(); while !cur_text.is_empty() { // Try to parse both literals and bracketed format parts util we // run out of text cur_text = FormatString::parse_literal(cur_text) - .or_else(|_| FormatString::parse_spec(cur_text, &args)) + .or_else(|_| FormatString::parse_spec(cur_text)) .map(|(part, new_text)| { parts.push(part); new_text @@ -1061,21 +1068,17 @@ mod tests { let expected = Ok(FormatString { format_parts: vec![ FormatPart::Literal("abcd".to_owned()), - FormatPart::Field( - FieldName { - field_type: FieldType::IndexSpec(1), - parts: Vec::new(), - }, - String::new(), - ), + FormatPart::Field { + field_name: "1".to_owned(), + preconversion_spec: None, + format_spec: String::new(), + }, FormatPart::Literal(":".to_owned()), - FormatPart::Field( - FieldName { - field_type: FieldType::KeywordSpec("key".to_owned()), - parts: Vec::new(), - }, - String::new(), - ), + FormatPart::Field { + field_name: "key".to_owned(), + preconversion_spec: None, + format_spec: String::new(), + }, ], }); @@ -1098,13 +1101,11 @@ mod tests { let expected = Ok(FormatString { format_parts: vec![ FormatPart::Literal("{".to_owned()), - FormatPart::Field( - FieldName { - field_type: FieldType::KeywordSpec("key".to_owned()), - parts: Vec::new(), - }, - String::new(), - ), + FormatPart::Field { + field_name: "key".to_owned(), + preconversion_spec: None, + format_spec: String::new(), + }, FormatPart::Literal("}ddfe".to_owned()), ], }); diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 90e9629e6..a81468f42 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::iter::FromIterator; use std::mem::size_of; use std::ops::{DerefMut, Range}; @@ -47,6 +48,12 @@ impl From> for PyList { } } +impl FromIterator for PyList { + fn from_iter>(iter: T) -> Self { + Vec::from_iter(iter).into() + } +} + impl PyValue for PyList { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.list_type() diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 21b57976e..e4e1e7d1c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -19,7 +19,7 @@ use super::objiter; use super::objnone::PyNone; use super::objsequence::{PySliceableSequence, SequenceIndex}; use super::objtype::{self, PyClassRef}; -use crate::format::{FormatParseError, FormatSpec, FormatString, FromTemplate}; +use crate::format::{FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, @@ -587,31 +587,10 @@ impl PyString { } #[pymethod] - fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - return Err(vm.new_type_error( - "descriptor 'format' of 'str' object needs an argument".to_owned(), - )); - } - - let zelf = &args.args[0]; - if !objtype::isinstance(&zelf, &vm.ctx.types.str_type) { - let zelf_typ = zelf.class(); - let actual_type = vm.to_pystr(&zelf_typ)?; - return Err(vm.new_type_error(format!( - "descriptor 'format' requires a 'str' object but received a '{}'", - actual_type - ))); - } - let format_string_text = borrow_value(zelf); - match FormatString::from_str(format_string_text, &args) { + fn format(&self, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + match FormatString::from_str(&self.value) { Ok(format_string) => format_string.format(&args, vm), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_owned())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_owned())), - }, + Err(err) => Err(err.into_pyobject(vm)), } } @@ -620,24 +599,10 @@ impl PyString { /// Return a formatted version of S, using substitutions from mapping. /// The substitutions are identified by braces ('{' and '}'). #[pymethod] - fn format_map(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.len() != 2 { - return Err(vm.new_type_error(format!( - "format_map() takes exactly one argument ({} given)", - args.args.len() - 1 - ))); - } - - let zelf = &args.args[0]; - let format_string_text = borrow_value(zelf); - match FormatString::from_str(format_string_text, &args) { - Ok(format_string) => format_string.format_map(&args.args[1], vm), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_owned())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_owned())), - }, + fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match FormatString::from_str(&self.value) { + Ok(format_string) => format_string.format_map(&mapping, vm), + Err(err) => Err(err.into_pyobject(vm)), } } diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 215dce21f..7ab1484a4 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -1,12 +1,96 @@ /* String builtin module */ -use crate::pyobject::PyObjectRef; -use crate::vm::VirtualMachine; +pub(crate) use _string::make_module; -pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { - // let ctx = &vm.ctx; +#[pymodule] +mod _string { + use std::mem; - // Constants: - py_module!(vm, "_string", {}) + use crate::format::{ + FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, + }; + use crate::obj::objlist::PyList; + use crate::obj::objstr::PyStringRef; + use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult}; + use crate::vm::VirtualMachine; + + fn create_format_part( + literal: String, + field_name: Option, + format_spec: Option, + preconversion_spec: Option, + vm: &VirtualMachine, + ) -> PyResult { + let tuple = ( + literal, + field_name, + format_spec, + preconversion_spec.map(|c| c.to_string()), + ); + tuple.into_pyobject(vm) + } + + #[pyfunction] + fn formatter_parser(text: PyStringRef, vm: &VirtualMachine) -> PyResult { + let format_string = + FormatString::from_str(text.as_str()).map_err(|e| e.into_pyobject(vm))?; + + let mut result = Vec::new(); + let mut literal = String::new(); + for part in format_string.format_parts { + match part { + FormatPart::Field { + field_name, + preconversion_spec, + format_spec, + } => { + result.push(create_format_part( + mem::take(&mut literal), + Some(field_name), + Some(format_spec), + preconversion_spec, + vm, + )?); + } + FormatPart::Literal(text) => literal.push_str(&text), + } + } + if !literal.is_empty() { + result.push(create_format_part( + mem::take(&mut literal), + None, + None, + None, + vm, + )?); + } + Ok(result.into()) + } + + #[pyfunction] + fn formatter_field_name_split( + text: PyStringRef, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, PyList)> { + let field_name = FieldName::parse(text.as_str()).map_err(|e| e.into_pyobject(vm))?; + + let first = match field_name.field_type { + FieldType::AutoSpec => vm.new_str("".to_owned()), + FieldType::IndexSpec(index) => index.into_pyobject(vm)?, + FieldType::KeywordSpec(attribute) => attribute.into_pyobject(vm)?, + }; + + let rest = field_name + .parts + .iter() + .map(|p| match p { + FieldNamePart::Attribute(attribute) => (true, attribute).into_pyobject(vm).unwrap(), + FieldNamePart::StringIndex(index) => (false, index).into_pyobject(vm).unwrap(), + FieldNamePart::Index(index) => (false, *index).into_pyobject(vm).unwrap(), + }) + .collect(); + + Ok((first, rest)) + } } From 8ae65dad76374e75ab9411d4f652359529261569 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 11:48:40 +1200 Subject: [PATCH 162/209] Fix bug in format auto spec indexing. --- vm/src/format.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index c217911ae..c489b67ed 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -856,7 +856,7 @@ impl FormatString { auto_argument_index += 1; arguments .args - .get(auto_argument_index) + .get(auto_argument_index - 1) .cloned() .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) } From 815572beb32ef766185713544e6d9bcdb988ae6b Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 11:50:26 +1200 Subject: [PATCH 163/209] Copy test_string.py from CPython 3.8.3. --- Lib/test/test_string.py | 480 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100644 Lib/test/test_string.py diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py new file mode 100644 index 000000000..0be28fdb6 --- /dev/null +++ b/Lib/test/test_string.py @@ -0,0 +1,480 @@ +import unittest +import string +from string import Template + + +class ModuleTest(unittest.TestCase): + + def test_attrs(self): + # While the exact order of the items in these attributes is not + # technically part of the "language spec", in practice there is almost + # certainly user code that depends on the order, so de-facto it *is* + # part of the spec. + self.assertEqual(string.whitespace, ' \t\n\r\x0b\x0c') + self.assertEqual(string.ascii_lowercase, 'abcdefghijklmnopqrstuvwxyz') + self.assertEqual(string.ascii_uppercase, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ') + self.assertEqual(string.ascii_letters, string.ascii_lowercase + string.ascii_uppercase) + self.assertEqual(string.digits, '0123456789') + self.assertEqual(string.hexdigits, string.digits + 'abcdefABCDEF') + self.assertEqual(string.octdigits, '01234567') + self.assertEqual(string.punctuation, '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~') + self.assertEqual(string.printable, string.digits + string.ascii_lowercase + string.ascii_uppercase + string.punctuation + string.whitespace) + + def test_capwords(self): + self.assertEqual(string.capwords('abc def ghi'), 'Abc Def Ghi') + self.assertEqual(string.capwords('abc\tdef\nghi'), 'Abc Def Ghi') + self.assertEqual(string.capwords('abc\t def \nghi'), 'Abc Def Ghi') + self.assertEqual(string.capwords('ABC DEF GHI'), 'Abc Def Ghi') + self.assertEqual(string.capwords('ABC-DEF-GHI', '-'), 'Abc-Def-Ghi') + self.assertEqual(string.capwords('ABC-def DEF-ghi GHI'), 'Abc-def Def-ghi Ghi') + self.assertEqual(string.capwords(' aBc DeF '), 'Abc Def') + self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') + self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') + + def test_basic_formatter(self): + fmt = string.Formatter() + self.assertEqual(fmt.format("foo"), "foo") + self.assertEqual(fmt.format("foo{0}", "bar"), "foobar") + self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6") + self.assertRaises(TypeError, fmt.format) + self.assertRaises(TypeError, string.Formatter.format) + + def test_format_keyword_arguments(self): + fmt = string.Formatter() + self.assertEqual(fmt.format("-{arg}-", arg='test'), '-test-') + self.assertRaises(KeyError, fmt.format, "-{arg}-") + self.assertEqual(fmt.format("-{self}-", self='test'), '-test-') + self.assertRaises(KeyError, fmt.format, "-{self}-") + self.assertEqual(fmt.format("-{format_string}-", format_string='test'), + '-test-') + self.assertRaises(KeyError, fmt.format, "-{format_string}-") + with self.assertRaisesRegex(TypeError, "format_string"): + fmt.format(format_string="-{arg}-", arg='test') + + def test_auto_numbering(self): + fmt = string.Formatter() + self.assertEqual(fmt.format('foo{}{}', 'bar', 6), + 'foo{}{}'.format('bar', 6)) + self.assertEqual(fmt.format('foo{1}{num}{1}', None, 'bar', num=6), + 'foo{1}{num}{1}'.format(None, 'bar', num=6)) + self.assertEqual(fmt.format('{:^{}}', 'bar', 6), + '{:^{}}'.format('bar', 6)) + self.assertEqual(fmt.format('{:^{}} {}', 'bar', 6, 'X'), + '{:^{}} {}'.format('bar', 6, 'X')) + self.assertEqual(fmt.format('{:^{pad}}{}', 'foo', 'bar', pad=6), + '{:^{pad}}{}'.format('foo', 'bar', pad=6)) + + with self.assertRaises(ValueError): + fmt.format('foo{1}{}', 'bar', 6) + + with self.assertRaises(ValueError): + fmt.format('foo{}{1}', 'bar', 6) + + def test_conversion_specifiers(self): + fmt = string.Formatter() + self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-") + self.assertEqual(fmt.format("{0!s}", 'test'), 'test') + self.assertRaises(ValueError, fmt.format, "{0!h}", 'test') + # issue13579 + self.assertEqual(fmt.format("{0!a}", 42), '42') + self.assertEqual(fmt.format("{0!a}", string.ascii_letters), + "'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'") + self.assertEqual(fmt.format("{0!a}", chr(255)), "'\\xff'") + self.assertEqual(fmt.format("{0!a}", chr(256)), "'\\u0100'") + + def test_name_lookup(self): + fmt = string.Formatter() + class AnyAttr: + def __getattr__(self, attr): + return attr + x = AnyAttr() + self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack') + with self.assertRaises(AttributeError): + fmt.format("{0.lumber}{0.jack}", '') + + def test_index_lookup(self): + fmt = string.Formatter() + lookup = ["eggs", "and", "spam"] + self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs') + with self.assertRaises(IndexError): + fmt.format("{0[2]}{0[0]}", []) + with self.assertRaises(KeyError): + fmt.format("{0[2]}{0[0]}", {}) + + def test_override_get_value(self): + class NamespaceFormatter(string.Formatter): + def __init__(self, namespace={}): + string.Formatter.__init__(self) + self.namespace = namespace + + def get_value(self, key, args, kwds): + if isinstance(key, str): + try: + # Check explicitly passed arguments first + return kwds[key] + except KeyError: + return self.namespace[key] + else: + string.Formatter.get_value(key, args, kwds) + + fmt = NamespaceFormatter({'greeting':'hello'}) + self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!') + + + def test_override_format_field(self): + class CallFormatter(string.Formatter): + def format_field(self, value, format_spec): + return format(value(), format_spec) + + fmt = CallFormatter() + self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*') + + + def test_override_convert_field(self): + class XFormatter(string.Formatter): + def convert_field(self, value, conversion): + if conversion == 'x': + return None + return super().convert_field(value, conversion) + + fmt = XFormatter() + self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None") + + + def test_override_parse(self): + class BarFormatter(string.Formatter): + # returns an iterable that contains tuples of the form: + # (literal_text, field_name, format_spec, conversion) + def parse(self, format_string): + for field in format_string.split('|'): + if field[0] == '+': + # it's markup + field_name, _, format_spec = field[1:].partition(':') + yield '', field_name, format_spec, None + else: + yield field, None, None, None + + fmt = BarFormatter() + self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *') + + def test_check_unused_args(self): + class CheckAllUsedFormatter(string.Formatter): + def check_unused_args(self, used_args, args, kwargs): + # Track which arguments actually got used + unused_args = set(kwargs.keys()) + unused_args.update(range(0, len(args))) + + for arg in used_args: + unused_args.remove(arg) + + if unused_args: + raise ValueError("unused arguments") + + fmt = CheckAllUsedFormatter() + self.assertEqual(fmt.format("{0}", 10), "10") + self.assertEqual(fmt.format("{0}{i}", 10, i=100), "10100") + self.assertEqual(fmt.format("{0}{i}{1}", 10, 20, i=100), "1010020") + self.assertRaises(ValueError, fmt.format, "{0}{i}{1}", 10, 20, i=100, j=0) + self.assertRaises(ValueError, fmt.format, "{0}", 10, 20) + self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100) + self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100) + + def test_vformat_recursion_limit(self): + fmt = string.Formatter() + args = () + kwargs = dict(i=100) + with self.assertRaises(ValueError) as err: + fmt._vformat("{i}", args, kwargs, set(), -1) + self.assertIn("recursion", str(err.exception)) + + +# Template tests (formerly housed in test_pep292.py) + +class Bag: + pass + +class Mapping: + def __getitem__(self, name): + obj = self + for part in name.split('.'): + try: + obj = getattr(obj, part) + except AttributeError: + raise KeyError(name) + return obj + + +class TestTemplate(unittest.TestCase): + def test_regular_templates(self): + s = Template('$who likes to eat a bag of $what worth $$100') + self.assertEqual(s.substitute(dict(who='tim', what='ham')), + 'tim likes to eat a bag of ham worth $100') + self.assertRaises(KeyError, s.substitute, dict(who='tim')) + self.assertRaises(TypeError, Template.substitute) + + def test_regular_templates_with_braces(self): + s = Template('$who likes ${what} for ${meal}') + d = dict(who='tim', what='ham', meal='dinner') + self.assertEqual(s.substitute(d), 'tim likes ham for dinner') + self.assertRaises(KeyError, s.substitute, + dict(who='tim', what='ham')) + + def test_regular_templates_with_upper_case(self): + s = Template('$WHO likes ${WHAT} for ${MEAL}') + d = dict(WHO='tim', WHAT='ham', MEAL='dinner') + self.assertEqual(s.substitute(d), 'tim likes ham for dinner') + + def test_regular_templates_with_non_letters(self): + s = Template('$_wh0_ likes ${_w_h_a_t_} for ${mea1}') + d = dict(_wh0_='tim', _w_h_a_t_='ham', mea1='dinner') + self.assertEqual(s.substitute(d), 'tim likes ham for dinner') + + def test_escapes(self): + eq = self.assertEqual + s = Template('$who likes to eat a bag of $$what worth $$100') + eq(s.substitute(dict(who='tim', what='ham')), + 'tim likes to eat a bag of $what worth $100') + s = Template('$who likes $$') + eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $') + + def test_percents(self): + eq = self.assertEqual + s = Template('%(foo)s $foo ${foo}') + d = dict(foo='baz') + eq(s.substitute(d), '%(foo)s baz baz') + eq(s.safe_substitute(d), '%(foo)s baz baz') + + def test_stringification(self): + eq = self.assertEqual + s = Template('tim has eaten $count bags of ham today') + d = dict(count=7) + eq(s.substitute(d), 'tim has eaten 7 bags of ham today') + eq(s.safe_substitute(d), 'tim has eaten 7 bags of ham today') + s = Template('tim has eaten ${count} bags of ham today') + eq(s.substitute(d), 'tim has eaten 7 bags of ham today') + + def test_tupleargs(self): + eq = self.assertEqual + s = Template('$who ate ${meal}') + d = dict(who=('tim', 'fred'), meal=('ham', 'kung pao')) + eq(s.substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')") + eq(s.safe_substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')") + + def test_SafeTemplate(self): + eq = self.assertEqual + s = Template('$who likes ${what} for ${meal}') + eq(s.safe_substitute(dict(who='tim')), 'tim likes ${what} for ${meal}') + eq(s.safe_substitute(dict(what='ham')), '$who likes ham for ${meal}') + eq(s.safe_substitute(dict(what='ham', meal='dinner')), + '$who likes ham for dinner') + eq(s.safe_substitute(dict(who='tim', what='ham')), + 'tim likes ham for ${meal}') + eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), + 'tim likes ham for dinner') + + def test_invalid_placeholders(self): + raises = self.assertRaises + s = Template('$who likes $') + raises(ValueError, s.substitute, dict(who='tim')) + s = Template('$who likes ${what)') + raises(ValueError, s.substitute, dict(who='tim')) + s = Template('$who likes $100') + raises(ValueError, s.substitute, dict(who='tim')) + # Template.idpattern should match to only ASCII characters. + # https://bugs.python.org/issue31672 + s = Template("$who likes $\u0131") # (DOTLESS I) + raises(ValueError, s.substitute, dict(who='tim')) + s = Template("$who likes $\u0130") # (LATIN CAPITAL LETTER I WITH DOT ABOVE) + raises(ValueError, s.substitute, dict(who='tim')) + + def test_idpattern_override(self): + class PathPattern(Template): + idpattern = r'[_a-z][._a-z0-9]*' + m = Mapping() + m.bag = Bag() + m.bag.foo = Bag() + m.bag.foo.who = 'tim' + m.bag.what = 'ham' + s = PathPattern('$bag.foo.who likes to eat a bag of $bag.what') + self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham') + + def test_flags_override(self): + class MyPattern(Template): + flags = 0 + s = MyPattern('$wHO likes ${WHAT} for ${meal}') + d = dict(wHO='tim', WHAT='ham', meal='dinner', w='fred') + self.assertRaises(ValueError, s.substitute, d) + self.assertEqual(s.safe_substitute(d), 'fredHO likes ${WHAT} for dinner') + + def test_idpattern_override_inside_outside(self): + # bpo-1198569: Allow the regexp inside and outside braces to be + # different when deriving from Template. + class MyPattern(Template): + idpattern = r'[a-z]+' + braceidpattern = r'[A-Z]+' + flags = 0 + m = dict(foo='foo', BAR='BAR') + s = MyPattern('$foo ${BAR}') + self.assertEqual(s.substitute(m), 'foo BAR') + + def test_idpattern_override_inside_outside_invalid_unbraced(self): + # bpo-1198569: Allow the regexp inside and outside braces to be + # different when deriving from Template. + class MyPattern(Template): + idpattern = r'[a-z]+' + braceidpattern = r'[A-Z]+' + flags = 0 + m = dict(foo='foo', BAR='BAR') + s = MyPattern('$FOO') + self.assertRaises(ValueError, s.substitute, m) + s = MyPattern('${bar}') + self.assertRaises(ValueError, s.substitute, m) + + def test_pattern_override(self): + class MyPattern(Template): + pattern = r""" + (?P@{2}) | + @(?P[_a-z][._a-z0-9]*) | + @{(?P[_a-z][._a-z0-9]*)} | + (?P@) + """ + m = Mapping() + m.bag = Bag() + m.bag.foo = Bag() + m.bag.foo.who = 'tim' + m.bag.what = 'ham' + s = MyPattern('@bag.foo.who likes to eat a bag of @bag.what') + self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham') + + class BadPattern(Template): + pattern = r""" + (?P.*) | + (?P@{2}) | + @(?P[_a-z][._a-z0-9]*) | + @{(?P[_a-z][._a-z0-9]*)} | + (?P@) | + """ + s = BadPattern('@bag.foo.who likes to eat a bag of @bag.what') + self.assertRaises(ValueError, s.substitute, {}) + self.assertRaises(ValueError, s.safe_substitute, {}) + + def test_braced_override(self): + class MyTemplate(Template): + pattern = r""" + \$(?: + (?P$) | + (?P[_a-z][_a-z0-9]*) | + @@(?P[_a-z][_a-z0-9]*)@@ | + (?P) | + ) + """ + + tmpl = 'PyCon in $@@location@@' + t = MyTemplate(tmpl) + self.assertRaises(KeyError, t.substitute, {}) + val = t.substitute({'location': 'Cleveland'}) + self.assertEqual(val, 'PyCon in Cleveland') + + def test_braced_override_safe(self): + class MyTemplate(Template): + pattern = r""" + \$(?: + (?P$) | + (?P[_a-z][_a-z0-9]*) | + @@(?P[_a-z][_a-z0-9]*)@@ | + (?P) | + ) + """ + + tmpl = 'PyCon in $@@location@@' + t = MyTemplate(tmpl) + self.assertEqual(t.safe_substitute(), tmpl) + val = t.safe_substitute({'location': 'Cleveland'}) + self.assertEqual(val, 'PyCon in Cleveland') + + def test_invalid_with_no_lines(self): + # The error formatting for invalid templates + # has a special case for no data that the default + # pattern can't trigger (always has at least '$') + # So we craft a pattern that is always invalid + # with no leading data. + class MyTemplate(Template): + pattern = r""" + (?P) | + unreachable( + (?P) | + (?P) | + (?P) + ) + """ + s = MyTemplate('') + with self.assertRaises(ValueError) as err: + s.substitute({}) + self.assertIn('line 1, col 1', str(err.exception)) + + def test_unicode_values(self): + s = Template('$who likes $what') + d = dict(who='t\xffm', what='f\xfe\fed') + self.assertEqual(s.substitute(d), 't\xffm likes f\xfe\x0ced') + + def test_keyword_arguments(self): + eq = self.assertEqual + s = Template('$who likes $what') + eq(s.substitute(who='tim', what='ham'), 'tim likes ham') + eq(s.substitute(dict(who='tim'), what='ham'), 'tim likes ham') + eq(s.substitute(dict(who='fred', what='kung pao'), + who='tim', what='ham'), + 'tim likes ham') + s = Template('the mapping is $mapping') + eq(s.substitute(dict(foo='none'), mapping='bozo'), + 'the mapping is bozo') + eq(s.substitute(dict(mapping='one'), mapping='two'), + 'the mapping is two') + + s = Template('the self is $self') + eq(s.substitute(self='bozo'), 'the self is bozo') + + def test_keyword_arguments_safe(self): + eq = self.assertEqual + raises = self.assertRaises + s = Template('$who likes $what') + eq(s.safe_substitute(who='tim', what='ham'), 'tim likes ham') + eq(s.safe_substitute(dict(who='tim'), what='ham'), 'tim likes ham') + eq(s.safe_substitute(dict(who='fred', what='kung pao'), + who='tim', what='ham'), + 'tim likes ham') + s = Template('the mapping is $mapping') + eq(s.safe_substitute(dict(foo='none'), mapping='bozo'), + 'the mapping is bozo') + eq(s.safe_substitute(dict(mapping='one'), mapping='two'), + 'the mapping is two') + d = dict(mapping='one') + raises(TypeError, s.substitute, d, {}) + raises(TypeError, s.safe_substitute, d, {}) + + s = Template('the self is $self') + eq(s.safe_substitute(self='bozo'), 'the self is bozo') + + def test_delimiter_override(self): + eq = self.assertEqual + raises = self.assertRaises + class AmpersandTemplate(Template): + delimiter = '&' + s = AmpersandTemplate('this &gift is for &{who} &&') + eq(s.substitute(gift='bud', who='you'), 'this bud is for you &') + raises(KeyError, s.substitute) + eq(s.safe_substitute(gift='bud', who='you'), 'this bud is for you &') + eq(s.safe_substitute(), 'this &gift is for &{who} &') + s = AmpersandTemplate('this &gift is for &{who} &') + raises(ValueError, s.substitute, dict(gift='bud', who='you')) + eq(s.safe_substitute(), 'this &gift is for &{who} &') + + class PieDelims(Template): + delimiter = '@' + s = PieDelims('@who likes to eat a bag of @{what} worth $100') + self.assertEqual(s.substitute(dict(who='tim', what='ham')), + 'tim likes to eat a bag of ham worth $100') + + +if __name__ == '__main__': + unittest.main() From 0bfe830e7aa557c8782cdaea7277590ded4f1af1 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 12:34:05 +1200 Subject: [PATCH 164/209] Mark failing test_string.py tests. --- Lib/test/test_string.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py index 0be28fdb6..242d211e5 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string.py @@ -272,6 +272,8 @@ class TestTemplate(unittest.TestCase): eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), 'tim likes ham for dinner') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_invalid_placeholders(self): raises = self.assertRaises s = Template('$who likes $') @@ -298,6 +300,8 @@ class TestTemplate(unittest.TestCase): s = PathPattern('$bag.foo.who likes to eat a bag of $bag.what') self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_flags_override(self): class MyPattern(Template): flags = 0 @@ -317,6 +321,8 @@ class TestTemplate(unittest.TestCase): s = MyPattern('$foo ${BAR}') self.assertEqual(s.substitute(m), 'foo BAR') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_idpattern_override_inside_outside_invalid_unbraced(self): # bpo-1198569: Allow the regexp inside and outside braces to be # different when deriving from Template. @@ -455,6 +461,8 @@ class TestTemplate(unittest.TestCase): s = Template('the self is $self') eq(s.safe_substitute(self='bozo'), 'the self is bozo') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_delimiter_override(self): eq = self.assertEqual raises = self.assertRaises From fa3e571889ab2ec8e7812026980ae8a13779e318 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 12:52:35 +1200 Subject: [PATCH 165/209] Unmark tests that are now passing in test_unicode.py. --- Lib/test/test_unicode.py | 6 ------ vm/src/format.rs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 09782eb96..69aff1518 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1273,8 +1273,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual("{!s}".format(n), 'N(data)') self.assertRaises(TypeError, "{}".format, n) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_map(self): self.assertEqual(''.format_map({}), '') self.assertEqual('a'.format_map({}), 'a') @@ -2958,8 +2956,6 @@ class CAPITest(unittest.TestCase): self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) class StringModuleTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_formatter_parser(self): def parse(format): return list(_string.formatter_parser(format)) @@ -2993,8 +2989,6 @@ class StringModuleTest(unittest.TestCase): self.assertRaises(TypeError, _string.formatter_parser, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_formatter_field_name_split(self): def split(name): items = list(_string.formatter_field_name_split(name)) diff --git a/vm/src/format.rs b/vm/src/format.rs index c489b67ed..391294c42 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -685,7 +685,7 @@ impl FormatString { if first_char == '{' || first_char == '}' { let maybe_next_char = chars.next(); // if we see a bracket, it has to be escaped by doubling up to be in a literal - if maybe_next_char.is_some() && maybe_next_char.unwrap() != first_char { + if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char { return Err(FormatParseError::UnescapedStartBracketInLiteral); } else { return Ok((first_char, chars.as_str())); From 058e73a37627847b4043922ef419001b70d10792 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 13:14:34 +1200 Subject: [PATCH 166/209] Fix compilation of rust unit tests. --- vm/src/format.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 391294c42..42c7e64d1 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1083,7 +1083,7 @@ mod tests { }); assert_eq!( - FormatString::from_str("abcd{1}:{key}", &PyFuncArgs::default()), + FormatString::from_str("abcd{1}:{key}"), expected ); } @@ -1091,7 +1091,7 @@ mod tests { #[test] fn test_format_parse_fail() { assert_eq!( - FormatString::from_str("{s", &PyFuncArgs::default()), + FormatString::from_str("{s"), Err(FormatParseError::UnmatchedBracket) ); } @@ -1111,7 +1111,7 @@ mod tests { }); assert_eq!( - FormatString::from_str("{{{key}}}ddfe", &PyFuncArgs::default()), + FormatString::from_str("{{{key}}}ddfe"), expected ); } From 1c18d990447c00beb9f1b4363395d37dea95adc4 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 13:23:59 +1200 Subject: [PATCH 167/209] Run rustfmt --- vm/src/format.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 42c7e64d1..ce9aad7ee 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1082,10 +1082,7 @@ mod tests { ], }); - assert_eq!( - FormatString::from_str("abcd{1}:{key}"), - expected - ); + assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected); } #[test] @@ -1110,10 +1107,7 @@ mod tests { ], }); - assert_eq!( - FormatString::from_str("{{{key}}}ddfe"), - expected - ); + assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected); } #[test] From e9a2944138c4f4b5641e04b8636eab8ebeaf6761 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sat, 1 Aug 2020 13:48:44 +1200 Subject: [PATCH 168/209] Fix clippy warnings --- vm/src/format.rs | 32 ++++++++++++++++---------------- vm/src/stdlib/string.rs | 6 +++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index ce9aad7ee..547e4bebd 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -626,9 +626,9 @@ impl FieldNamePart { #[derive(Debug, PartialEq)] pub(crate) enum FieldType { - AutoSpec, - IndexSpec(usize), - KeywordSpec(String), + Auto, + Index(usize), + Keyword(String), } #[derive(Debug, PartialEq)] @@ -646,11 +646,11 @@ impl FieldName { } let field_type = if first.is_empty() { - FieldType::AutoSpec + FieldType::Auto } else if let Ok(index) = first.parse::() { - FieldType::IndexSpec(index) + FieldType::Index(index) } else { - FieldType::KeywordSpec(first) + FieldType::Keyword(first) }; let mut parts = Vec::new(); @@ -846,7 +846,7 @@ impl FormatString { let mut auto_argument_index: usize = 0; let mut seen_index = false; self.format_internal(vm, &mut |field_type| match field_type { - FieldType::AutoSpec => { + FieldType::Auto => { if seen_index { return Err(vm.new_value_error( "cannot switch from manual field specification to automatic field numbering" @@ -860,7 +860,7 @@ impl FormatString { .cloned() .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) } - FieldType::IndexSpec(index) => { + FieldType::Index(index) => { if auto_argument_index != 0 { return Err(vm.new_value_error( "cannot switch from automatic field numbering to manual field specification" @@ -874,7 +874,7 @@ impl FormatString { .cloned() .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) } - FieldType::KeywordSpec(keyword) => arguments + FieldType::Keyword(keyword) => arguments .get_optional_kwarg(&keyword) .ok_or_else(|| vm.new_key_error(vm.new_str(keyword.to_owned()))), }) @@ -882,10 +882,10 @@ impl FormatString { pub(crate) fn format_map(&self, dict: &PyObjectRef, vm: &VirtualMachine) -> PyResult { self.format_internal(vm, &mut |field_type| match field_type { - FieldType::AutoSpec | FieldType::IndexSpec(_) => { + FieldType::Auto | FieldType::Index(_) => { Err(vm.new_value_error("Format string contains positional fields".to_owned())) } - FieldType::KeywordSpec(keyword) => dict.get_item(keyword, &vm), + FieldType::Keyword(keyword) => dict.get_item(keyword, &vm), }) } } @@ -896,7 +896,7 @@ fn call_object_format( preconversion_spec: Option, format_spec: &str, ) -> PyResult { - let argument = match preconversion_spec.and_then(|c| FormatPreconversor::from_char(c)) { + let argument = match preconversion_spec.and_then(FormatPreconversor::from_char) { Some(FormatPreconversor::Str) => vm.call_method(&argument, "__str__", vec![])?, Some(FormatPreconversor::Repr) => vm.call_method(&argument, "__repr__", vec![])?, Some(FormatPreconversor::Ascii) => vm.call_method(&argument, "__repr__", vec![])?, @@ -1126,28 +1126,28 @@ mod tests { assert_eq!( FieldName::parse(""), Ok(FieldName { - field_type: FieldType::AutoSpec, + field_type: FieldType::Auto, parts: Vec::new(), }) ); assert_eq!( FieldName::parse("0"), Ok(FieldName { - field_type: FieldType::IndexSpec(0), + field_type: FieldType::Index(0), parts: Vec::new(), }) ); assert_eq!( FieldName::parse("key"), Ok(FieldName { - field_type: FieldType::KeywordSpec("key".to_owned()), + field_type: FieldType::Keyword("key".to_owned()), parts: Vec::new(), }) ); assert_eq!( FieldName::parse("key.attr[0][string]"), Ok(FieldName { - field_type: FieldType::KeywordSpec("key".to_owned()), + field_type: FieldType::Keyword("key".to_owned()), parts: vec![ FieldNamePart::Attribute("attr".to_owned()), FieldNamePart::Index(0), diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 7ab1484a4..8319f04b8 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -76,9 +76,9 @@ mod _string { let field_name = FieldName::parse(text.as_str()).map_err(|e| e.into_pyobject(vm))?; let first = match field_name.field_type { - FieldType::AutoSpec => vm.new_str("".to_owned()), - FieldType::IndexSpec(index) => index.into_pyobject(vm)?, - FieldType::KeywordSpec(attribute) => attribute.into_pyobject(vm)?, + FieldType::Auto => vm.new_str("".to_owned()), + FieldType::Index(index) => index.into_pyobject(vm)?, + FieldType::Keyword(attribute) => attribute.into_pyobject(vm)?, }; let rest = field_name From 928bca88c6edf162d0c62e6344f84d16b7ce4f1b Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 2 Aug 2020 07:37:47 +1200 Subject: [PATCH 169/209] Code review changes. --- vm/src/format.rs | 68 +++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 547e4bebd..133cc9409 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -591,36 +591,40 @@ pub(crate) enum FieldNamePart { impl FieldNamePart { fn parse_part( chars: &mut impl PeekingNext, - ) -> Result { - let ch = chars.next().unwrap(); - if ch == '.' { - let mut attribute = String::new(); - for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { - attribute.push(ch); - } - if attribute.is_empty() { - Err(FormatParseError::EmptyAttribute) - } else { - Ok(FieldNamePart::Attribute(attribute)) - } - } else if ch == '[' { - let mut index = String::new(); - for ch in chars { - if ch == ']' { - return if index.is_empty() { + ) -> Result, FormatParseError> { + chars + .next() + .map(|ch| match ch { + '.' => { + let mut attribute = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + attribute.push(ch); + } + if attribute.is_empty() { Err(FormatParseError::EmptyAttribute) - } else if let Ok(index) = index.parse::() { - Ok(FieldNamePart::Index(index)) } else { - Ok(FieldNamePart::StringIndex(index)) - }; + Ok(FieldNamePart::Attribute(attribute)) + } } - index.push(ch); - } - Err(FormatParseError::MissingRightBracket) - } else { - Err(FormatParseError::InvalidCharacterAfterRightBracket) - } + '[' => { + let mut index = String::new(); + for ch in chars { + if ch == ']' { + return if index.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else if let Ok(index) = index.parse::() { + Ok(FieldNamePart::Index(index)) + } else { + Ok(FieldNamePart::StringIndex(index)) + }; + } + index.push(ch); + } + Err(FormatParseError::MissingRightBracket) + } + _ => Err(FormatParseError::InvalidCharacterAfterRightBracket), + }) + .transpose() } } @@ -633,8 +637,8 @@ pub(crate) enum FieldType { #[derive(Debug, PartialEq)] pub(crate) struct FieldName { - pub(crate) field_type: FieldType, - pub(crate) parts: Vec, + pub field_type: FieldType, + pub parts: Vec, } impl FieldName { @@ -654,8 +658,8 @@ impl FieldName { }; let mut parts = Vec::new(); - while chars.peek().is_some() { - parts.push(FieldNamePart::parse_part(&mut chars)?) + while let Some(part) = FieldNamePart::parse_part(&mut chars)? { + parts.push(part) } Ok(FieldName { field_type, parts }) @@ -674,7 +678,7 @@ pub(crate) enum FormatPart { #[derive(Debug, PartialEq)] pub(crate) struct FormatString { - pub(crate) format_parts: Vec, + pub format_parts: Vec, } impl FormatString { From 7e69e5ef2c092f3b1d3a708023c6e14b5bfebd7b Mon Sep 17 00:00:00 2001 From: lynskylate Date: Sun, 2 Aug 2020 03:43:19 +0800 Subject: [PATCH 170/209] Fix scope change --- compiler/src/symboltable.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 79e0b2bed..e8b80f09e 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -281,11 +281,6 @@ impl<'a> SymbolTableAnalyzer<'a> { fn analyze_unknown_symbol(&self, symbol: &mut Symbol) { if symbol.is_assigned || symbol.is_parameter { symbol.scope = SymbolScope::Local; - } else if !symbol.is_assigned && symbol.is_referenced { - // if symbol is referenced in its block, but not assigned to. - // https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.Symbol.is_free - symbol.scope = SymbolScope::Unknown; - symbol.is_free = true; } else { // Interesting stuff about the __class__ variable: // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object From 639d21acc97bc73a2e32993ae50f9f79e6a8a84b Mon Sep 17 00:00:00 2001 From: Basix Date: Sun, 2 Aug 2020 13:29:34 +0900 Subject: [PATCH 171/209] Implement os.chmod on Windows --- vm/src/stdlib/os.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 3bb7465ba..ec989b880 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -2291,6 +2291,27 @@ mod nt { get_stats().map_err(|e| convert_io_error(vm, e)) } + #[pyfunction] + fn os_chmod( + path: PyPathLike, + dir_fd: DirFd, + mode: u32, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + const S_IWRITE: u32 = 128; + let path = make_path(vm, &path, &dir_fd); + let metadata = if follow_symlinks.follow_symlinks { + fs::metadata(path) + } else { + fs::symlink_metadata(path) + }; + let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + let mut permissions = meta.permissions(); + permissions.set_readonly(mode & S_IWRITE != 0); + fs::set_permissions(path, permissions).map_err(|err| convert_io_error(vm, err)) + } + // cwait is available on MSVC only (according to CPython) #[cfg(target_env = "msvc")] extern "C" { From 43e6837ddb4119a48ac86bf108e4fcd5e05d01ad Mon Sep 17 00:00:00 2001 From: Basix Date: Sun, 2 Aug 2020 14:01:55 +0900 Subject: [PATCH 172/209] Rename os_chmod to chmod --- vm/src/stdlib/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index ec989b880..b03479ab3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -2292,7 +2292,7 @@ mod nt { } #[pyfunction] - fn os_chmod( + fn chmod( path: PyPathLike, dir_fd: DirFd, mode: u32, From ad7f1719988bdd5865ac993c0432b26f57db8005 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 2 Aug 2020 14:07:16 +0900 Subject: [PATCH 173/209] Copy test_repl.py from CPython 3.8.3 --- Lib/test/test_repl.py | 98 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 Lib/test/test_repl.py diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py new file mode 100644 index 000000000..71f192f90 --- /dev/null +++ b/Lib/test/test_repl.py @@ -0,0 +1,98 @@ +"""Test the interactive interpreter.""" + +import sys +import os +import unittest +import subprocess +from textwrap import dedent +from test.support import cpython_only, SuppressCrashReport +from test.support.script_helper import kill_python + +def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): + """Run the Python REPL with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ + + # To run the REPL without using a terminal, spawn python with the command + # line option '-i' and the process name set to ''. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, executable=sys.executable, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + +class TestInteractiveInterpreter(unittest.TestCase): + + @cpython_only + def test_no_memory(self): + # Issue #30696: Fix the interactive interpreter looping endlessly when + # no memory. Check also that the fix does not break the interactive + # loop when an exception is raised. + user_input = """ + import sys, _testcapi + 1/0 + print('After the exception.') + _testcapi.set_nomemory(0) + sys.exit(0) + """ + user_input = dedent(user_input) + user_input = user_input.encode() + p = spawn_repl() + with SuppressCrashReport(): + p.stdin.write(user_input) + output = kill_python(p) + self.assertIn(b'After the exception.', output) + # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr. + self.assertIn(p.returncode, (1, 120)) + + @cpython_only + def test_multiline_string_parsing(self): + # bpo-39209: Multiline string tokens need to be handled in the tokenizer + # in two places: the interactive path and the non-interactive path. + user_input = '''\ + x = """ + + + + + 0KiB + 0 + 1.3 + 0 + + + 16738211KiB + 237.15 + 1.3 + 0 + + never + none + + + """ + ''' + user_input = dedent(user_input) + user_input = user_input.encode() + p = spawn_repl() + with SuppressCrashReport(): + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + +if __name__ == "__main__": + unittest.main() From 31061b0464c5358ab53cc47a5124ea3befcbd9d2 Mon Sep 17 00:00:00 2001 From: lynskylate Date: Sun, 2 Aug 2020 15:11:08 +0800 Subject: [PATCH 174/209] Optimize code --- vm/src/stdlib/symtable.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index bab151f00..2172f974c 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -136,23 +136,25 @@ impl PySymbolTable { #[pymethod(name = "get_symbols")] fn get_symbols(&self, vm: &VirtualMachine) -> PyResult { - let mut symbols: Vec = vec![]; - for symbol in self.symtable.symbols.values() { - symbols.push( - PySymbol { - symbol: symbol.clone(), + let symbols = self + .symtable + .symbols + .values() + .map(|s| { + (PySymbol { + symbol: s.clone(), namespaces: self .symtable .sub_tables .iter() - .filter(|&table| table.name == symbol.name) + .filter(|&table| table.name == s.name) .cloned() .collect(), - } + }) .into_ref(vm) - .into_object(), - ) - } + .into_object() + }) + .collect(); Ok(vm.ctx.new_list(symbols)) } From 43434d8559cd56440ba95036f3f16340e58cac74 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 2 Aug 2020 19:04:31 +1200 Subject: [PATCH 175/209] Fix zlib decompress --- Lib/test/test_zlib.py | 32 +------ vm/src/stdlib/zlib.rs | 218 ++++++++++++++++++++---------------------- 2 files changed, 108 insertions(+), 142 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 4097b8eaf..759e4b7f3 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -185,8 +185,6 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 @@ -215,8 +213,6 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): def test_big_decompress_buffer(self, size): self.check_big_decompress_buffer(size, zlib.decompress) - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_4G, memuse=1) def test_large_bufsize(self, size): # Test decompress(bufsize) parameter greater than the internal limit @@ -321,8 +317,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y2 = dco.flush() self.assertEqual(data, y1 + y2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompinc(self, flush=False, source=None, cx=256, dcx=64): # compress object in steps, decompress object in steps source = source or HAMLET_SCENE @@ -364,13 +358,9 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertEqual(data, b''.join(bufs)) # Failure means: "decompressobj with init options failed" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompincflush(self): self.test_decompinc(flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompimax(self, source=None, cx=256, dcx=64): # compress in steps, decompress in length-restricted steps source = source or HAMLET_SCENE @@ -398,8 +388,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): bufs.append(dco.flush()) self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompressmaxlen(self, flush=False): # Check a decompression object with max_length specified data = HAMLET_SCENE * 128 @@ -432,8 +420,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): bufs.append(chunk) self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompressmaxlenflush(self): self.test_decompressmaxlen(flush=True) @@ -445,7 +431,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertRaises(ValueError, dco.decompress, b"", -1) self.assertEqual(b'', dco.unconsumed_tail) - @unittest.skip('TODO: RUSTPYTHON') def test_maxlen_large(self): # Sizes up to sys.maxsize should be accepted, although zlib is # internally limited to expressing sizes with unsigned int @@ -540,7 +525,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # if decompressed data is different from the input data, choke. self.assertEqual(expanded, data, "17K random source doesn't match") - @unittest.skip('TODO: RUSTPYTHON') def test_empty_flush(self): # Test that calling .flush() on unused objects works. # (Bug #1083110 -- calling .flush() on decompress objects @@ -584,8 +568,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): self.assertEqual(do.decompress(d1), piece[100:]) self.assertEqual(do.decompress(d2), piece[:-100]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompress_incomplete_stream(self): # This is 'foo', deflated x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' @@ -599,8 +581,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y += dco.flush() self.assertEqual(y, b'foo') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompress_eof(self): x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' dco = zlib.decompressobj() @@ -612,7 +592,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.flush() self.assertTrue(dco.eof) - @unittest.skip('TODO: RUSTPYTHON') def test_decompress_eof_incomplete_stream(self): x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' dco = zlib.decompressobj() @@ -622,8 +601,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.flush() self.assertFalse(dco.eof) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompress_unused_data(self): # Repeated calls to decompress() after EOF should accumulate data in # dco.unused_data, instead of just storing the arg to the last call. @@ -661,7 +638,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): uncomp = dco.decompress(comp) + dco.flush() self.assertEqual(zdict, uncomp) - @unittest.skip('TODO: RUSTPYTHON') def test_flush_with_freed_input(self): # Issue #16411: decompressor accesses input to last decompress() call # in flush(), even if this object has been freed in the meanwhile. @@ -674,7 +650,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): data = zlib.compress(input2) self.assertEqual(dco.flush(), input1[1:]) - @unittest.skip('TODO: RUSTPYTHON') @bigmemtest(size=_4G, memuse=1) def test_flush_large_length(self, size): # Test flush(length) parameter greater than internal limit UINT_MAX @@ -684,7 +659,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): dco.decompress(data, 1) self.assertEqual(dco.flush(size), input[1:]) - @unittest.skip('TODO: RUSTPYTHON') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_flush_custom_length(self): input = HAMLET_SCENE * 10 data = zlib.compress(input, 1) @@ -781,14 +757,12 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): compress = lambda s: c.compress(s) + c.flush() self.check_big_compress_buffer(size, compress) - @unittest.skip('TODO: RUSTPYTHON') @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): d = zlib.decompressobj() decompress = lambda s: d.decompress(s) + d.flush() self.check_big_decompress_buffer(size, decompress) - @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=4) def test_64bit_compress(self, size): @@ -802,7 +776,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): finally: comp = uncomp = data = None - @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=3) def test_large_unused_data(self, size): @@ -817,7 +790,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): finally: unused = comp = do = None - @unittest.skip('TODO: RUSTPYTHON') @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_4G + 100, memuse=5) def test_large_unconsumed_tail(self, size): diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index a95cc81b4..6d6d8064c 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -80,9 +80,7 @@ fn zlib_crc32(data: PyBytesRef, begin_state: OptionalArg, vm: &VirtualMachi } /// Returns a bytes object containing compressed data. -fn zlib_compress(data: PyBytesRef, level: OptionalArg, vm: &VirtualMachine) -> PyResult { - let input_bytes = data.get_value(); - +fn zlib_compress(data: PyBytesLike, level: OptionalArg, vm: &VirtualMachine) -> PyResult { let level = level.unwrap_or(libz::Z_DEFAULT_COMPRESSION); let compression = match level { @@ -94,7 +92,7 @@ fn zlib_compress(data: PyBytesRef, level: OptionalArg, vm: &VirtualMachine) }; let mut encoder = ZlibEncoder::new(Vec::new(), compression); - encoder.write_all(input_bytes).unwrap(); + data.with_ref(|input_bytes| encoder.write_all(input_bytes).unwrap()); let encoded_bytes = encoder.finish().unwrap(); Ok(vm.ctx.new_bytes(encoded_bytes)) @@ -106,46 +104,92 @@ fn header_from_wbits(wbits: OptionalArg) -> (bool, u8) { (wbits > 0, wbits.abs() as u8) } -/// Returns a bytes object containing the uncompressed data. -fn zlib_decompress( - data: PyBytesRef, - wbits: OptionalArg, - bufsize: OptionalArg, +fn decompress( + data: &[u8], + d: &mut Decompress, + bufsize: usize, + max_length: Option, vm: &VirtualMachine, -) -> PyResult> { - let data = data.get_value(); - - let (header, wbits) = header_from_wbits(wbits); - let bufsize = bufsize.unwrap_or(DEF_BUF_SIZE); - - let mut d = Decompress::new_with_window_bits(header, wbits); +) -> PyResult<(Vec, bool)> { + if data.is_empty() { + return Ok((Vec::new(), true)); + } + let orig_in = d.total_in(); let mut buf = Vec::new(); - // TODO: maybe deduplicate this with the Decompress.{decompress,flush} - 'outer: for chunk in data.chunks(CHUNKSIZE) { + for mut chunk in data.chunks(CHUNKSIZE) { // if this is the final chunk, finish it - let flush = if d.total_in() == (data.len() - chunk.len()) as u64 { + let flush = if d.total_in() - orig_in == (data.len() - chunk.len()) as u64 { FlushDecompress::Finish } else { FlushDecompress::None }; loop { - buf.reserve(bufsize); + let additional = if let Some(max_length) = max_length { + std::cmp::min(bufsize, max_length - buf.capacity()) + } else { + bufsize + }; + + buf.reserve_exact(additional); + let prev_in = d.total_in(); match d.decompress_vec(chunk, &mut buf, flush) { - // we've run out of space, loop again and allocate more - Ok(_) if buf.len() == buf.capacity() => {} // we've reached the end of the stream, we're done Ok(Status::StreamEnd) => { - break 'outer; + buf.shrink_to_fit(); + return Ok((buf, true)); + } + // we have hit the maximum length that we can decompress, so stop + Ok(_) if max_length.map_or(false, |max_length| buf.len() == max_length) => { + return Ok((buf, false)); + } + Ok(_) => { + chunk = &chunk[(d.total_in() - prev_in) as usize..]; + + if !chunk.is_empty() { + // there is more input to process + continue; + } else if flush == FlushDecompress::Finish { + if buf.len() == buf.capacity() { + // we've run out of space, loop again and allocate more room + continue; + } else { + // we need more input to continue + buf.shrink_to_fit(); + return Ok((buf, false)); + } + } else { + // progress onto next chunk + break; + } } - // we've reached the end of this chunk of the data, do the next one - Ok(_) => break, Err(_) => return Err(zlib_error("invalid input data", vm)), } } } - buf.shrink_to_fit(); - Ok(buf) + unreachable!("Didn't reach end of stream or capacity limit") +} + +/// Returns a bytes object containing the uncompressed data. +fn zlib_decompress( + data: PyBytesLike, + wbits: OptionalArg, + bufsize: OptionalArg, + vm: &VirtualMachine, +) -> PyResult> { + data.with_ref(|data| { + let (header, wbits) = header_from_wbits(wbits); + let bufsize = bufsize.unwrap_or(DEF_BUF_SIZE); + + let mut d = Decompress::new_with_window_bits(header, wbits); + decompress(data, &mut d, bufsize, None, vm).and_then(|(buf, stream_end)| { + if stream_end { + Ok(buf) + } else { + Err(zlib_error("incomplete or truncated stream", vm)) + } + }) + }) } fn zlib_decompressobj( @@ -193,7 +237,7 @@ impl PyDecompress { self.unconsumed_tail.lock().clone() } - fn save_unconsumed_input( + fn save_unused_input( &self, d: &mut Decompress, data: &[u8], @@ -213,67 +257,41 @@ impl PyDecompress { .collect(); *unused_data = PyBytes::new(unused).into_ref(vm); } - - let mut unconsumed_tail = self.unconsumed_tail.lock(); - if !leftover.is_empty() || unconsumed_tail.len() > 0 { - *unconsumed_tail = PyBytes::new(leftover.to_owned()).into_ref(vm); - } } #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let limited = args.max_length == 0; + let max_length = if args.max_length == 0 { + None + } else { + Some(args.max_length) + }; let data = args.data.get_value(); let mut d = self.decompress.lock(); - let orig_in = d.total_in(); - let mut buf = Vec::new(); - let mut stream_end = false; - 'outer: for chunk in data.chunks(CHUNKSIZE) { - // if this is the final chunk, finish it - let flush = if d.total_in() - orig_in == (data.len() - chunk.len()) as u64 { - FlushDecompress::Finish - } else { - FlushDecompress::None - }; - loop { - let additional = if limited { - std::cmp::min(DEF_BUF_SIZE, args.max_length - buf.capacity()) - } else { - DEF_BUF_SIZE - }; - buf.reserve(additional); - match d.decompress_vec(chunk, &mut buf, flush) { - // we've run out of space - Ok(_) if buf.len() == buf.capacity() => { - if limited && buf.len() == args.max_length { - // if we have a maximum length we can decompress and we've hit it, stop - break 'outer; - } else { - // otherwise, loop again and allocate more - continue; - } - } - // we've reached the end of the stream, we're done - Ok(Status::StreamEnd) => { - stream_end = true; - self.eof.store(true); - break 'outer; - } - // we've reached the end of this chunk of the data, do the next one - Ok(_) => break, - Err(_) => { - self.save_unconsumed_input(&mut d, data, stream_end, orig_in, vm); - return Err(zlib_error("invalid input data", vm)); - } - } + let (ret, stream_end) = match decompress(data, &mut d, DEF_BUF_SIZE, max_length, vm) { + Ok((buf, true)) => { + self.eof.store(true); + (Ok(buf), true) } + Ok((buf, false)) => (Ok(buf), false), + Err(err) => (Err(err), false), + }; + self.save_unused_input(&mut d, data, stream_end, orig_in, vm); + + let leftover = if !stream_end { + &data[(d.total_in() - orig_in) as usize..] + } else { + b"" + }; + let mut unconsumed_tail = self.unconsumed_tail.lock(); + if !leftover.is_empty() || unconsumed_tail.len() > 0 { + *unconsumed_tail = PyBytes::new(leftover.to_owned()).into_ref(vm); } - buf.shrink_to_fit(); - self.save_unconsumed_input(&mut d, data, stream_end, orig_in, vm); - Ok(buf) + + ret } #[pymethod] @@ -289,48 +307,24 @@ impl PyDecompress { OptionalArg::Missing => DEF_BUF_SIZE, }; - let data = self.unconsumed_tail.lock(); + let mut data = self.unconsumed_tail.lock(); let mut d = self.decompress.lock(); let orig_in = d.total_in(); - let mut buf = Vec::new(); - let mut stream_end = false; - 'outer: for chunk in data.chunks(CHUNKSIZE) { - // if this is the final chunk, finish it - let flush = if d.total_in() - orig_in == (data.len() - chunk.len()) as u64 { - FlushDecompress::Finish - } else { - FlushDecompress::None - }; - loop { - buf.reserve(length); - match d.decompress_vec(chunk, &mut buf, flush) { - // we've run out of space, loop again and allocate more - Ok(_) if buf.len() == buf.capacity() => {} - // we've reached the end of the stream, we're done - Ok(Status::StreamEnd) => { - stream_end = true; - self.eof.store(true); - // self->is_initialised = 0; - break 'outer; - } - // we've reached the end of this chunk of the data, do the next one - Ok(_) => break, - Err(_) => { - self.save_unconsumed_input(&mut d, &data, stream_end, orig_in, vm); - return Err(zlib_error("invalid input data", vm)); - } - } - } - } - buf.shrink_to_fit(); - self.save_unconsumed_input(&mut d, &data, stream_end, orig_in, vm); + let (ret, stream_end) = match decompress(&data, &mut d, length, None, vm) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + self.save_unused_input(&mut d, &data, stream_end, orig_in, vm); + + *data = PyBytes::new(Vec::new()).into_ref(vm); + // TODO: drop the inner decompressor, somehow // if stream_end { // // } - Ok(buf) + ret } } From fa1a436a8f5dcaec496e0480d69a18c46ab43ae4 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 2 Aug 2020 19:26:06 +1200 Subject: [PATCH 176/209] Fix test_zipimport.py from CPython 3.8.5 --- Lib/test/test_zipimport.py | 777 +++++++++++++++++++++++++++++++++++++ 1 file changed, 777 insertions(+) create mode 100644 Lib/test/test_zipimport.py diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py new file mode 100644 index 000000000..2af8689c1 --- /dev/null +++ b/Lib/test/test_zipimport.py @@ -0,0 +1,777 @@ +import sys +import os +import marshal +import importlib +import importlib.util +import struct +import time +import unittest +import unittest.mock + +from test import support + +from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED + +import zipimport +import linecache +import doctest +import inspect +import io +from traceback import extract_tb, extract_stack, print_tb +try: + import zlib +except ImportError: + zlib = None + +test_src = """\ +def get_name(): + return __name__ +def get_file(): + return __file__ +""" +test_co = compile(test_src, "", "exec") +raise_src = 'def do_raise(): raise TypeError\n' + +def make_pyc(co, mtime, size): + data = marshal.dumps(co) + if type(mtime) is type(0.0): + # Mac mtimes need a bit of special casing + if mtime < 0x7fffffff: + mtime = int(mtime) + else: + mtime = int(-0x100000000 + int(mtime)) + pyc = (importlib.util.MAGIC_NUMBER + + struct.pack("", "exec"), NOW, len(src)) + files = {TESTMOD + pyc_ext: (NOW, pyc), + "some.data": (NOW, "some data")} + self.doTest(pyc_ext, files, TESTMOD) + + def testDefaultOptimizationLevel(self): + # zipimport should use the default optimization level (#28131) + src = """if 1: # indent hack + def test(val): + assert(val) + return val\n""" + files = {TESTMOD + '.py': (NOW, src)} + self.makeZip(files) + sys.path.insert(0, TEMP_ZIP) + mod = importlib.import_module(TESTMOD) + self.assertEqual(mod.test(1), 1) + self.assertRaises(AssertionError, mod.test, False) + + def testImport_WithStuff(self): + # try importing from a zipfile which contains additional + # stuff at the beginning of the file + files = {TESTMOD + ".py": (NOW, test_src)} + self.doTest(".py", files, TESTMOD, + stuff=b"Some Stuff"*31) + + def assertModuleSource(self, module): + self.assertEqual(inspect.getsource(module), test_src) + + def testGetSource(self): + files = {TESTMOD + ".py": (NOW, test_src)} + self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) + + def testGetCompiledSource(self): + pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src)) + files = {TESTMOD + ".py": (NOW, test_src), + TESTMOD + pyc_ext: (NOW, pyc)} + self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) + + def runDoctest(self, callback): + files = {TESTMOD + ".py": (NOW, test_src), + "xyz.txt": (NOW, ">>> log.append(True)\n")} + self.doTest(".py", files, TESTMOD, call=callback) + + def doDoctestFile(self, module): + log = [] + old_master, doctest.master = doctest.master, None + try: + doctest.testfile( + 'xyz.txt', package=module, module_relative=True, + globs=locals() + ) + finally: + doctest.master = old_master + self.assertEqual(log,[True]) + + def testDoctestFile(self): + self.runDoctest(self.doDoctestFile) + + def doDoctestSuite(self, module): + log = [] + doctest.DocFileTest( + 'xyz.txt', package=module, module_relative=True, + globs=locals() + ).run() + self.assertEqual(log,[True]) + + def testDoctestSuite(self): + self.runDoctest(self.doDoctestSuite) + + def doTraceback(self, module): + try: + module.do_raise() + except: + tb = sys.exc_info()[2].tb_next + + f,lno,n,line = extract_tb(tb, 1)[0] + self.assertEqual(line, raise_src.strip()) + + f,lno,n,line = extract_stack(tb.tb_frame, 1)[0] + self.assertEqual(line, raise_src.strip()) + + s = io.StringIO() + print_tb(tb, 1, s) + self.assertTrue(s.getvalue().endswith(raise_src)) + else: + raise AssertionError("This ought to be impossible") + + def testTraceback(self): + files = {TESTMOD + ".py": (NOW, raise_src)} + self.doTest(None, files, TESTMOD, call=self.doTraceback) + + @unittest.skipIf(support.TESTFN_UNENCODABLE is None, + "need an unencodable filename") + def testUnencodable(self): + filename = support.TESTFN_UNENCODABLE + ".zip" + self.addCleanup(support.unlink, filename) + with ZipFile(filename, "w") as z: + zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) + zinfo.compress_type = self.compression + z.writestr(zinfo, test_src) + zipimport.zipimporter(filename).load_module(TESTMOD) + + def testBytesPath(self): + filename = support.TESTFN + ".zip" + self.addCleanup(support.unlink, filename) + with ZipFile(filename, "w") as z: + zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) + zinfo.compress_type = self.compression + z.writestr(zinfo, test_src) + + zipimport.zipimporter(filename) + zipimport.zipimporter(os.fsencode(filename)) + with self.assertRaises(TypeError): + zipimport.zipimporter(bytearray(os.fsencode(filename))) + with self.assertRaises(TypeError): + zipimport.zipimporter(memoryview(os.fsencode(filename))) + + def testComment(self): + files = {TESTMOD + ".py": (NOW, test_src)} + self.doTest(".py", files, TESTMOD, comment=b"comment") + + def testBeginningCruftAndComment(self): + files = {TESTMOD + ".py": (NOW, test_src)} + self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") + + def testLargestPossibleComment(self): + files = {TESTMOD + ".py": (NOW, test_src)} + self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + + +@support.requires_zlib +class CompressedZipImportTestCase(UncompressedZipImportTestCase): + compression = ZIP_DEFLATED + + +class BadFileZipImportTestCase(unittest.TestCase): + def assertZipFailure(self, filename): + self.assertRaises(zipimport.ZipImportError, + zipimport.zipimporter, filename) + + def testNoFile(self): + self.assertZipFailure('AdfjdkFJKDFJjdklfjs') + + def testEmptyFilename(self): + self.assertZipFailure('') + + def testBadArgs(self): + self.assertRaises(TypeError, zipimport.zipimporter, None) + self.assertRaises(TypeError, zipimport.zipimporter, TESTMOD, kwd=None) + self.assertRaises(TypeError, zipimport.zipimporter, + list(os.fsencode(TESTMOD))) + + def testFilenameTooLong(self): + self.assertZipFailure('A' * 33000) + + def testEmptyFile(self): + support.unlink(TESTMOD) + support.create_empty_file(TESTMOD) + self.assertZipFailure(TESTMOD) + + def testFileUnreadable(self): + support.unlink(TESTMOD) + fd = os.open(TESTMOD, os.O_CREAT, 000) + try: + os.close(fd) + + with self.assertRaises(zipimport.ZipImportError) as cm: + zipimport.zipimporter(TESTMOD) + finally: + # If we leave "the read-only bit" set on Windows, nothing can + # delete TESTMOD, and later tests suffer bogus failures. + os.chmod(TESTMOD, 0o666) + support.unlink(TESTMOD) + + def testNotZipFile(self): + support.unlink(TESTMOD) + fp = open(TESTMOD, 'w+') + fp.write('a' * 22) + fp.close() + self.assertZipFailure(TESTMOD) + + # XXX: disabled until this works on Big-endian machines + def _testBogusZipFile(self): + support.unlink(TESTMOD) + fp = open(TESTMOD, 'w+') + fp.write(struct.pack('=I', 0x06054B50)) + fp.write('a' * 18) + fp.close() + z = zipimport.zipimporter(TESTMOD) + + try: + self.assertRaises(TypeError, z.find_module, None) + self.assertRaises(TypeError, z.load_module, None) + self.assertRaises(TypeError, z.is_package, None) + self.assertRaises(TypeError, z.get_code, None) + self.assertRaises(TypeError, z.get_data, None) + self.assertRaises(TypeError, z.get_source, None) + + error = zipimport.ZipImportError + self.assertEqual(z.find_module('abc'), None) + + self.assertRaises(error, z.load_module, 'abc') + self.assertRaises(error, z.get_code, 'abc') + self.assertRaises(OSError, z.get_data, 'abc') + self.assertRaises(error, z.get_source, 'abc') + self.assertRaises(error, z.is_package, 'abc') + finally: + zipimport._zip_directory_cache.clear() + + +def test_main(): + try: + support.run_unittest( + UncompressedZipImportTestCase, + CompressedZipImportTestCase, + BadFileZipImportTestCase, + ) + finally: + support.unlink(TESTMOD) + +if __name__ == "__main__": + test_main() From dc2ca629a8a6cc5a12836c3b9d48cab6f19f3984 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 2 Aug 2020 19:47:55 +1200 Subject: [PATCH 177/209] Mark failing test_zipimport.py tests. --- Lib/test/test_zipimport.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 2af8689c1..89bbc7020 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -191,6 +191,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): TESTMOD + pyc_ext: (NOW, test_pyc)} self.doTest(pyc_ext, files, TESTMOD) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testUncheckedHashBasedPyc(self): source = b"state = 'old'" source_hash = importlib.util.source_hash(source) @@ -205,6 +207,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEqual(mod.state, 'old') self.doTest(None, files, TESTMOD, call=check) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.mock.patch('_imp.check_hash_based_pycs', 'always') def test_checked_hash_based_change_pyc(self): source = b"state = 'old'" @@ -224,6 +228,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): files = {TESTMOD + ".py": (NOW, "")} self.doTest(None, files, TESTMOD) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) @@ -232,6 +238,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): TESTMOD + pyc_ext: (NOW, badmagic_pyc)} self.doTest(".py", files, TESTMOD) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) @@ -244,6 +252,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): else: self.fail("expected ImportError; import from bad pyc") + # TODO: RUSTPYTHON + @unittest.expectedFailure def testBadMTime(self): badtime_pyc = bytearray(test_pyc) # flip the second bit -- not the first as that one isn't stored in the @@ -713,6 +723,8 @@ class BadFileZipImportTestCase(unittest.TestCase): support.create_empty_file(TESTMOD) self.assertZipFailure(TESTMOD) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testFileUnreadable(self): support.unlink(TESTMOD) fd = os.open(TESTMOD, os.O_CREAT, 000) From d9f507e4a3d0f57702cd71e5fee6989c844d7a27 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 2 Aug 2020 17:52:37 +0900 Subject: [PATCH 178/209] Add target_is_directory parameter for symlink function --- vm/src/stdlib/os.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b03479ab3..690472aff 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -156,6 +156,12 @@ pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { convert_io_error(vm, io::Error::last_os_error()) } +#[derive(FromArgs, Default)] +pub struct TargetIsDirectory { + #[pyarg(keyword_only, default = "false")] + target_is_directory: bool, +} + #[derive(FromArgs, Default)] pub struct DirFd { #[pyarg(keyword_only, default = "None")] @@ -596,6 +602,7 @@ mod _os { pub(super) fn symlink( src: PyPathLike, dst: PyPathLike, + _target_is_directory: TargetIsDirectory, dir_fd: DirFd, vm: &VirtualMachine, ) -> PyResult<()> { @@ -1196,6 +1203,7 @@ mod posix { pub(super) fn symlink( src: PyPathLike, dst: PyPathLike, + _target_is_directory: TargetIsDirectory, dir_fd: DirFd, vm: &VirtualMachine, ) -> PyResult<()> { @@ -2183,6 +2191,7 @@ mod nt { pub(super) fn symlink( src: PyPathLike, dst: PyPathLike, + _target_is_directory: TargetIsDirectory, _dir_fd: DirFd, vm: &VirtualMachine, ) -> PyResult<()> { From 7bbc729477de2b204817281b785f864bff5b4e04 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 3 Aug 2020 00:31:47 +0900 Subject: [PATCH 179/209] BufferedIO.read avoid clone --- vm/src/stdlib/io.rs | 66 ++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 18112ecb9..a2acff91d 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -26,7 +26,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; fn byte_count(bytes: OptionalOption) -> i64 { - bytes.flatten().unwrap_or(-1 as i64) + bytes.flatten().unwrap_or(-1) } fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] @@ -83,28 +83,32 @@ impl BufferedIO { } //Read k bytes from the object and return. - fn read(&mut self, bytes: i64) -> Option> { - let mut buffer = Vec::new(); - + fn read(&mut self, bytes: Option) -> Option> { //for a defined number of bytes, i.e. bytes != -1 - if bytes >= 0 { - let mut handle = self.cursor.clone().take(bytes as u64); - //read handle into buffer - - if handle.read_to_end(&mut buffer).is_err() { - return None; + match bytes.and_then(|v| v.to_usize()) { + Some(bytes) => { + let mut buffer = unsafe { + // Do not move or edit any part of this block without a safety validation. + // `set_len` is guaranteed to be safe only when the new length is less than or equal to the capacity + let mut buffer = Vec::with_capacity(bytes); + buffer.set_len(bytes); + buffer + }; + //read handle into buffer + self.cursor + .read_exact(&mut buffer) + .map_or(None, |_| Some(buffer)) } - //the take above consumes the struct value - //we add this back in with the takes into_inner method - self.cursor = handle.into_inner(); - } else { - //read handle into buffer - if self.cursor.read_to_end(&mut buffer).is_err() { - return None; + None => { + let mut buffer = Vec::new(); + //read handle into buffer + if self.cursor.read_to_end(&mut buffer).is_err() { + None + } else { + Some(buffer) + } } - }; - - Some(buffer) + } } fn tell(&self) -> u64 { @@ -209,7 +213,7 @@ impl PyStringIORef { //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - let data = match self.buffer(vm)?.read(byte_count(bytes)) { + let data = match self.buffer(vm)?.read(bytes.flatten()) { Some(value) => value, None => Vec::new(), }; @@ -263,11 +267,12 @@ fn string_io_new( _args: StringIOArgs, vm: &VirtualMachine, ) -> PyResult { - let flatten = object.flatten(); - let input = flatten.map_or_else(Vec::new, |v| objstr::borrow_value(&v).as_bytes().to_vec()); + let raw_bytes = object + .flatten() + .map_or_else(Vec::new, |v| objstr::borrow_value(&v).as_bytes().to_vec()); PyStringIO { - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(input))), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), closed: AtomicCell::new(false), } .into_ref_with_type(vm, cls) @@ -312,7 +317,7 @@ impl PyBytesIORef { //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - match self.buffer(vm)?.read(byte_count(bytes)) { + match self.buffer(vm)?.read(bytes.flatten()) { Some(value) => Ok(vm.ctx.new_bytes(value)), None => Err(vm.new_value_error("Error Retrieving Value".to_owned())), } @@ -363,10 +368,9 @@ fn bytes_io_new( object: OptionalArg>, vm: &VirtualMachine, ) -> PyResult { - let raw_bytes = match object { - OptionalArg::Present(Some(ref input)) => input.get_value().to_vec(), - _ => vec![], - }; + let raw_bytes = object + .flatten() + .map_or_else(Vec::new, |input| input.get_value().to_vec()); PyBytesIO { buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), @@ -1446,7 +1450,7 @@ mod tests { cursor: Cursor::new(data.clone()), }; - assert_eq!(buffered.read(bytes).unwrap(), data); + assert_eq!(buffered.read(Some(bytes)).unwrap(), data); } #[test] @@ -1458,7 +1462,7 @@ mod tests { }; assert_eq!(buffered.seek(SeekFrom::Start(count)).unwrap(), count); - assert_eq!(buffered.read(count.clone() as i64).unwrap(), vec![3, 4]); + assert_eq!(buffered.read(Some(count as i64)).unwrap(), vec![3, 4]); } #[test] From ba8c0d9a82d232ad30450124acfadbff8179e988 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 00:14:11 -0500 Subject: [PATCH 180/209] Use sys.excepthook when exceptions bubble to the top level --- examples/freeze/main.rs | 2 +- examples/mini_repl.rs | 2 +- src/main.rs | 2 +- src/shell.rs | 2 +- vm/src/builtins.rs | 3 ++- vm/src/exceptions.rs | 55 ++++++++++++++++++++++++++++++++--------- vm/src/frame.rs | 4 ++- vm/src/lib.rs | 1 + vm/src/py_io.rs | 30 ++++++++++++++++++++++ vm/src/sysmodule.rs | 43 +++++++++++++++++++++----------- vm/src/vm.rs | 6 ++--- 11 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 vm/src/py_io.rs diff --git a/examples/freeze/main.rs b/examples/freeze/main.rs index 81e8028b2..39a335619 100644 --- a/examples/freeze/main.rs +++ b/examples/freeze/main.rs @@ -19,7 +19,7 @@ fn main() -> vm::pyobject::PyResult<()> { ); if let Err(err) = res { - vm::exceptions::print_exception(&vm, &err) + vm::exceptions::print_exception(&vm, err); } Ok(()) diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index 92d78eb1d..fc514a108 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -103,7 +103,7 @@ fn main() -> vm::pyobject::PyResult<()> { } } Err(e) => { - vm::exceptions::print_exception(&vm, &e); + vm::exceptions::print_exception(&vm, e); } } } diff --git a/src/main.rs b/src/main.rs index 604b891b4..136007857 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,7 +75,7 @@ fn main() { } } } else { - print_exception(&vm, &err); + print_exception(&vm, err); } process::exit(1); } diff --git a/src/shell.rs b/src/shell.rs index 2785c8ae1..c8ca36122 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -131,7 +131,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { repl.save_history(&repl_history_path).unwrap(); return Err(exc); } - print_exception(vm, &exc); + print_exception(vm, exc); } } repl.save_history(&repl_history_path).unwrap(); diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 95fb403b7..ae2cf4723 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -35,6 +35,7 @@ mod decl { use crate::scope::Scope; #[cfg(feature = "rustpython-parser")] use crate::stdlib::ast; + use crate::sysmodule; use crate::vm::VirtualMachine; use rustpython_common::hash::PyHash; @@ -647,7 +648,7 @@ mod decl { pub fn print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { let file = match options.file { Some(f) => f, - None => vm.get_attribute(vm.sys_module.clone(), "stdout")?, + None => sysmodule::get_stdout(vm)?, }; let write = |obj: PyStringRef| vm.call_method(&file, "write", vec![obj.into_object()]); diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index be9833224..56faa5ede 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -5,7 +5,7 @@ use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtraceback::PyTracebackRef; use crate::obj::objtuple::{PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; -use crate::py_serde; +use crate::py_io::{self, Write}; use crate::pyobject::{ PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -13,11 +13,12 @@ use crate::pyobject::{ use crate::slots::PyTpFlags; use crate::types::create_type; use crate::VirtualMachine; +use crate::{py_serde, sysmodule}; use itertools::Itertools; use std::fmt; use std::fs::File; -use std::io::{self, BufRead, BufReader, Write}; +use std::io::{self, BufRead, BufReader}; use crossbeam_utils::atomic::AtomicCell; @@ -146,18 +147,35 @@ impl PyBaseException { } } -/// Print exception chain -pub fn print_exception(vm: &VirtualMachine, exc: &PyBaseExceptionRef) { - let stderr = io::stderr(); - let mut stderr = stderr.lock(); - let _ = write_exception(&mut stderr, vm, exc); +/// Print exception chain by calling sys.excepthook +pub fn print_exception(vm: &VirtualMachine, exc: PyBaseExceptionRef) { + let write_fallback = |exc, errstr| { + if let Ok(stderr) = sysmodule::get_stderr(vm) { + let mut stderr = py_io::PyWriter(stderr, vm); + // if this fails stderr might be closed -- ignore it + let _ = writeln!(stderr, "{}", errstr); + let _ = write_exception(&mut stderr, vm, exc); + } else { + eprintln!("{}\nlost sys.stderr", errstr); + let _ = write_exception(&mut io::stderr(), vm, exc); + } + }; + if let Ok(excepthook) = vm.get_attribute(vm.sys_module.clone(), "excepthook") { + let (exc_type, exc_val, exc_tb) = split(exc, vm); + if let Err(eh_exc) = vm.invoke(&excepthook, vec![exc_type, exc_val, exc_tb]) { + write_fallback(&eh_exc, "Error in sys.excepthook:"); + write_fallback(&eh_exc, "Original exception was:"); + } + } else { + write_fallback(&exc, "missing sys.excepthook"); + } } pub fn write_exception( output: &mut W, vm: &VirtualMachine, exc: &PyBaseExceptionRef, -) -> io::Result<()> { +) -> Result<(), W::Error> { if let Some(cause) = exc.cause() { write_exception(output, vm, &cause)?; writeln!( @@ -175,7 +193,11 @@ pub fn write_exception( write_exception_inner(output, vm, exc) } -fn print_source_line(output: &mut W, filename: &str, lineno: usize) -> io::Result<()> { +fn print_source_line( + output: &mut W, + filename: &str, + lineno: usize, +) -> Result<(), W::Error> { // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/master/Python/traceback.c#L393 // TODO: support different encodings let file = match File::open(filename) { @@ -198,7 +220,10 @@ fn print_source_line(output: &mut W, filename: &str, lineno: usize) -> } /// Print exception occurrence location from traceback element -fn write_traceback_entry(output: &mut W, tb_entry: &PyTracebackRef) -> io::Result<()> { +fn write_traceback_entry( + output: &mut W, + tb_entry: &PyTracebackRef, +) -> Result<(), W::Error> { let filename = tb_entry.frame.code.source_path.to_owned(); writeln!( output, @@ -215,7 +240,7 @@ pub fn write_exception_inner( output: &mut W, vm: &VirtualMachine, exc: &PyBaseExceptionRef, -) -> io::Result<()> { +) -> Result<(), W::Error> { if let Some(tb) = exc.traceback.read().clone() { writeln!(output, "Traceback (most recent call last):")?; for tb in tb.iter() { @@ -341,6 +366,14 @@ impl ExceptionCtor { } } +pub fn split( + exc: PyBaseExceptionRef, + vm: &VirtualMachine, +) -> (PyObjectRef, PyObjectRef, PyObjectRef) { + let tb = exc.traceback().map_or(vm.get_none(), |tb| tb.into_object()); + (exc.class().into_object(), exc.into_object(), tb) +} + /// Similar to PyErr_NormalizeException in CPython pub fn normalize( exc_type: PyObjectRef, diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 02f036b62..da35ba907 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -663,7 +663,9 @@ impl ExecutingFrame<'_> { bytecode::Instruction::PrintExpr => { let expr = self.pop_value(); - let displayhook = vm.get_attribute(vm.sys_module.clone(), "displayhook")?; + let displayhook = vm + .get_attribute(vm.sys_module.clone(), "displayhook") + .map_err(|_| vm.new_runtime_error("lost sys.displayhook".to_owned()))?; vm.invoke(&displayhook, vec![expr])?; Ok(None) diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 246e6f80e..238f664bd 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -68,6 +68,7 @@ mod frozen; pub mod function; pub mod import; pub mod obj; +mod py_io; pub mod py_serde; pub mod pyobject; mod pystr; diff --git a/vm/src/py_io.rs b/vm/src/py_io.rs new file mode 100644 index 000000000..2f85d36ea --- /dev/null +++ b/vm/src/py_io.rs @@ -0,0 +1,30 @@ +use crate::exceptions::PyBaseExceptionRef; +use crate::pyobject::PyObjectRef; +use crate::VirtualMachine; +use std::{fmt, io}; + +pub trait Write { + type Error; + fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error>; +} + +impl Write for W +where + W: io::Write, +{ + type Error = io::Error; + fn write_fmt(&mut self, args: fmt::Arguments) -> io::Result<()> { + ::write_fmt(self, args) + } +} + +pub struct PyWriter<'vm>(pub PyObjectRef, pub &'vm VirtualMachine); + +impl Write for PyWriter<'_> { + type Error = PyBaseExceptionRef; + fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error> { + let PyWriter(obj, vm) = self; + vm.call_method(obj, "write", vec![vm.new_str(args.to_string())]) + .map(drop) + } +} diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index b8efb3bba..eb4fe3757 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,14 +1,11 @@ use std::{env, mem, path}; -use crate::builtins; use crate::frame::FrameRef; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ - IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult, TypeProtocol, -}; -use crate::version; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult}; use crate::vm::{PySettings, VirtualMachine}; +use crate::{builtins, exceptions, py_io, version}; use rustpython_common::hash::{PyHash, PyUHash}; use rustpython_common::rc::PyRc; @@ -196,17 +193,11 @@ fn sys_intern(value: PyStringRef) -> PyStringRef { } fn sys_exc_info(vm: &VirtualMachine) -> PyObjectRef { - let exc_info = match vm.current_exception() { - Some(exception) => vec![ - exception.class().into_object(), - exception.clone().into_object(), - exception - .traceback() - .map_or(vm.get_none(), |tb| tb.into_object()), - ], - None => vec![vm.get_none(), vm.get_none(), vm.get_none()], + let (ty, val, tb) = match vm.current_exception() { + Some(exception) => exceptions::split(exception, vm), + None => (vm.get_none(), vm.get_none(), vm.get_none()), }; - vm.ctx.new_tuple(exc_info) + vm.ctx.new_tuple(vec![ty, val, tb]) } fn sys_git_info(vm: &VirtualMachine) -> PyObjectRef { @@ -312,6 +303,26 @@ fn sys_getwindowsversion(vm: &VirtualMachine) -> PyResult PyResult { + vm.get_attribute(vm.sys_module.clone(), "stdout") + .map_err(|_| vm.new_runtime_error("lost sys.stdout".to_owned())) +} +pub fn get_stderr(vm: &VirtualMachine) -> PyResult { + vm.get_attribute(vm.sys_module.clone(), "stderr") + .map_err(|_| vm.new_runtime_error("lost sys.stderr".to_owned())) +} + +fn sys_excepthook( + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + let exc = exceptions::normalize(exc_type, exc_val, exc_tb, vm)?; + let stderr = get_stderr(vm)?; + exceptions::write_exception(&mut py_io::PyWriter(stderr, vm), vm, &exc) +} + const PLATFORM: &str = { cfg_if::cfg_if! { if #[cfg(any(target_os = "linux", target_os = "android"))] { @@ -553,6 +564,8 @@ settrace() -- set the global debug tracing function "audit" => ctx.new_function(sys_audit), "displayhook" => ctx.new_function(sys_displayhook), "__displayhook__" => ctx.new_function(sys_displayhook), + "excepthook" => ctx.new_function(sys_excepthook), + "__excepthook__" => ctx.new_function(sys_excepthook), }); #[cfg(windows)] diff --git a/vm/src/vm.rs b/vm/src/vm.rs index c58486dcc..206e89ba5 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -570,7 +570,7 @@ impl VirtualMachine { } // TODO: #[track_caller] when stabilized - fn _py_panic_failed(&self, exc: &PyBaseExceptionRef, msg: &str) -> ! { + fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] { let show_backtrace = env::var_os("RUST_BACKTRACE").map_or(false, |v| &v != "0"); @@ -598,11 +598,11 @@ impl VirtualMachine { } pub fn unwrap_pyresult(&self, result: PyResult) -> T { result.unwrap_or_else(|exc| { - self._py_panic_failed(&exc, "called `vm.unwrap_pyresult()` on an `Err` value") + self._py_panic_failed(exc, "called `vm.unwrap_pyresult()` on an `Err` value") }) } pub fn expect_pyresult(&self, result: PyResult, msg: &str) -> T { - result.unwrap_or_else(|exc| self._py_panic_failed(&exc, msg)) + result.unwrap_or_else(|exc| self._py_panic_failed(exc, msg)) } pub fn new_scope_with_builtins(&self) -> Scope { From e1656149aa3778af14363b72aab8e13a83ee693c Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 20:57:56 -0500 Subject: [PATCH 181/209] Add test/test_sys.py from CPython 3.8.2 --- Lib/test/test_sys.py | 1446 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1446 insertions(+) create mode 100644 Lib/test/test_sys.py diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py new file mode 100644 index 000000000..af0e54bd0 --- /dev/null +++ b/Lib/test/test_sys.py @@ -0,0 +1,1446 @@ +from test import support +from test.support.script_helper import assert_python_ok, assert_python_failure +import builtins +import codecs +import gc +import locale +import operator +import os +import struct +import subprocess +import sys +import sysconfig +import test.support +import textwrap +import unittest +import warnings + + +# count the number of test runs, used to create unique +# strings to intern in test_intern() +INTERN_NUMRUNS = 0 + + +class DisplayHookTest(unittest.TestCase): + + def test_original_displayhook(self): + dh = sys.__displayhook__ + + with support.captured_stdout() as out: + dh(42) + + self.assertEqual(out.getvalue(), "42\n") + self.assertEqual(builtins._, 42) + + del builtins._ + + with support.captured_stdout() as out: + dh(None) + + self.assertEqual(out.getvalue(), "") + self.assertTrue(not hasattr(builtins, "_")) + + # sys.displayhook() requires arguments + self.assertRaises(TypeError, dh) + + stdout = sys.stdout + try: + del sys.stdout + self.assertRaises(RuntimeError, dh, 42) + finally: + sys.stdout = stdout + + def test_lost_displayhook(self): + displayhook = sys.displayhook + try: + del sys.displayhook + code = compile("42", "", "single") + self.assertRaises(RuntimeError, eval, code) + finally: + sys.displayhook = displayhook + + def test_custom_displayhook(self): + def baddisplayhook(obj): + raise ValueError + + with support.swap_attr(sys, 'displayhook', baddisplayhook): + code = compile("42", "", "single") + self.assertRaises(ValueError, eval, code) + + +class ExceptHookTest(unittest.TestCase): + + def test_original_excepthook(self): + try: + raise ValueError(42) + except ValueError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + + self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + + self.assertRaises(TypeError, sys.__excepthook__) + + def test_excepthook_bytes_filename(self): + # bpo-37467: sys.excepthook() must not crash if a filename + # is a bytes string + with warnings.catch_warnings(): + warnings.simplefilter('ignore', BytesWarning) + + try: + raise SyntaxError("msg", (b"bytes_filename", 123, 0, "text")) + except SyntaxError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + + err = err.getvalue() + self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) + self.assertIn(""" text\n""", err) + self.assertTrue(err.endswith("SyntaxError: msg\n")) + + def test_excepthook(self): + with test.support.captured_output("stderr") as stderr: + sys.excepthook(1, '1', 1) + self.assertTrue("TypeError: print_exception(): Exception expected for " \ + "value, str found" in stderr.getvalue()) + + # FIXME: testing the code for a lost or replaced excepthook in + # Python/pythonrun.c::PyErr_PrintEx() is tricky. + + +class SysModuleTest(unittest.TestCase): + + def tearDown(self): + test.support.reap_children() + + def test_exit(self): + # call with two arguments + self.assertRaises(TypeError, sys.exit, 42, 42) + + # call without argument + with self.assertRaises(SystemExit) as cm: + sys.exit() + self.assertIsNone(cm.exception.code) + + rc, out, err = assert_python_ok('-c', 'import sys; sys.exit()') + self.assertEqual(rc, 0) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + # call with integer argument + with self.assertRaises(SystemExit) as cm: + sys.exit(42) + self.assertEqual(cm.exception.code, 42) + + # call with tuple argument with one entry + # entry will be unpacked + with self.assertRaises(SystemExit) as cm: + sys.exit((42,)) + self.assertEqual(cm.exception.code, 42) + + # call with string argument + with self.assertRaises(SystemExit) as cm: + sys.exit("exit") + self.assertEqual(cm.exception.code, "exit") + + # call with tuple argument with two entries + with self.assertRaises(SystemExit) as cm: + sys.exit((17, 23)) + self.assertEqual(cm.exception.code, (17, 23)) + + # test that the exit machinery handles SystemExits properly + rc, out, err = assert_python_failure('-c', 'raise SystemExit(47)') + self.assertEqual(rc, 47) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + def check_exit_message(code, expected, **env_vars): + rc, out, err = assert_python_failure('-c', code, **env_vars) + self.assertEqual(rc, 1) + self.assertEqual(out, b'') + self.assertTrue(err.startswith(expected), + "%s doesn't start with %s" % (ascii(err), ascii(expected))) + + # test that stderr buffer is flushed before the exit message is written + # into stderr + check_exit_message( + r'import sys; sys.stderr.write("unflushed,"); sys.exit("message")', + b"unflushed,message") + + # test that the exit message is written with backslashreplace error + # handler to stderr + check_exit_message( + r'import sys; sys.exit("surrogates:\uDCFF")', + b"surrogates:\\udcff") + + # test that the unicode message is encoded to the stderr encoding + # instead of the default encoding (utf8) + check_exit_message( + r'import sys; sys.exit("h\xe9")', + b"h\xe9", PYTHONIOENCODING='latin-1') + + def test_getdefaultencoding(self): + self.assertRaises(TypeError, sys.getdefaultencoding, 42) + # can't check more than the type, as the user might have changed it + self.assertIsInstance(sys.getdefaultencoding(), str) + + # testing sys.settrace() is done in test_sys_settrace.py + # testing sys.setprofile() is done in test_sys_setprofile.py + + def test_setcheckinterval(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.assertRaises(TypeError, sys.setcheckinterval) + orig = sys.getcheckinterval() + for n in 0, 100, 120, orig: # orig last to restore starting state + sys.setcheckinterval(n) + self.assertEqual(sys.getcheckinterval(), n) + + def test_switchinterval(self): + self.assertRaises(TypeError, sys.setswitchinterval) + self.assertRaises(TypeError, sys.setswitchinterval, "a") + self.assertRaises(ValueError, sys.setswitchinterval, -1.0) + self.assertRaises(ValueError, sys.setswitchinterval, 0.0) + orig = sys.getswitchinterval() + # sanity check + self.assertTrue(orig < 0.5, orig) + try: + for n in 0.00001, 0.05, 3.0, orig: + sys.setswitchinterval(n) + self.assertAlmostEqual(sys.getswitchinterval(), n) + finally: + sys.setswitchinterval(orig) + + def test_recursionlimit(self): + self.assertRaises(TypeError, sys.getrecursionlimit, 42) + oldlimit = sys.getrecursionlimit() + self.assertRaises(TypeError, sys.setrecursionlimit) + self.assertRaises(ValueError, sys.setrecursionlimit, -42) + sys.setrecursionlimit(10000) + self.assertEqual(sys.getrecursionlimit(), 10000) + sys.setrecursionlimit(oldlimit) + + def test_recursionlimit_recovery(self): + if hasattr(sys, 'gettrace') and sys.gettrace(): + self.skipTest('fatal error if run with a trace function') + + oldlimit = sys.getrecursionlimit() + def f(): + f() + try: + for depth in (10, 25, 50, 75, 100, 250, 1000): + try: + sys.setrecursionlimit(depth) + except RecursionError: + # Issue #25274: The recursion limit is too low at the + # current recursion depth + continue + + # Issue #5392: test stack overflow after hitting recursion + # limit twice + self.assertRaises(RecursionError, f) + self.assertRaises(RecursionError, f) + finally: + sys.setrecursionlimit(oldlimit) + + @test.support.cpython_only + def test_setrecursionlimit_recursion_depth(self): + # Issue #25274: Setting a low recursion limit must be blocked if the + # current recursion depth is already higher than the "lower-water + # mark". Otherwise, it may not be possible anymore to + # reset the overflowed flag to 0. + + from _testcapi import get_recursion_depth + + def set_recursion_limit_at_depth(depth, limit): + recursion_depth = get_recursion_depth() + if recursion_depth >= depth: + with self.assertRaises(RecursionError) as cm: + sys.setrecursionlimit(limit) + self.assertRegex(str(cm.exception), + "cannot set the recursion limit to [0-9]+ " + "at the recursion depth [0-9]+: " + "the limit is too low") + else: + set_recursion_limit_at_depth(depth, limit) + + oldlimit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(1000) + + for limit in (10, 25, 50, 75, 100, 150, 200): + # formula extracted from _Py_RecursionLimitLowerWaterMark() + if limit > 200: + depth = limit - 50 + else: + depth = limit * 3 // 4 + set_recursion_limit_at_depth(depth, limit) + finally: + sys.setrecursionlimit(oldlimit) + + def test_recursionlimit_fatalerror(self): + # A fatal error occurs if a second recursion limit is hit when recovering + # from a first one. + code = textwrap.dedent(""" + import sys + + def f(): + try: + f() + except RecursionError: + f() + + sys.setrecursionlimit(%d) + f()""") + with test.support.SuppressCrashReport(): + for i in (50, 1000): + sub = subprocess.Popen([sys.executable, '-c', code % i], + stderr=subprocess.PIPE) + err = sub.communicate()[1] + self.assertTrue(sub.returncode, sub.returncode) + self.assertIn( + b"Fatal Python error: Cannot recover from stack overflow", + err) + + def test_getwindowsversion(self): + # Raise SkipTest if sys doesn't have getwindowsversion attribute + test.support.get_attribute(sys, "getwindowsversion") + v = sys.getwindowsversion() + self.assertEqual(len(v), 5) + self.assertIsInstance(v[0], int) + self.assertIsInstance(v[1], int) + self.assertIsInstance(v[2], int) + self.assertIsInstance(v[3], int) + self.assertIsInstance(v[4], str) + self.assertRaises(IndexError, operator.getitem, v, 5) + self.assertIsInstance(v.major, int) + self.assertIsInstance(v.minor, int) + self.assertIsInstance(v.build, int) + self.assertIsInstance(v.platform, int) + self.assertIsInstance(v.service_pack, str) + self.assertIsInstance(v.service_pack_minor, int) + self.assertIsInstance(v.service_pack_major, int) + self.assertIsInstance(v.suite_mask, int) + self.assertIsInstance(v.product_type, int) + self.assertEqual(v[0], v.major) + self.assertEqual(v[1], v.minor) + self.assertEqual(v[2], v.build) + self.assertEqual(v[3], v.platform) + self.assertEqual(v[4], v.service_pack) + + # This is how platform.py calls it. Make sure tuple + # still has 5 elements + maj, min, buildno, plat, csd = sys.getwindowsversion() + + def test_call_tracing(self): + self.assertRaises(TypeError, sys.call_tracing, type, 2) + + @unittest.skipUnless(hasattr(sys, "setdlopenflags"), + 'test needs sys.setdlopenflags()') + def test_dlopenflags(self): + self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertRaises(TypeError, sys.getdlopenflags, 42) + oldflags = sys.getdlopenflags() + self.assertRaises(TypeError, sys.setdlopenflags) + sys.setdlopenflags(oldflags+1) + self.assertEqual(sys.getdlopenflags(), oldflags+1) + sys.setdlopenflags(oldflags) + + @test.support.refcount_test + def test_refcount(self): + # n here must be a global in order for this test to pass while + # tracing with a python function. Tracing calls PyFrame_FastToLocals + # which will add a copy of any locals to the frame object, causing + # the reference count to increase by 2 instead of 1. + global n + self.assertRaises(TypeError, sys.getrefcount) + c = sys.getrefcount(None) + n = None + self.assertEqual(sys.getrefcount(None), c+1) + del n + self.assertEqual(sys.getrefcount(None), c) + if hasattr(sys, "gettotalrefcount"): + self.assertIsInstance(sys.gettotalrefcount(), int) + + def test_getframe(self): + self.assertRaises(TypeError, sys._getframe, 42, 42) + self.assertRaises(ValueError, sys._getframe, 2000000000) + self.assertTrue( + SysModuleTest.test_getframe.__code__ \ + is sys._getframe().f_code + ) + + # sys._current_frames() is a CPython-only gimmick. + @test.support.reap_threads + def test_current_frames(self): + import threading + import traceback + + # Spawn a thread that blocks at a known place. Then the main + # thread does sys._current_frames(), and verifies that the frames + # returned make sense. + entered_g = threading.Event() + leave_g = threading.Event() + thread_info = [] # the thread's id + + def f123(): + g456() + + def g456(): + thread_info.append(threading.get_ident()) + entered_g.set() + leave_g.wait() + + t = threading.Thread(target=f123) + t.start() + entered_g.wait() + + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_frames() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + + # Verify that the captured main-thread frame is _this_ frame. + frame = d.pop(main_id) + self.assertTrue(frame is sys._getframe()) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a litte tricky, since various bits of + # threading.py are also in the thread's call stack. + frame = d.pop(thread_id) + stack = traceback.extract_stack(frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") + + self.assertEqual(sourceline, "g456()") + + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + + # Reap the spawned thread. + leave_g.set() + t.join() + + def test_attributes(self): + self.assertIsInstance(sys.api_version, int) + self.assertIsInstance(sys.argv, list) + self.assertIn(sys.byteorder, ("little", "big")) + self.assertIsInstance(sys.builtin_module_names, tuple) + self.assertIsInstance(sys.copyright, str) + self.assertIsInstance(sys.exec_prefix, str) + self.assertIsInstance(sys.base_exec_prefix, str) + self.assertIsInstance(sys.executable, str) + self.assertEqual(len(sys.float_info), 11) + self.assertEqual(sys.float_info.radix, 2) + self.assertEqual(len(sys.int_info), 2) + self.assertTrue(sys.int_info.bits_per_digit % 5 == 0) + self.assertTrue(sys.int_info.sizeof_digit >= 1) + self.assertEqual(type(sys.int_info.bits_per_digit), int) + self.assertEqual(type(sys.int_info.sizeof_digit), int) + self.assertIsInstance(sys.hexversion, int) + + self.assertEqual(len(sys.hash_info), 9) + self.assertLess(sys.hash_info.modulus, 2**sys.hash_info.width) + # sys.hash_info.modulus should be a prime; we do a quick + # probable primality test (doesn't exclude the possibility of + # a Carmichael number) + for x in range(1, 100): + self.assertEqual( + pow(x, sys.hash_info.modulus-1, sys.hash_info.modulus), + 1, + "sys.hash_info.modulus {} is a non-prime".format( + sys.hash_info.modulus) + ) + self.assertIsInstance(sys.hash_info.inf, int) + self.assertIsInstance(sys.hash_info.nan, int) + self.assertIsInstance(sys.hash_info.imag, int) + algo = sysconfig.get_config_var("Py_HASH_ALGORITHM") + if sys.hash_info.algorithm in {"fnv", "siphash24"}: + self.assertIn(sys.hash_info.hash_bits, {32, 64}) + self.assertIn(sys.hash_info.seed_bits, {32, 64, 128}) + + if algo == 1: + self.assertEqual(sys.hash_info.algorithm, "siphash24") + elif algo == 2: + self.assertEqual(sys.hash_info.algorithm, "fnv") + else: + self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"}) + else: + # PY_HASH_EXTERNAL + self.assertEqual(algo, 0) + self.assertGreaterEqual(sys.hash_info.cutoff, 0) + self.assertLess(sys.hash_info.cutoff, 8) + + self.assertIsInstance(sys.maxsize, int) + self.assertIsInstance(sys.maxunicode, int) + self.assertEqual(sys.maxunicode, 0x10FFFF) + self.assertIsInstance(sys.platform, str) + self.assertIsInstance(sys.prefix, str) + self.assertIsInstance(sys.base_prefix, str) + self.assertIsInstance(sys.version, str) + vi = sys.version_info + self.assertIsInstance(vi[:], tuple) + self.assertEqual(len(vi), 5) + self.assertIsInstance(vi[0], int) + self.assertIsInstance(vi[1], int) + self.assertIsInstance(vi[2], int) + self.assertIn(vi[3], ("alpha", "beta", "candidate", "final")) + self.assertIsInstance(vi[4], int) + self.assertIsInstance(vi.major, int) + self.assertIsInstance(vi.minor, int) + self.assertIsInstance(vi.micro, int) + self.assertIn(vi.releaselevel, ("alpha", "beta", "candidate", "final")) + self.assertIsInstance(vi.serial, int) + self.assertEqual(vi[0], vi.major) + self.assertEqual(vi[1], vi.minor) + self.assertEqual(vi[2], vi.micro) + self.assertEqual(vi[3], vi.releaselevel) + self.assertEqual(vi[4], vi.serial) + self.assertTrue(vi > (1,0,0)) + self.assertIsInstance(sys.float_repr_style, str) + self.assertIn(sys.float_repr_style, ('short', 'legacy')) + if not sys.platform.startswith('win'): + self.assertIsInstance(sys.abiflags, str) + + def test_thread_info(self): + info = sys.thread_info + self.assertEqual(len(info), 3) + self.assertIn(info.name, ('nt', 'pthread', 'solaris', None)) + self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) + + def test_43581(self): + # Can't use sys.stdout, as this is a StringIO object when + # the test runs under regrtest. + self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) + + def test_intern(self): + global INTERN_NUMRUNS + INTERN_NUMRUNS += 1 + self.assertRaises(TypeError, sys.intern) + s = "never interned before" + str(INTERN_NUMRUNS) + self.assertTrue(sys.intern(s) is s) + s2 = s.swapcase().swapcase() + self.assertTrue(sys.intern(s2) is s) + + # Subclasses of string can't be interned, because they + # provide too much opportunity for insane things to happen. + # We don't want them in the interned dict and if they aren't + # actually interned, we don't want to create the appearance + # that they are by allowing intern() to succeed. + class S(str): + def __hash__(self): + return 123 + + self.assertRaises(TypeError, sys.intern, S("abc")) + + def test_sys_flags(self): + self.assertTrue(sys.flags) + attrs = ("debug", + "inspect", "interactive", "optimize", "dont_write_bytecode", + "no_user_site", "no_site", "ignore_environment", "verbose", + "bytes_warning", "quiet", "hash_randomization", "isolated", + "dev_mode", "utf8_mode") + for attr in attrs: + self.assertTrue(hasattr(sys.flags, attr), attr) + attr_type = bool if attr == "dev_mode" else int + self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) + self.assertTrue(repr(sys.flags)) + self.assertEqual(len(sys.flags), len(attrs)) + + self.assertIn(sys.flags.utf8_mode, {0, 1, 2}) + + def assert_raise_on_new_sys_type(self, sys_attr): + # Users are intentionally prevented from creating new instances of + # sys.flags, sys.version_info, and sys.getwindowsversion. + attr_type = type(sys_attr) + with self.assertRaises(TypeError): + attr_type() + with self.assertRaises(TypeError): + attr_type.__new__(attr_type) + + def test_sys_flags_no_instantiation(self): + self.assert_raise_on_new_sys_type(sys.flags) + + def test_sys_version_info_no_instantiation(self): + self.assert_raise_on_new_sys_type(sys.version_info) + + def test_sys_getwindowsversion_no_instantiation(self): + # Skip if not being run on Windows. + test.support.get_attribute(sys, "getwindowsversion") + self.assert_raise_on_new_sys_type(sys.getwindowsversion()) + + @test.support.cpython_only + def test_clear_type_cache(self): + sys._clear_type_cache() + + def test_ioencoding(self): + env = dict(os.environ) + + # Test character: cent sign, encoded as 0x4A (ASCII J) in CP424, + # not representable in ASCII. + + env["PYTHONIOENCODING"] = "cp424" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout = subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + expected = ("\xa2" + os.linesep).encode("cp424") + self.assertEqual(out, expected) + + env["PYTHONIOENCODING"] = "ascii:replace" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout = subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, b'?') + + env["PYTHONIOENCODING"] = "ascii" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = "ascii:" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = ":surrogateescape" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xdcbd))'], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, b'\xbd') + + @unittest.skipUnless(test.support.FS_NONASCII, + 'requires OS support of non-ASCII encodings') + @unittest.skipUnless(sys.getfilesystemencoding() == locale.getpreferredencoding(False), + 'requires FS encoding to match locale') + def test_ioencoding_nonascii(self): + env = dict(os.environ) + + env["PYTHONIOENCODING"] = "" + p = subprocess.Popen([sys.executable, "-c", + 'print(%a)' % test.support.FS_NONASCII], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, os.fsencode(test.support.FS_NONASCII)) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable(self): + # sys.executable should be absolute + self.assertEqual(os.path.abspath(sys.executable), sys.executable) + + # Issue #7774: Ensure that sys.executable is an empty string if argv[0] + # has been set to a non existent program name and Python is unable to + # retrieve the real program name + + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + python_dir = os.path.dirname(os.path.realpath(sys.executable)) + p = subprocess.Popen( + ["nonexistent", "-c", + 'import sys; print(sys.executable.encode("ascii", "backslashreplace"))'], + executable=sys.executable, stdout=subprocess.PIPE, cwd=python_dir) + stdout = p.communicate()[0] + executable = stdout.strip().decode("ASCII") + p.wait() + self.assertIn(executable, ["b''", repr(sys.executable.encode("ascii", "backslashreplace"))]) + + def check_fsencoding(self, fs_encoding, expected=None): + self.assertIsNotNone(fs_encoding) + codecs.lookup(fs_encoding) + if expected: + self.assertEqual(fs_encoding, expected) + + def test_getfilesystemencoding(self): + fs_encoding = sys.getfilesystemencoding() + if sys.platform == 'darwin': + expected = 'utf-8' + else: + expected = None + self.check_fsencoding(fs_encoding, expected) + + def c_locale_get_error_handler(self, locale, isolated=False, encoding=None): + # Force the POSIX locale + env = os.environ.copy() + env["LC_ALL"] = locale + env["PYTHONCOERCECLOCALE"] = "0" + code = '\n'.join(( + 'import sys', + 'def dump(name):', + ' std = getattr(sys, name)', + ' print("%s: %s" % (name, std.errors))', + 'dump("stdin")', + 'dump("stdout")', + 'dump("stderr")', + )) + args = [sys.executable, "-X", "utf8=0", "-c", code] + if isolated: + args.append("-I") + if encoding is not None: + env['PYTHONIOENCODING'] = encoding + else: + env.pop('PYTHONIOENCODING', None) + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + universal_newlines=True) + stdout, stderr = p.communicate() + return stdout + + def check_locale_surrogateescape(self, locale): + out = self.c_locale_get_error_handler(locale, isolated=True) + self.assertEqual(out, + 'stdin: surrogateescape\n' + 'stdout: surrogateescape\n' + 'stderr: backslashreplace\n') + + # replace the default error handler + out = self.c_locale_get_error_handler(locale, encoding=':ignore') + self.assertEqual(out, + 'stdin: ignore\n' + 'stdout: ignore\n' + 'stderr: backslashreplace\n') + + # force the encoding + out = self.c_locale_get_error_handler(locale, encoding='iso8859-1') + self.assertEqual(out, + 'stdin: strict\n' + 'stdout: strict\n' + 'stderr: backslashreplace\n') + out = self.c_locale_get_error_handler(locale, encoding='iso8859-1:') + self.assertEqual(out, + 'stdin: strict\n' + 'stdout: strict\n' + 'stderr: backslashreplace\n') + + # have no any effect + out = self.c_locale_get_error_handler(locale, encoding=':') + self.assertEqual(out, + 'stdin: surrogateescape\n' + 'stdout: surrogateescape\n' + 'stderr: backslashreplace\n') + out = self.c_locale_get_error_handler(locale, encoding='') + self.assertEqual(out, + 'stdin: surrogateescape\n' + 'stdout: surrogateescape\n' + 'stderr: backslashreplace\n') + + def test_c_locale_surrogateescape(self): + self.check_locale_surrogateescape('C') + + def test_posix_locale_surrogateescape(self): + self.check_locale_surrogateescape('POSIX') + + def test_implementation(self): + # This test applies to all implementations equally. + + levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} + + self.assertTrue(hasattr(sys.implementation, 'name')) + self.assertTrue(hasattr(sys.implementation, 'version')) + self.assertTrue(hasattr(sys.implementation, 'hexversion')) + self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + + version = sys.implementation.version + self.assertEqual(version[:2], (version.major, version.minor)) + + hexversion = (version.major << 24 | version.minor << 16 | + version.micro << 8 | levels[version.releaselevel] << 4 | + version.serial << 0) + self.assertEqual(sys.implementation.hexversion, hexversion) + + # PEP 421 requires that .name be lower case. + self.assertEqual(sys.implementation.name, + sys.implementation.name.lower()) + + @test.support.cpython_only + def test_debugmallocstats(self): + # Test sys._debugmallocstats() + from test.support.script_helper import assert_python_ok + args = ['-c', 'import sys; sys._debugmallocstats()'] + ret, out, err = assert_python_ok(*args) + self.assertIn(b"free PyDictObjects", err) + + # The function has no parameter + self.assertRaises(TypeError, sys._debugmallocstats, True) + + @unittest.skipUnless(hasattr(sys, "getallocatedblocks"), + "sys.getallocatedblocks unavailable on this build") + def test_getallocatedblocks(self): + try: + import _testcapi + except ImportError: + with_pymalloc = support.with_pymalloc() + else: + try: + alloc_name = _testcapi.pymem_getallocatorsname() + except RuntimeError as exc: + # "cannot get allocators name" (ex: tracemalloc is used) + with_pymalloc = True + else: + with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug')) + + # Some sanity checks + a = sys.getallocatedblocks() + self.assertIs(type(a), int) + if with_pymalloc: + self.assertGreater(a, 0) + else: + # When WITH_PYMALLOC isn't available, we don't know anything + # about the underlying implementation: the function might + # return 0 or something greater. + self.assertGreaterEqual(a, 0) + try: + # While we could imagine a Python session where the number of + # multiple buffer objects would exceed the sharing of references, + # it is unlikely to happen in a normal test run. + self.assertLess(a, sys.gettotalrefcount()) + except AttributeError: + # gettotalrefcount() not available + pass + gc.collect() + b = sys.getallocatedblocks() + self.assertLessEqual(b, a) + gc.collect() + c = sys.getallocatedblocks() + self.assertIn(c, range(b - 50, b + 50)) + + @test.support.requires_type_collecting + def test_is_finalizing(self): + self.assertIs(sys.is_finalizing(), False) + # Don't use the atexit module because _Py_Finalizing is only set + # after calling atexit callbacks + code = """if 1: + import sys + + class AtExit: + is_finalizing = sys.is_finalizing + print = print + + def __del__(self): + self.print(self.is_finalizing(), flush=True) + + # Keep a reference in the __main__ module namespace, so the + # AtExit destructor will be called at Python exit + ref = AtExit() + """ + rc, stdout, stderr = assert_python_ok('-c', code) + self.assertEqual(stdout.rstrip(), b'True') + + @test.support.requires_type_collecting + def test_issue20602(self): + # sys.flags and sys.float_info were wiped during shutdown. + code = """if 1: + import sys + class A: + def __del__(self, sys=sys): + print(sys.flags) + print(sys.float_info) + a = A() + """ + rc, out, err = assert_python_ok('-c', code) + out = out.splitlines() + self.assertIn(b'sys.flags', out[0]) + self.assertIn(b'sys.float_info', out[1]) + + @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), + 'need sys.getandroidapilevel()') + def test_getandroidapilevel(self): + level = sys.getandroidapilevel() + self.assertIsInstance(level, int) + self.assertGreater(level, 0) + + def test_sys_tracebacklimit(self): + code = """if 1: + import sys + def f1(): + 1 / 0 + def f2(): + f1() + sys.tracebacklimit = %r + f2() + """ + def check(tracebacklimit, expected): + p = subprocess.Popen([sys.executable, '-c', code % tracebacklimit], + stderr=subprocess.PIPE) + out = p.communicate()[1] + self.assertEqual(out.splitlines(), expected) + + traceback = [ + b'Traceback (most recent call last):', + b' File "", line 8, in ', + b' File "", line 6, in f2', + b' File "", line 4, in f1', + b'ZeroDivisionError: division by zero' + ] + check(10, traceback) + check(3, traceback) + check(2, traceback[:1] + traceback[2:]) + check(1, traceback[:1] + traceback[3:]) + check(0, [traceback[-1]]) + check(-1, [traceback[-1]]) + check(1<<1000, traceback) + check(-1<<1000, [traceback[-1]]) + check(None, traceback) + + def test_no_duplicates_in_meta_path(self): + self.assertEqual(len(sys.meta_path), len(set(sys.meta_path))) + + @unittest.skipUnless(hasattr(sys, "_enablelegacywindowsfsencoding"), + 'needs sys._enablelegacywindowsfsencoding()') + def test__enablelegacywindowsfsencoding(self): + code = ('import sys', + 'sys._enablelegacywindowsfsencoding()', + 'print(sys.getfilesystemencoding(), sys.getfilesystemencodeerrors())') + rc, out, err = assert_python_ok('-c', '; '.join(code)) + out = out.decode('ascii', 'replace').rstrip() + self.assertEqual(out, 'mbcs replace') + + +@test.support.cpython_only +class UnraisableHookTest(unittest.TestCase): + def write_unraisable_exc(self, exc, err_msg, obj): + import _testcapi + import types + err_msg2 = f"Exception ignored {err_msg}" + try: + _testcapi.write_unraisable_exc(exc, err_msg, obj) + return types.SimpleNamespace(exc_type=type(exc), + exc_value=exc, + exc_traceback=exc.__traceback__, + err_msg=err_msg2, + object=obj) + finally: + # Explicitly break any reference cycle + exc = None + + def test_original_unraisablehook(self): + for err_msg in (None, "original hook"): + with self.subTest(err_msg=err_msg): + obj = "an object" + + with test.support.captured_output("stderr") as stderr: + with test.support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + self.write_unraisable_exc(ValueError(42), err_msg, obj) + + err = stderr.getvalue() + if err_msg is not None: + self.assertIn(f'Exception ignored {err_msg}: {obj!r}\n', err) + else: + self.assertIn(f'Exception ignored in: {obj!r}\n', err) + self.assertIn('Traceback (most recent call last):\n', err) + self.assertIn('ValueError: 42\n', err) + + def test_original_unraisablehook_err(self): + # bpo-22836: PyErr_WriteUnraisable() should give sensible reports + class BrokenDel: + def __del__(self): + exc = ValueError("del is broken") + # The following line is included in the traceback report: + raise exc + + class BrokenStrException(Exception): + def __str__(self): + raise Exception("str() is broken") + + class BrokenExceptionDel: + def __del__(self): + exc = BrokenStrException() + # The following line is included in the traceback report: + raise exc + + for test_class in (BrokenDel, BrokenExceptionDel): + with self.subTest(test_class): + obj = test_class() + with test.support.captured_stderr() as stderr, \ + test.support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + # Trigger obj.__del__() + del obj + + report = stderr.getvalue() + self.assertIn("Exception ignored", report) + self.assertIn(test_class.__del__.__qualname__, report) + self.assertIn("test_sys.py", report) + self.assertIn("raise exc", report) + if test_class is BrokenExceptionDel: + self.assertIn("BrokenStrException", report) + self.assertIn("", report) + else: + self.assertIn("ValueError", report) + self.assertIn("del is broken", report) + self.assertTrue(report.endswith("\n")) + + + def test_original_unraisablehook_wrong_type(self): + exc = ValueError(42) + with test.support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + with self.assertRaises(TypeError): + sys.unraisablehook(exc) + + def test_custom_unraisablehook(self): + hook_args = None + + def hook_func(args): + nonlocal hook_args + hook_args = args + + obj = object() + try: + with test.support.swap_attr(sys, 'unraisablehook', hook_func): + expected = self.write_unraisable_exc(ValueError(42), + "custom hook", obj) + for attr in "exc_type exc_value exc_traceback err_msg object".split(): + self.assertEqual(getattr(hook_args, attr), + getattr(expected, attr), + (hook_args, expected)) + finally: + # expected and hook_args contain an exception: break reference cycle + expected = None + hook_args = None + + def test_custom_unraisablehook_fail(self): + def hook_func(*args): + raise Exception("hook_func failed") + + with test.support.captured_output("stderr") as stderr: + with test.support.swap_attr(sys, 'unraisablehook', hook_func): + self.write_unraisable_exc(ValueError(42), + "custom hook fail", None) + + err = stderr.getvalue() + self.assertIn(f'Exception ignored in sys.unraisablehook: ' + f'{hook_func!r}\n', + err) + self.assertIn('Traceback (most recent call last):\n', err) + self.assertIn('Exception: hook_func failed\n', err) + + +@test.support.cpython_only +class SizeofTest(unittest.TestCase): + + def setUp(self): + self.P = struct.calcsize('P') + self.longdigit = sys.int_info.sizeof_digit + import _testcapi + self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD + + check_sizeof = test.support.check_sizeof + + def test_gc_head_size(self): + # Check that the gc header size is added to objects tracked by the gc. + vsize = test.support.calcvobjsize + gc_header_size = self.gc_headsize + # bool objects are not gc tracked + self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit) + # but lists are + self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size) + + def test_errors(self): + class BadSizeof: + def __sizeof__(self): + raise ValueError + self.assertRaises(ValueError, sys.getsizeof, BadSizeof()) + + class InvalidSizeof: + def __sizeof__(self): + return None + self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof()) + sentinel = ["sentinel"] + self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel) + + class FloatSizeof: + def __sizeof__(self): + return 4.5 + self.assertRaises(TypeError, sys.getsizeof, FloatSizeof()) + self.assertIs(sys.getsizeof(FloatSizeof(), sentinel), sentinel) + + class OverflowSizeof(int): + def __sizeof__(self): + return int(self) + self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), + sys.maxsize + self.gc_headsize) + with self.assertRaises(OverflowError): + sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) + with self.assertRaises(ValueError): + sys.getsizeof(OverflowSizeof(-1)) + with self.assertRaises((ValueError, OverflowError)): + sys.getsizeof(OverflowSizeof(-sys.maxsize - 1)) + + def test_default(self): + size = test.support.calcvobjsize + self.assertEqual(sys.getsizeof(True), size('') + self.longdigit) + self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit) + + def test_objecttypes(self): + # check all types defined in Objects/ + calcsize = struct.calcsize + size = test.support.calcobjsize + vsize = test.support.calcvobjsize + check = self.check_sizeof + # bool + check(True, vsize('') + self.longdigit) + # buffer + # XXX + # builtin_function_or_method + check(len, size('5P')) + # bytearray + samples = [b'', b'u'*100000] + for sample in samples: + x = bytearray(sample) + check(x, vsize('n2Pi') + x.__alloc__()) + # bytearray_iterator + check(iter(bytearray()), size('nP')) + # bytes + check(b'', vsize('n') + 1) + check(b'x' * 10, vsize('n') + 11) + # cell + def get_cell(): + x = 42 + def inner(): + return x + return inner + check(get_cell().__closure__[0], size('P')) + # code + def check_code_size(a, expected_size): + self.assertGreaterEqual(sys.getsizeof(a), expected_size) + check_code_size(get_cell().__code__, size('6i13P')) + check_code_size(get_cell.__code__, size('6i13P')) + def get_cell2(x): + def inner(): + return x + return inner + check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n')) + # complex + check(complex(0,1), size('2d')) + # method_descriptor (descriptor object) + check(str.lower, size('3PPP')) + # classmethod_descriptor (descriptor object) + # XXX + # member_descriptor (descriptor object) + import datetime + check(datetime.timedelta.days, size('3PP')) + # getset_descriptor (descriptor object) + import collections + check(collections.defaultdict.default_factory, size('3PP')) + # wrapper_descriptor (descriptor object) + check(int.__add__, size('3P2P')) + # method-wrapper (descriptor object) + check({}.__iter__, size('2P')) + # empty dict + check({}, size('nQ2P')) + # dict + check({"a": 1}, size('nQ2P') + calcsize('2nP2n') + 8 + (8*2//3)*calcsize('n2P')) + longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} + check(longdict, size('nQ2P') + calcsize('2nP2n') + 16 + (16*2//3)*calcsize('n2P')) + # dictionary-keyview + check({}.keys(), size('P')) + # dictionary-valueview + check({}.values(), size('P')) + # dictionary-itemview + check({}.items(), size('P')) + # dictionary iterator + check(iter({}), size('P2nPn')) + # dictionary-keyiterator + check(iter({}.keys()), size('P2nPn')) + # dictionary-valueiterator + check(iter({}.values()), size('P2nPn')) + # dictionary-itemiterator + check(iter({}.items()), size('P2nPn')) + # dictproxy + class C(object): pass + check(C.__dict__, size('P')) + # BaseException + check(BaseException(), size('5Pb')) + # UnicodeEncodeError + check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pb 2P2nP')) + # UnicodeDecodeError + check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pb 2P2nP')) + # UnicodeTranslateError + check(UnicodeTranslateError("", 0, 1, ""), size('5Pb 2P2nP')) + # ellipses + check(Ellipsis, size('')) + # EncodingMap + import codecs, encodings.iso8859_3 + x = codecs.charmap_build(encodings.iso8859_3.decoding_table) + check(x, size('32B2iB')) + # enumerate + check(enumerate([]), size('n3P')) + # reverse + check(reversed(''), size('nP')) + # float + check(float(0), size('d')) + # sys.floatinfo + check(sys.float_info, vsize('') + self.P * len(sys.float_info)) + # frame + import inspect + CO_MAXBLOCKS = 20 + x = inspect.currentframe() + ncells = len(x.f_code.co_cellvars) + nfrees = len(x.f_code.co_freevars) + extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ + ncells + nfrees - 1 + check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + # function + def func(): pass + check(func, size('13P')) + class c(): + @staticmethod + def foo(): + pass + @classmethod + def bar(cls): + pass + # staticmethod + check(foo, size('PP')) + # classmethod + check(bar, size('PP')) + # generator + def get_gen(): yield 1 + check(get_gen(), size('Pb2PPP4P')) + # iterator + check(iter('abc'), size('lP')) + # callable-iterator + import re + check(re.finditer('',''), size('2P')) + # list + samples = [[], [1,2,3], ['1', '2', '3']] + for sample in samples: + check(sample, vsize('Pn') + len(sample)*self.P) + # sortwrapper (list) + # XXX + # cmpwrapper (list) + # XXX + # listiterator (list) + check(iter([]), size('lP')) + # listreverseiterator (list) + check(reversed([]), size('nP')) + # int + check(0, vsize('')) + check(1, vsize('') + self.longdigit) + check(-1, vsize('') + self.longdigit) + PyLong_BASE = 2**sys.int_info.bits_per_digit + check(int(PyLong_BASE), vsize('') + 2*self.longdigit) + check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) + check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) + # module + check(unittest, size('PnPPP')) + # None + check(None, size('')) + # NotImplementedType + check(NotImplemented, size('')) + # object + check(object(), size('')) + # property (descriptor object) + class C(object): + def getx(self): return self.__x + def setx(self, value): self.__x = value + def delx(self): del self.__x + x = property(getx, setx, delx, "") + check(x, size('4Pi')) + # PyCapsule + # XXX + # rangeiterator + check(iter(range(1)), size('4l')) + # reverse + check(reversed(''), size('nP')) + # range + check(range(1), size('4P')) + check(range(66000), size('4P')) + # set + # frozenset + PySet_MINSIZE = 8 + samples = [[], range(10), range(50)] + s = size('3nP' + PySet_MINSIZE*'nP' + '2nP') + for sample in samples: + minused = len(sample) + if minused == 0: tmp = 1 + # the computation of minused is actually a bit more complicated + # but this suffices for the sizeof test + minused = minused*2 + newsize = PySet_MINSIZE + while newsize <= minused: + newsize = newsize << 1 + if newsize <= 8: + check(set(sample), s) + check(frozenset(sample), s) + else: + check(set(sample), s + newsize*calcsize('nP')) + check(frozenset(sample), s + newsize*calcsize('nP')) + # setiterator + check(iter(set()), size('P3n')) + # slice + check(slice(0), size('3P')) + # super + check(super(int), size('3P')) + # tuple + check((), vsize('')) + check((1,2,3), vsize('') + 3*self.P) + # type + # static type: PyTypeObject + fmt = 'P2nPI13Pl4Pn9Pn11PIPPP' + if hasattr(sys, 'getcounts'): + fmt += '3n2P' + s = vsize(fmt) + check(int, s) + # class + s = vsize(fmt + # PyTypeObject + '3P' # PyAsyncMethods + '36P' # PyNumberMethods + '3P' # PyMappingMethods + '10P' # PySequenceMethods + '2P' # PyBufferProcs + '4P') + class newstyleclass(object): pass + # Separate block for PyDictKeysObject with 8 keys and 5 entries + check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P")) + # dict with shared keys + check(newstyleclass().__dict__, size('nQ2P') + 5*self.P) + o = newstyleclass() + o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1 + # Separate block for PyDictKeysObject with 16 keys and 10 entries + check(newstyleclass, s + calcsize("2nP2n0P") + 16 + 10*calcsize("n2P")) + # dict with shared keys + check(newstyleclass().__dict__, size('nQ2P') + 10*self.P) + # unicode + # each tuple contains a string and its expected character size + # don't put any static strings here, as they may contain + # wchar_t or UTF-8 representations + samples = ['1'*100, '\xff'*50, + '\u0100'*40, '\uffff'*100, + '\U00010000'*30, '\U0010ffff'*100] + asciifields = "nnbP" + compactfields = asciifields + "nPn" + unicodefields = compactfields + "P" + for s in samples: + maxchar = ord(max(s)) + if maxchar < 128: + L = size(asciifields) + len(s) + 1 + elif maxchar < 256: + L = size(compactfields) + len(s) + 1 + elif maxchar < 65536: + L = size(compactfields) + 2*(len(s) + 1) + else: + L = size(compactfields) + 4*(len(s) + 1) + check(s, L) + # verify that the UTF-8 size is accounted for + s = chr(0x4000) # 4 bytes canonical representation + check(s, size(compactfields) + 4) + # compile() will trigger the generation of the UTF-8 + # representation as a side effect + compile(s, "", "eval") + check(s, size(compactfields) + 4 + 4) + # TODO: add check that forces the presence of wchar_t representation + # TODO: add check that forces layout of unicodefields + # weakref + import weakref + check(weakref.ref(int), size('2Pn2P')) + # weakproxy + # XXX + # weakcallableproxy + check(weakref.proxy(int), size('2Pn2P')) + + def check_slots(self, obj, base, extra): + expected = sys.getsizeof(base) + struct.calcsize(extra) + if gc.is_tracked(obj) and not gc.is_tracked(base): + expected += self.gc_headsize + self.assertEqual(sys.getsizeof(obj), expected) + + def test_slots(self): + # check all subclassable types defined in Objects/ that allow + # non-empty __slots__ + check = self.check_slots + class BA(bytearray): + __slots__ = 'a', 'b', 'c' + check(BA(), bytearray(), '3P') + class D(dict): + __slots__ = 'a', 'b', 'c' + check(D(x=[]), {'x': []}, '3P') + class L(list): + __slots__ = 'a', 'b', 'c' + check(L(), [], '3P') + class S(set): + __slots__ = 'a', 'b', 'c' + check(S(), set(), '3P') + class FS(frozenset): + __slots__ = 'a', 'b', 'c' + check(FS(), frozenset(), '3P') + from collections import OrderedDict + class OD(OrderedDict): + __slots__ = 'a', 'b', 'c' + check(OD(x=[]), OrderedDict(x=[]), '3P') + + def test_pythontypes(self): + # check all types defined in Python/ + size = test.support.calcobjsize + vsize = test.support.calcvobjsize + check = self.check_sizeof + # _ast.AST + import _ast + check(_ast.AST(), size('P')) + try: + raise TypeError + except TypeError: + tb = sys.exc_info()[2] + # traceback + if tb is not None: + check(tb, size('2P2i')) + # symtable entry + # XXX + # sys.flags + check(sys.flags, vsize('') + self.P * len(sys.flags)) + + def test_asyncgen_hooks(self): + old = sys.get_asyncgen_hooks() + self.assertIsNone(old.firstiter) + self.assertIsNone(old.finalizer) + + firstiter = lambda *a: None + sys.set_asyncgen_hooks(firstiter=firstiter) + hooks = sys.get_asyncgen_hooks() + self.assertIs(hooks.firstiter, firstiter) + self.assertIs(hooks[0], firstiter) + self.assertIs(hooks.finalizer, None) + self.assertIs(hooks[1], None) + + finalizer = lambda *a: None + sys.set_asyncgen_hooks(finalizer=finalizer) + hooks = sys.get_asyncgen_hooks() + self.assertIs(hooks.firstiter, firstiter) + self.assertIs(hooks[0], firstiter) + self.assertIs(hooks.finalizer, finalizer) + self.assertIs(hooks[1], finalizer) + + sys.set_asyncgen_hooks(*old) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + +if __name__ == "__main__": + unittest.main() From 4bd1153b2492ccc5a89ededd103f895299b8a04e Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 20:59:36 -0500 Subject: [PATCH 182/209] Add/fix a bunch of sysmodule attributes --- src/main.rs | 19 +++++- vm/src/exceptions.rs | 15 ++++- vm/src/import.rs | 2 +- vm/src/sysmodule.rs | 134 ++++++++++++++++++++++++++++++++++--------- vm/src/version.rs | 18 +++--- 5 files changed, 150 insertions(+), 38 deletions(-) diff --git a/src/main.rs b/src/main.rs index 136007857..48bf381db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,13 +64,13 @@ fn main() { return; } if let Ok(s) = vm.to_str(&arg) { - println!("{}", s); + eprintln!("{}", s); } } }), _ => { if let Ok(r) = vm.to_repr(args.as_object()) { - println!("{}", r); + eprintln!("{}", r); } } } @@ -158,6 +158,17 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { Arg::with_name("ignore-environment") .short("E") .help("Ignore environment variables PYTHON* such as PYTHONPATH"), + ) + .arg( + Arg::with_name("isolate") + .short("I") + .help("isolate Python from the user's environment (implies -E and -s)"), + ) + .arg( + Arg::with_name("implementation-option") + .short("X") + .takes_value(true) + .help("set implementation-specific option"), ); #[cfg(feature = "flame-it")] let app = app @@ -179,7 +190,8 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { /// Create settings by examining command line arguments and environment /// variables. fn create_settings(matches: &ArgMatches) -> PySettings { - let ignore_environment = matches.is_present("ignore-environment"); + let ignore_environment = + matches.is_present("ignore-environment") || matches.is_present("isolate"); let mut settings: PySettings = Default::default(); settings.ignore_environment = ignore_environment; @@ -232,6 +244,7 @@ fn create_settings(matches: &ArgMatches) -> PySettings { settings.no_site = matches.is_present("no-site"); if matches.is_present("no-user-site") + || matches.is_present("isolate") || (!ignore_environment && env::var_os("PYTHONNOUSERSITE").is_some()) { settings.no_user_site = true; diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 56faa5ede..a20af790b 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -661,6 +661,19 @@ fn key_error_str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyStringRef { } } +fn system_exit_code(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyObjectRef { + match exc.args.read().as_slice().first() { + Some(code) => match_class!(match code { + ref tup @ PyTuple => match tup.as_slice() { + [x] => x.clone(), + _ => code.clone(), + }, + other => other.clone(), + }), + None => vm.get_none(), + } +} + pub fn init(ctx: &PyContext) { let excs = &ctx.exceptions; @@ -676,7 +689,7 @@ pub fn init(ctx: &PyContext) { }); extend_class!(ctx, &excs.system_exit, { - "code" => ctx.new_readonly_getset("code", make_arg_getter(0)), + "code" => ctx.new_readonly_getset("code", system_exit_code), }); extend_class!(ctx, &excs.import_error, { diff --git a/vm/src/import.rs b/vm/src/import.rs index c36c15827..f6d51944e 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -45,7 +45,7 @@ pub fn init_importlib(vm: &mut VirtualMachine, initialize_parameter: InitParamet Ok(()) })(); if zipimport_res.is_err() { - eprintln!("couldn't init zipimport") + warn!("couldn't init zipimport") } } InitParameter::NoInitialize => { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index eb4fe3757..b9f74875d 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,8 +1,10 @@ +use num_traits::ToPrimitive; use std::{env, mem, path}; use crate::frame::FrameRef; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; use crate::pyobject::{IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult}; use crate::vm::{PySettings, VirtualMachine}; use crate::{builtins, exceptions, py_io, version}; @@ -26,7 +28,11 @@ fn argv(vm: &VirtualMachine) -> PyObjectRef { fn executable(ctx: &PyContext) -> PyObjectRef { if let Some(exec_path) = env::args().next() { - if path::Path::new(&exec_path).is_absolute() { + let path = path::Path::new(&exec_path); + if !path.exists() { + return ctx.new_str(""); + } + if path.is_absolute() { return ctx.new_str(exec_path); } if let Ok(dir) = env::current_dir() { @@ -67,50 +73,50 @@ fn getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult Self { // Start with sensible defaults: let mut flags: SysFlags = Default::default(); - flags.debug = settings.debug; - flags.inspect = settings.inspect; + flags.debug = settings.debug as u8; + flags.inspect = settings.inspect as u8; flags.optimize = settings.optimize; - flags.no_user_site = settings.no_user_site; - flags.no_site = settings.no_site; - flags.ignore_environment = settings.ignore_environment; + flags.no_user_site = settings.no_user_site as u8; + flags.no_site = settings.no_site as u8; + flags.ignore_environment = settings.ignore_environment as u8; flags.verbose = settings.verbose; - flags.quiet = settings.quiet; - flags.dont_write_bytecode = settings.dont_write_bytecode; + flags.quiet = settings.quiet as u8; + flags.dont_write_bytecode = settings.dont_write_bytecode as u8; flags } } @@ -173,7 +179,13 @@ fn sys_getrecursionlimit(vm: &VirtualMachine) -> usize { vm.recursion_limit.get() } -fn sys_setrecursionlimit(recursion_limit: usize, vm: &VirtualMachine) -> PyResult { +fn sys_setrecursionlimit(recursion_limit: i32, vm: &VirtualMachine) -> PyResult { + let recursion_limit = recursion_limit + .to_usize() + .filter(|&u| u >= 1) + .ok_or_else(|| { + vm.new_value_error("recursion limit must be greater than or equal to one".to_owned()) + })?; let recursion_depth = vm.frames.borrow().len(); if recursion_limit > recursion_depth + 1 { @@ -209,7 +221,7 @@ fn sys_git_info(vm: &VirtualMachine) -> PyObjectRef { } fn sys_exit(code: OptionalArg, vm: &VirtualMachine) -> PyResult { - let code = code.unwrap_or_else(|| vm.new_int(0)); + let code = code.unwrap_or_else(|| vm.get_none()); Err(vm.new_exception(vm.ctx.exceptions.system_exit.clone(), vec![code])) } @@ -350,7 +362,6 @@ pub fn sysconfigdata_name() -> String { } #[pystruct_sequence(module = "sys", name = "hash_info")] -#[derive(Debug)] struct PyHashInfo { width: usize, modulus: PyUHash, @@ -360,9 +371,10 @@ struct PyHashInfo { algorithm: &'static str, hash_bits: usize, seed_bits: usize, + cutoff: usize, } impl PyHashInfo { - pub const INFO: Self = { + const INFO: Self = { use rustpython_common::hash::*; PyHashInfo { width: BITS, @@ -373,20 +385,73 @@ impl PyHashInfo { algorithm: ALGO, hash_bits: HASH_BITS, seed_bits: SEED_BITS, + cutoff: 0, // no small string optimizations } }; } +#[pystruct_sequence(module = "sys", name = "float_info")] +struct PyFloatInfo { + max: f64, + max_exp: i32, + max_10_exp: i32, + min: f64, + min_exp: i32, + min_10_exp: i32, + dig: u32, + mant_dig: u32, + epsilon: f64, + radix: u32, + rounds: i32, +} +impl PyFloatInfo { + const INFO: Self = PyFloatInfo { + max: f64::MAX, + max_exp: f64::MAX_EXP, + max_10_exp: f64::MAX_10_EXP, + min: f64::MIN, + min_exp: f64::MIN_EXP, + min_10_exp: f64::MIN_10_EXP, + dig: f64::DIGITS, + mant_dig: f64::MANTISSA_DIGITS, + epsilon: f64::EPSILON, + radix: f64::RADIX, + rounds: 1, // FE_TONEAREST + }; +} + +#[pystruct_sequence(module = "sys", name = "int_info")] +struct PyIntInfo { + bits_per_digit: usize, + sizeof_digit: usize, +} +impl PyIntInfo { + const INFO: Self = PyIntInfo { + bits_per_digit: 30, //? + sizeof_digit: std::mem::size_of::(), + }; +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; let flags_type = SysFlags::make_class(ctx); + extend_class!(ctx, flags_type, { + (slot new) => |_cls: PyClassRef, _args: PyFuncArgs, vm: &VirtualMachine| -> PyResult { + Err(vm.new_type_error("cannot create 'sys.flags' instances".to_owned())) + }, + }); let flags = SysFlags::from_settings(&vm.state.settings) .into_struct_sequence(vm, flags_type) .unwrap(); let version_info_type = version::VersionInfo::make_class(ctx); - let version_info = version::get_version_info() + extend_class!(ctx, version_info_type, { + (slot new) => |_cls: PyClassRef, _args: PyFuncArgs, vm: &VirtualMachine| -> PyResult { + Err(vm.new_type_error("cannot create 'sys.version_info' instances".to_owned())) + }, + }); + let version_info = version::VersionInfo::VERSION .into_struct_sequence(vm, version_info_type) .unwrap(); @@ -395,11 +460,23 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR .into_struct_sequence(vm, hash_info_type) .unwrap(); + let float_info_type = PyFloatInfo::make_class(ctx); + let float_info = PyFloatInfo::INFO + .into_struct_sequence(vm, float_info_type) + .unwrap(); + + let int_info_type = PyIntInfo::make_class(ctx); + let int_info = PyIntInfo::INFO + .into_struct_sequence(vm, int_info_type) + .unwrap(); + // TODO Add crate version to this namespace let implementation = py_namespace!(vm, { "name" => ctx.new_str("rustpython".to_owned()), "cache_tag" => ctx.new_str("rustpython-01".to_owned()), "_multiarch" => ctx.new_str(MULTIARCH.to_owned()), + "version" => version_info.clone(), + "hexversion" => ctx.new_int(version::VERSION_HEX), }); let path = ctx.new_list( @@ -532,7 +609,7 @@ settrace() -- set the global debug tracing function "gettrace" => ctx.new_function(sys_gettrace), "hash_info" => hash_info, "intern" => ctx.new_function(sys_intern), - "maxunicode" => ctx.new_int(0x0010_FFFF), + "maxunicode" => ctx.new_int(std::char::MAX as u32), "maxsize" => ctx.new_int(std::isize::MAX), "path" => path, "ps1" => ctx.new_str(">>>>> ".to_owned()), @@ -566,6 +643,11 @@ settrace() -- set the global debug tracing function "__displayhook__" => ctx.new_function(sys_displayhook), "excepthook" => ctx.new_function(sys_excepthook), "__excepthook__" => ctx.new_function(sys_excepthook), + "hexversion" => ctx.new_int(version::VERSION_HEX), + "api_version" => ctx.new_int(0x0), // what C api? + "float_info" => float_info, + "int_info" => int_info, + "float_repr_style" => ctx.new_str("short".to_owned()), }); #[cfg(windows)] diff --git a/vm/src/version.rs b/vm/src/version.rs index f9dd90207..66c2d5fa1 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -1,12 +1,20 @@ /* Several function to retrieve version information. */ +use chrono::prelude::DateTime; +use chrono::Local; +use std::time::{Duration, UNIX_EPOCH}; + const MAJOR: usize = 3; const MINOR: usize = 5; const MICRO: usize = 0; const RELEASELEVEL: &str = "alpha"; +const RELEASELEVEL_N: usize = 0xA; const SERIAL: usize = 0; +pub const VERSION_HEX: usize = + (MAJOR << 24) | (MINOR << 16) | (MICRO << 8) | (RELEASELEVEL_N << 4) | SERIAL; + #[pystruct_sequence(module = "sys", name = "version_info")] #[derive(Default, Debug)] pub struct VersionInfo { @@ -16,10 +24,6 @@ pub struct VersionInfo { releaselevel: &'static str, serial: usize, } -extern crate chrono; -use chrono::prelude::DateTime; -use chrono::Local; -use std::time::{Duration, UNIX_EPOCH}; pub fn get_version() -> String { format!( @@ -30,14 +34,14 @@ pub fn get_version() -> String { ) } -pub fn get_version_info() -> VersionInfo { - VersionInfo { +impl VersionInfo { + pub const VERSION: VersionInfo = VersionInfo { major: MAJOR, minor: MINOR, micro: MICRO, releaselevel: RELEASELEVEL, serial: SERIAL, - } + }; } pub fn get_version_number() -> String { From a822ef62758d38a58df65e5bd77e31c7a736fe15 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 21:00:04 -0500 Subject: [PATCH 183/209] Mark unsupported tests for test_sys.py, remove redundant test_sys_executable --- Lib/test/test_sys.py | 35 ++++++++++++++----- Lib/test/test_sys_executable.py | 62 --------------------------------- 2 files changed, 27 insertions(+), 70 deletions(-) delete mode 100644 Lib/test/test_sys_executable.py diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index af0e54bd0..ca107100e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2,7 +2,7 @@ from test import support from test.support.script_helper import assert_python_ok, assert_python_failure import builtins import codecs -import gc +# import gc import locale import operator import os @@ -81,6 +81,7 @@ class ExceptHookTest(unittest.TestCase): self.assertRaises(TypeError, sys.__excepthook__) + @unittest.skip("TODO: RUSTPYTHON; SyntaxError formatting in arbitrary tracebacks") def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -98,6 +99,7 @@ class ExceptHookTest(unittest.TestCase): self.assertIn(""" text\n""", err) self.assertTrue(err.endswith("SyntaxError: msg\n")) + @unittest.skip("TODO: RUSTPYTHON; print argument error to stderr in sys.excepthook instead of throwing") def test_excepthook(self): with test.support.captured_output("stderr") as stderr: sys.excepthook(1, '1', 1) @@ -169,15 +171,17 @@ class SysModuleTest(unittest.TestCase): # test that the exit message is written with backslashreplace error # handler to stderr - check_exit_message( - r'import sys; sys.exit("surrogates:\uDCFF")', - b"surrogates:\\udcff") + # TODO: RUSTPYTHON; allow surrogates in strings + # check_exit_message( + # r'import sys; sys.exit("surrogates:\uDCFF")', + # b"surrogates:\\udcff") # test that the unicode message is encoded to the stderr encoding # instead of the default encoding (utf8) - check_exit_message( - r'import sys; sys.exit("h\xe9")', - b"h\xe9", PYTHONIOENCODING='latin-1') + # TODO: RUSTPYTHON; handle PYTHONIOENCODING + # check_exit_message( + # r'import sys; sys.exit("h\xe9")', + # b"h\xe9", PYTHONIOENCODING='latin-1') def test_getdefaultencoding(self): self.assertRaises(TypeError, sys.getdefaultencoding, 42) @@ -187,6 +191,7 @@ class SysModuleTest(unittest.TestCase): # testing sys.settrace() is done in test_sys_settrace.py # testing sys.setprofile() is done in test_sys_setprofile.py + @unittest.skip("RUSTPYTHON: don't have sys.setcheckinterval") def test_setcheckinterval(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -196,6 +201,7 @@ class SysModuleTest(unittest.TestCase): sys.setcheckinterval(n) self.assertEqual(sys.getcheckinterval(), n) + @unittest.skip("RUSTPYTHON: don't have sys.setswitchinterval") def test_switchinterval(self): self.assertRaises(TypeError, sys.setswitchinterval) self.assertRaises(TypeError, sys.setswitchinterval, "a") @@ -278,6 +284,7 @@ class SysModuleTest(unittest.TestCase): finally: sys.setrecursionlimit(oldlimit) + @unittest.skip("TODO: RUSTPYTHON; super recursion detection") def test_recursionlimit_fatalerror(self): # A fatal error occurs if a second recursion limit is hit when recovering # from a first one. @@ -332,6 +339,7 @@ class SysModuleTest(unittest.TestCase): # still has 5 elements maj, min, buildno, plat, csd = sys.getwindowsversion() + @unittest.skip("TODO: RUSTPYTHON; sys.call_tracing") def test_call_tracing(self): self.assertRaises(TypeError, sys.call_tracing, type, 2) @@ -371,6 +379,8 @@ class SysModuleTest(unittest.TestCase): ) # sys._current_frames() is a CPython-only gimmick. + # XXX RUSTPYTHON: above comment is from original cpython test; not sure why the cpython_only decorator wasn't added + @test.support.cpython_only @test.support.reap_threads def test_current_frames(self): import threading @@ -482,7 +492,9 @@ class SysModuleTest(unittest.TestCase): self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"}) else: # PY_HASH_EXTERNAL - self.assertEqual(algo, 0) + # TODO: RUSTPYTHON; use siphash24 + # self.assertEqual(algo, 0) + pass self.assertGreaterEqual(sys.hash_info.cutoff, 0) self.assertLess(sys.hash_info.cutoff, 8) @@ -517,6 +529,7 @@ class SysModuleTest(unittest.TestCase): if not sys.platform.startswith('win'): self.assertIsInstance(sys.abiflags, str) + @unittest.skip("TODO: RUSTPYTHON; sys.thread_info") def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) @@ -528,6 +541,7 @@ class SysModuleTest(unittest.TestCase): # the test runs under regrtest. self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) + @unittest.skip("TODO: RUSTPYTHON; sys.intern() string interning") def test_intern(self): global INTERN_NUMRUNS INTERN_NUMRUNS += 1 @@ -588,6 +602,7 @@ class SysModuleTest(unittest.TestCase): def test_clear_type_cache(self): sys._clear_type_cache() + @unittest.skip("TODO: RUSTPYTHON; PYTHONIOENCODING var") def test_ioencoding(self): env = dict(os.environ) @@ -710,6 +725,7 @@ class SysModuleTest(unittest.TestCase): stdout, stderr = p.communicate() return stdout + @unittest.skip("TODO: RUSTPYTHON; surrogates in strings") def check_locale_surrogateescape(self, locale): out = self.c_locale_get_error_handler(locale, isolated=True) self.assertEqual(out, @@ -828,6 +844,7 @@ class SysModuleTest(unittest.TestCase): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + @unittest.skip("TODO: RUSTPYTHON; destructors + interpreter finalization") @test.support.requires_type_collecting def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) @@ -850,6 +867,7 @@ class SysModuleTest(unittest.TestCase): rc, stdout, stderr = assert_python_ok('-c', code) self.assertEqual(stdout.rstrip(), b'True') + @unittest.skip("TODO: RUSTPYTHON; __del__ destructors/interpreter shutdown") @test.support.requires_type_collecting def test_issue20602(self): # sys.flags and sys.float_info were wiped during shutdown. @@ -873,6 +891,7 @@ class SysModuleTest(unittest.TestCase): self.assertIsInstance(level, int) self.assertGreater(level, 0) + @unittest.skip("TODO: RUSTPYTHON; sys.tracebacklimit") def test_sys_tracebacklimit(self): code = """if 1: import sys diff --git a/Lib/test/test_sys_executable.py b/Lib/test/test_sys_executable.py deleted file mode 100644 index 7aacdb026..000000000 --- a/Lib/test/test_sys_executable.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -import subprocess -import sys -import os - -# These should go into a test_sys.py file, but given these are the only -# ones being tested currently that seems unnecessary. - - -class SysExecutableTest(unittest.TestCase): - - # This is a copy of test.test_sys.SysModuleTest.test_executable from cpython. - def cpython_tests(self): - # sys.executable should be absolute - self.assertEqual(os.path.abspath(sys.executable), sys.executable) - - # Issue #7774: Ensure that sys.executable is an empty string if argv[0] - # has been set to a non existent program name and Python is unable to - # retrieve the real program name - - # For a normal installation, it should work without 'cwd' - # argument. For test runs in the build directory, see #7774. - python_dir = os.path.dirname(os.path.realpath(sys.executable)) - p = subprocess.Popen( - [ - "nonexistent", - "-c", - 'import sys; print(sys.executable.encode("ascii", "backslashreplace"))', - ], - executable=sys.executable, - stdout=subprocess.PIPE, - cwd=python_dir, - ) - stdout = p.communicate()[0] - executable = stdout.strip().decode("ASCII") - p.wait() - self.assertIn( - executable, - ["b''", repr(sys.executable.encode("ascii", "backslashreplace"))], - ) - - def test_no_follow_symlink(self): - paths = [os.path.abspath("test_symlink"), "./test_symlink"] - for path in paths: - with self.subTest(path=path): - os.symlink(sys.executable, path) - command = [ - path, - "-c", - "import sys; print(sys.executable, end='')", - ] - try: - process = subprocess.run(command, capture_output=True) - finally: - os.remove(path) - self.assertEqual( - os.path.abspath(path), process.stdout.decode("utf-8") - ) - - -if __name__ == "__main__": - unittest.main() From 3d0453d47d0933b97f641bd75bd4505684286b5b Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Mon, 3 Aug 2020 17:35:56 +0900 Subject: [PATCH 184/209] Allow dead TargetIsDirectory temporary --- vm/src/stdlib/os.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 690472aff..a279df994 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -156,6 +156,7 @@ pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { convert_io_error(vm, io::Error::last_os_error()) } +#[allow(dead_code)] #[derive(FromArgs, Default)] pub struct TargetIsDirectory { #[pyarg(keyword_only, default = "false")] From 83e2a2d25f613ea401901e3e8b74151d41b9373d Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 10:31:35 -0500 Subject: [PATCH 185/209] Add _osx_support.py from CPython 3.8.2 --- Lib/_osx_support.py | 502 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 Lib/_osx_support.py diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py new file mode 100644 index 000000000..db6674ea2 --- /dev/null +++ b/Lib/_osx_support.py @@ -0,0 +1,502 @@ +"""Shared OS X support functions.""" + +import os +import re +import sys + +__all__ = [ + 'compiler_fixup', + 'customize_config_vars', + 'customize_compiler', + 'get_platform_osx', +] + +# configuration variables that may contain universal build flags, +# like "-arch" or "-isdkroot", that may need customization for +# the user environment +_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', + 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', + 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') + +# configuration variables that may contain compiler calls +_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') + +# prefix added to original configuration variable names +_INITPRE = '_OSX_SUPPORT_INITIAL_' + + +def _find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + if path is None: + path = os.environ['PATH'] + + paths = path.split(os.pathsep) + base, ext = os.path.splitext(executable) + + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if not os.path.isfile(executable): + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + else: + return executable + + +def _read_output(commandstring): + """Output from successful command execution or None""" + # Similar to os.popen(commandstring, "r").read(), + # but without actually using os.popen because that + # function is not usable during python bootstrap. + # tempfile is also not available then. + import contextlib + try: + import tempfile + fp = tempfile.NamedTemporaryFile() + except ImportError: + fp = open("/tmp/_osx_support.%s"%( + os.getpid(),), "w+b") + + with contextlib.closing(fp) as fp: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + return fp.read().decode('utf-8').strip() if not os.system(cmd) else None + + +def _find_build_tool(toolname): + """Find a build tool on current path or using xcrun""" + return (_find_executable(toolname) + or _read_output("/usr/bin/xcrun -find %s" % (toolname,)) + or '' + ) + +_SYSTEM_VERSION = None + +def _get_system_version(): + """Return the OS X system version as a string""" + # Reading this plist is a documented way to get the system + # version (see the documentation for the Gestalt Manager) + # We avoid using platform.mac_ver to avoid possible bootstrap issues during + # the build of Python itself (distutils is used to build standard library + # extensions). + + global _SYSTEM_VERSION + + if _SYSTEM_VERSION is None: + _SYSTEM_VERSION = '' + try: + f = open('/System/Library/CoreServices/SystemVersion.plist') + except OSError: + # We're on a plain darwin box, fall back to the default + # behaviour. + pass + else: + try: + m = re.search(r'ProductUserVisibleVersion\s*' + r'(.*?)', f.read()) + finally: + f.close() + if m is not None: + _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) + # else: fall back to the default behaviour + + return _SYSTEM_VERSION + +def _remove_original_values(_config_vars): + """Remove original unmodified values for testing""" + # This is needed for higher-level cross-platform tests of get_platform. + for k in list(_config_vars): + if k.startswith(_INITPRE): + del _config_vars[k] + +def _save_modified_value(_config_vars, cv, newvalue): + """Save modified and original unmodified value of configuration var""" + + oldvalue = _config_vars.get(cv, '') + if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars): + _config_vars[_INITPRE + cv] = oldvalue + _config_vars[cv] = newvalue + +def _supports_universal_builds(): + """Returns True if universal builds are supported on this system""" + # As an approximation, we assume that if we are running on 10.4 or above, + # then we are running with an Xcode environment that supports universal + # builds, in particular -isysroot and -arch arguments to the compiler. This + # is in support of allowing 10.4 universal builds to run on 10.3.x systems. + + osx_version = _get_system_version() + if osx_version: + try: + osx_version = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + osx_version = '' + return bool(osx_version >= (10, 4)) if osx_version else False + + +def _find_appropriate_compiler(_config_vars): + """Find appropriate C compiler for extension module builds""" + + # Issue #13590: + # The OSX location for the compiler varies between OSX + # (or rather Xcode) releases. With older releases (up-to 10.5) + # the compiler is in /usr/bin, with newer releases the compiler + # can only be found inside Xcode.app if the "Command Line Tools" + # are not installed. + # + # Furthermore, the compiler that can be used varies between + # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' + # as the compiler, after that 'clang' should be used because + # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that + # miscompiles Python. + + # skip checks if the compiler was overridden with a CC env variable + if 'CC' in os.environ: + return _config_vars + + # The CC config var might contain additional arguments. + # Ignore them while searching. + cc = oldcc = _config_vars['CC'].split()[0] + if not _find_executable(cc): + # Compiler is not found on the shell search PATH. + # Now search for clang, first on PATH (if the Command LIne + # Tools have been installed in / or if the user has provided + # another location via CC). If not found, try using xcrun + # to find an uninstalled clang (within a selected Xcode). + + # NOTE: Cannot use subprocess here because of bootstrap + # issues when building Python itself (and os.popen is + # implemented on top of subprocess and is therefore not + # usable as well) + + cc = _find_build_tool('clang') + + elif os.path.basename(cc).startswith('gcc'): + # Compiler is GCC, check if it is LLVM-GCC + data = _read_output("'%s' --version" + % (cc.replace("'", "'\"'\"'"),)) + if data and 'llvm-gcc' in data: + # Found LLVM-GCC, fall back to clang + cc = _find_build_tool('clang') + + if not cc: + raise SystemError( + "Cannot locate working compiler") + + if cc != oldcc: + # Found a replacement compiler. + # Modify config vars using new compiler, if not already explicitly + # overridden by an env variable, preserving additional arguments. + for cv in _COMPILER_CONFIG_VARS: + if cv in _config_vars and cv not in os.environ: + cv_split = _config_vars[cv].split() + cv_split[0] = cc if cv != 'CXX' else cc + '++' + _save_modified_value(_config_vars, cv, ' '.join(cv_split)) + + return _config_vars + + +def _remove_universal_flags(_config_vars): + """Remove all universal build arguments from config vars""" + + for cv in _UNIVERSAL_CONFIG_VARS: + # Do not alter a config var explicitly overridden by env var + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) + flags = re.sub('-isysroot [^ \t]*', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _remove_unsupported_archs(_config_vars): + """Remove any unsupported archs from config vars""" + # Different Xcode releases support different sets for '-arch' + # flags. In particular, Xcode 4.x no longer supports the + # PPC architectures. + # + # This code automatically removes '-arch ppc' and '-arch ppc64' + # when these are not supported. That makes it possible to + # build extensions on OSX 10.7 and later with the prebuilt + # 32-bit installer on the python.org website. + + # skip checks if the compiler was overridden with a CC env variable + if 'CC' in os.environ: + return _config_vars + + if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None: + # NOTE: Cannot use subprocess here because of bootstrap + # issues when building Python itself + status = os.system( + """echo 'int main{};' | """ + """'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null""" + %(_config_vars['CC'].replace("'", "'\"'\"'"),)) + if status: + # The compile failed for some reason. Because of differences + # across Xcode and compiler versions, there is no reliable way + # to be sure why it failed. Assume here it was due to lack of + # PPC support and remove the related '-arch' flags from each + # config variables not explicitly overridden by an environment + # variable. If the error was for some other reason, we hope the + # failure will show up again when trying to compile an extension + # module. + for cv in _UNIVERSAL_CONFIG_VARS: + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _override_all_archs(_config_vars): + """Allow override of all archs with ARCHFLAGS env var""" + # NOTE: This name was introduced by Apple in OSX 10.5 and + # is used by several scripting languages distributed with + # that OS release. + if 'ARCHFLAGS' in os.environ: + arch = os.environ['ARCHFLAGS'] + for cv in _UNIVERSAL_CONFIG_VARS: + if cv in _config_vars and '-arch' in _config_vars[cv]: + flags = _config_vars[cv] + flags = re.sub(r'-arch\s+\w+\s', ' ', flags) + flags = flags + ' ' + arch + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _check_for_unavailable_sdk(_config_vars): + """Remove references to any SDKs not available""" + # If we're on OSX 10.5 or later and the user tries to + # compile an extension using an SDK that is not present + # on the current machine it is better to not use an SDK + # than to fail. This is particularly important with + # the standalone Command Line Tools alternative to a + # full-blown Xcode install since the CLT packages do not + # provide SDKs. If the SDK is not present, it is assumed + # that the header files and dev libs have been installed + # to /usr and /System/Library by either a standalone CLT + # package or the CLT component within Xcode. + cflags = _config_vars.get('CFLAGS', '') + m = re.search(r'-isysroot\s+(\S+)', cflags) + if m is not None: + sdk = m.group(1) + if not os.path.exists(sdk): + for cv in _UNIVERSAL_CONFIG_VARS: + # Do not alter a config var explicitly overridden by env var + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def compiler_fixup(compiler_so, cc_args): + """ + This function will strip '-isysroot PATH' and '-arch ARCH' from the + compile flags if the user has specified one them in extra_compile_flags. + + This is needed because '-arch ARCH' adds another architecture to the + build, without a way to remove an architecture. Furthermore GCC will + barf if multiple '-isysroot' arguments are present. + """ + stripArch = stripSysroot = False + + compiler_so = list(compiler_so) + + if not _supports_universal_builds(): + # OSX before 10.4.0, these don't support -arch and -isysroot at + # all. + stripArch = stripSysroot = True + else: + stripArch = '-arch' in cc_args + stripSysroot = '-isysroot' in cc_args + + if stripArch or 'ARCHFLAGS' in os.environ: + while True: + try: + index = compiler_so.index('-arch') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + if 'ARCHFLAGS' in os.environ and not stripArch: + # User specified different -arch flags in the environ, + # see also distutils.sysconfig + compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() + + if stripSysroot: + while True: + try: + index = compiler_so.index('-isysroot') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + # Check if the SDK that is used during compilation actually exists, + # the universal build requires the usage of a universal SDK and not all + # users have that installed by default. + sysroot = None + if '-isysroot' in cc_args: + idx = cc_args.index('-isysroot') + sysroot = cc_args[idx+1] + elif '-isysroot' in compiler_so: + idx = compiler_so.index('-isysroot') + sysroot = compiler_so[idx+1] + + if sysroot and not os.path.isdir(sysroot): + from distutils import log + log.warn("Compiling with an SDK that doesn't seem to exist: %s", + sysroot) + log.warn("Please check your Xcode installation") + + return compiler_so + + +def customize_config_vars(_config_vars): + """Customize Python build configuration variables. + + Called internally from sysconfig with a mutable mapping + containing name/value pairs parsed from the configured + makefile used to build this interpreter. Returns + the mapping updated as needed to reflect the environment + in which the interpreter is running; in the case of + a Python from a binary installer, the installed + environment may be very different from the build + environment, i.e. different OS levels, different + built tools, different available CPU architectures. + + This customization is performed whenever + distutils.sysconfig.get_config_vars() is first + called. It may be used in environments where no + compilers are present, i.e. when installing pure + Python dists. Customization of compiler paths + and detection of unavailable archs is deferred + until the first extension module build is + requested (in distutils.sysconfig.customize_compiler). + + Currently called from distutils.sysconfig + """ + + if not _supports_universal_builds(): + # On Mac OS X before 10.4, check if -arch and -isysroot + # are in CFLAGS or LDFLAGS and remove them if they are. + # This is needed when building extensions on a 10.3 system + # using a universal build of python. + _remove_universal_flags(_config_vars) + + # Allow user to override all archs with ARCHFLAGS env var + _override_all_archs(_config_vars) + + # Remove references to sdks that are not found + _check_for_unavailable_sdk(_config_vars) + + return _config_vars + + +def customize_compiler(_config_vars): + """Customize compiler path and configuration variables. + + This customization is performed when the first + extension module build is requested + in distutils.sysconfig.customize_compiler). + """ + + # Find a compiler to use for extension module builds + _find_appropriate_compiler(_config_vars) + + # Remove ppc arch flags if not supported here + _remove_unsupported_archs(_config_vars) + + # Allow user to override all archs with ARCHFLAGS env var + _override_all_archs(_config_vars) + + return _config_vars + + +def get_platform_osx(_config_vars, osname, release, machine): + """Filter values for get_platform()""" + # called from get_platform() in sysconfig and distutils.util + # + # For our purposes, we'll assume that the system version from + # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set + # to. This makes the compatibility story a bit more sane because the + # machine is going to compile and link as if it were + # MACOSX_DEPLOYMENT_TARGET. + + macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + macrelease = _get_system_version() or macver + macver = macver or macrelease + + if macver: + release = macver + osname = "macosx" + + # Use the original CFLAGS value, if available, so that we + # return the same machine type for the platform string. + # Otherwise, distutils may consider this a cross-compiling + # case and disallow installs. + cflags = _config_vars.get(_INITPRE+'CFLAGS', + _config_vars.get('CFLAGS', '')) + if macrelease: + try: + macrelease = tuple(int(i) for i in macrelease.split('.')[0:2]) + except ValueError: + macrelease = (10, 0) + else: + # assume no universal support + macrelease = (10, 0) + + if (macrelease >= (10, 4)) and '-arch' in cflags.strip(): + # The universal build will build fat binaries, but not on + # systems before 10.4 + + machine = 'fat' + + archs = re.findall(r'-arch\s+(\S+)', cflags) + archs = tuple(sorted(set(archs))) + + if len(archs) == 1: + machine = archs[0] + elif archs == ('i386', 'ppc'): + machine = 'fat' + elif archs == ('i386', 'x86_64'): + machine = 'intel' + elif archs == ('i386', 'ppc', 'x86_64'): + machine = 'fat3' + elif archs == ('ppc64', 'x86_64'): + machine = 'fat64' + elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): + machine = 'universal' + else: + raise ValueError( + "Don't know machine value for archs=%r" % (archs,)) + + elif machine == 'i386': + # On OSX the machine type returned by uname is always the + # 32-bit variant, even if the executable architecture is + # the 64-bit variant + if sys.maxsize >= 2**32: + machine = 'x86_64' + + elif machine in ('PowerPC', 'Power_Macintosh'): + # Pick a sane name for the PPC architecture. + # See 'i386' case + if sys.maxsize >= 2**32: + machine = 'ppc64' + else: + machine = 'ppc' + + return (osname, release, machine) From a52e19bc211d0c54e5e8edd90e00731e7fad1424 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 10:33:12 -0500 Subject: [PATCH 186/209] Fix wasm --- vm/src/vm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 206e89ba5..99234ff4d 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -591,7 +591,7 @@ impl VirtualMachine { fn error(s: &str); } let mut s = Vec::::new(); - exceptions::write_exception(&mut s, self, exc).unwrap(); + exceptions::write_exception(&mut s, self, &exc).unwrap(); error(std::str::from_utf8(&s).unwrap()); panic!("{}; exception backtrace above", msg) } From 7eaf57bed05dae6a92ef979159462ff75526a36a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 28 Jul 2020 01:01:34 +0900 Subject: [PATCH 187/209] cloned() -> copied() --- vm/src/bytesinner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index c344ab9b6..02a3dd1f8 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -883,9 +883,9 @@ impl PyBytesInner { return self .elements .iter() - .cloned() + .copied() .filter(|x| *x != b'\t') - .collect::>(); + .collect(); } for i in &self.elements { From 67b5d61d1f69a5c613facffd23b3408527a2ddd5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 28 Jul 2020 11:34:23 +0900 Subject: [PATCH 188/209] Refactor DictKey --- common/src/hash.rs | 2 +- examples/mini_repl.rs | 2 +- vm/src/dictdatatype.rs | 84 ++++++++++++++++----------------------- vm/src/frame.rs | 6 +-- vm/src/obj/objdict.rs | 48 +++++++++++----------- vm/src/obj/objfunction.rs | 18 +++++---- vm/src/obj/objset.rs | 24 +++++------ vm/src/obj/objstr.rs | 8 ++-- vm/src/obj/objsuper.rs | 6 ++- vm/src/py_serde.rs | 2 +- vm/src/pyobject.rs | 33 +++++++-------- vm/src/stdlib/dis.rs | 2 +- vm/src/stdlib/errno.rs | 2 +- vm/src/stdlib/os.rs | 4 +- vm/src/stdlib/thread.rs | 4 +- wasm/lib/src/convert.rs | 8 +++- wasm/lib/src/vm_class.rs | 6 +-- 17 files changed, 124 insertions(+), 135 deletions(-) diff --git a/common/src/hash.rs b/common/src/hash.rs index 12b858e5b..65a11f224 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -68,7 +68,7 @@ pub fn hash_float(value: f64) -> PyHash { x as PyHash * value.signum() as PyHash } -pub fn hash_value(data: &T) -> PyHash { +pub fn hash_value(data: &T) -> PyHash { let mut hasher = DefaultHasher::new(); data.hash(&mut hasher); hasher.finish() as PyHash diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index 92d78eb1d..493c86aab 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -40,7 +40,7 @@ macro_rules! add_python_function { // inserts the first function found in the module into the provided scope. $scope.globals.set_item( - &def.obj_name, + def.obj_name.as_str(), $vm.context().new_pyfunction( vm::obj::objcode::PyCode::new(*def.clone()).into_ref(&$vm), $scope.clone(), diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index d18863884..d799d5513 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -130,18 +130,16 @@ impl Dict { } /// Store a key - pub fn insert( - &self, - vm: &VirtualMachine, - key: K, - value: T, - ) -> PyResult<()> { + pub fn insert(&self, vm: &VirtualMachine, key: K, value: T) -> PyResult<()> + where + K: DictKey, + { // This does not need to be accurate so we can take the lock mutiple times. if self.borrow_value().indices.len() > 2 * self.borrow_value().size { self.resize(); } loop { - match self.lookup(vm, key)? { + match self.lookup(vm, &key)? { LookupResult::Existing(index) => { let mut inner = self.borrow_value_mut(); // Update existing key @@ -166,7 +164,7 @@ impl Dict { } } - pub fn contains(&self, vm: &VirtualMachine, key: K) -> PyResult { + pub fn contains(&self, vm: &VirtualMachine, key: &K) -> PyResult { if let LookupResult::Existing(_) = self.lookup(vm, key)? { Ok(true) } else { @@ -176,7 +174,7 @@ impl Dict { /// Retrieve a key #[cfg_attr(feature = "flame-it", flame("Dict"))] - pub fn get(&self, vm: &VirtualMachine, key: K) -> PyResult> { + pub fn get(&self, vm: &VirtualMachine, key: &K) -> PyResult> { loop { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.borrow_value().entries[index] { @@ -304,7 +302,7 @@ impl Dict { /// Lookup the index for the given key. #[cfg_attr(feature = "flame-it", flame("Dict"))] - fn lookup(&self, vm: &VirtualMachine, key: K) -> PyResult { + fn lookup(&self, vm: &VirtualMachine, key: &K) -> PyResult { let hash_value = key.do_hash(vm)?; let perturb = hash_value; let mut hash_index: HashIndex = hash_value; @@ -359,7 +357,7 @@ impl Dict { } /// Retrieve and delete a key - pub fn pop(&self, vm: &VirtualMachine, key: K) -> PyResult> { + pub fn pop(&self, vm: &VirtualMachine, key: &K) -> PyResult> { loop { if let LookupResult::Existing(index) = self.lookup(vm, key)? { let mut inner = self.borrow_value_mut(); @@ -413,41 +411,41 @@ enum LookupResult { /// the dictionary. Typical usecases are: /// - PyObjectRef -> arbitrary python type used as key /// - str -> string reference used as key, this is often used internally -pub trait DictKey { - fn do_hash(self, vm: &VirtualMachine) -> PyResult; - fn do_is(self, other: &PyObjectRef) -> bool; - fn do_eq(self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult; +pub trait DictKey: IntoPyObject { + fn do_hash(&self, vm: &VirtualMachine) -> PyResult; + fn do_is(&self, other: &PyObjectRef) -> bool; + fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult; } /// Implement trait for PyObjectRef such that we can use python objects /// to index dictionaries. -impl DictKey for &PyObjectRef { - fn do_hash(self, vm: &VirtualMachine) -> PyResult { +impl DictKey for PyObjectRef { + fn do_hash(&self, vm: &VirtualMachine) -> PyResult { let raw_hash = vm._hash(self)?; let mut hasher = DefaultHasher::new(); raw_hash.hash(&mut hasher); Ok(hasher.finish() as HashValue) } - fn do_is(self, other: &PyObjectRef) -> bool { + fn do_is(&self, other: &PyObjectRef) -> bool { self.is(other) } - fn do_eq(self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { vm.identical_or_equal(self, other_key) } } -impl DictKey for &PyStringRef { - fn do_hash(self, _vm: &VirtualMachine) -> PyResult { +impl DictKey for PyStringRef { + fn do_hash(&self, _vm: &VirtualMachine) -> PyResult { Ok(self.hash()) } - fn do_is(self, other: &PyObjectRef) -> bool { + fn do_is(&self, other: &PyObjectRef) -> bool { self.is(other) } - fn do_eq(self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { if self.is(other_key) { Ok(true) } else if let Some(py_str_value) = other_key.payload::() { @@ -458,49 +456,37 @@ impl DictKey for &PyStringRef { } } +// AsRef fit this case but not possible in rust 1.46 + /// Implement trait for the str type, so that we can use strings /// to index dictionaries. impl DictKey for &str { - fn do_hash(self, _vm: &VirtualMachine) -> PyResult { + fn do_hash(&self, _vm: &VirtualMachine) -> PyResult { // follow a similar route as the hashing of PyStringRef - let raw_hash = hash::hash_value(&self.to_owned()).to_bigint().unwrap(); + let raw_hash = hash::hash_value(*self).to_bigint().unwrap(); let raw_hash = hash::hash_bigint(&raw_hash); let mut hasher = DefaultHasher::new(); raw_hash.hash(&mut hasher); Ok(hasher.finish() as HashValue) } - fn do_is(self, _other: &PyObjectRef) -> bool { + fn do_is(&self, _other: &PyObjectRef) -> bool { // No matter who the other pyobject is, we are never the same thing, since // we are a str, not a pyobject. false } - fn do_eq(self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { if let Some(py_str_value) = other_key.payload::() { - Ok(py_str_value.as_str() == self) + Ok(py_str_value.as_str() == *self) } else { // Fall back to PyString implementation. - let s = vm.new_str(self.to_owned()); + let s = vm.ctx.new_str(*self); s.do_eq(vm, other_key) } } } -impl DictKey for &String { - fn do_hash(self, vm: &VirtualMachine) -> PyResult { - self.as_str().do_hash(vm) - } - - fn do_is(self, other: &PyObjectRef) -> bool { - self.as_str().do_is(other) - } - - fn do_eq(self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { - self.as_str().do_eq(vm, other_key) - } -} - #[cfg(test)] mod tests { use super::{Dict, DictKey, VirtualMachine}; @@ -513,27 +499,27 @@ mod tests { let key1 = vm.new_bool(true); let value1 = vm.new_str("abc".to_owned()); - dict.insert(&vm, &key1, value1.clone()).unwrap(); + dict.insert(&vm, key1.clone(), value1.clone()).unwrap(); assert_eq!(1, dict.len()); let key2 = vm.new_str("x".to_owned()); let value2 = vm.new_str("def".to_owned()); - dict.insert(&vm, &key2, value2.clone()).unwrap(); + dict.insert(&vm, key2.clone(), value2.clone()).unwrap(); assert_eq!(2, dict.len()); - dict.insert(&vm, &key1, value2.clone()).unwrap(); + dict.insert(&vm, key1.clone(), value2.clone()).unwrap(); assert_eq!(2, dict.len()); dict.delete(&vm, &key1).unwrap(); assert_eq!(1, dict.len()); - dict.insert(&vm, &key1, value2.clone()).unwrap(); + dict.insert(&vm, key1.clone(), value2.clone()).unwrap(); assert_eq!(2, dict.len()); assert_eq!(true, dict.contains(&vm, &key1).unwrap()); - assert_eq!(true, dict.contains(&vm, "x").unwrap()); + assert_eq!(true, dict.contains(&vm, &"x").unwrap()); - let val = dict.get(&vm, "x").unwrap().unwrap(); + let val = dict.get(&vm, &"x").unwrap().unwrap(); vm.bool_eq(val, value2) .expect("retrieved value must be equal to inserted value."); } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 02f036b62..f9a6054c1 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -954,7 +954,7 @@ impl ExecutingFrame<'_> { let idx = self.pop_value(); let obj = self.pop_value(); let value = self.pop_value(); - obj.set_item(&idx, value, vm)?; + obj.set_item(idx, value, vm)?; Ok(None) } @@ -994,12 +994,12 @@ impl ExecutingFrame<'_> { return Err(vm.new_type_error(msg)); } } - map_obj.set_item(&key, value, vm).unwrap(); + map_obj.set_item(key, value, vm).unwrap(); } } } else { for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { - map_obj.set_item(&key, value, vm).unwrap(); + map_obj.set_item(key, value, vm).unwrap(); } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index db4ebebb0..c792c5dfc 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -70,13 +70,13 @@ impl PyDictRef { let dicted: Result = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { for (key, value) in dict_obj { - dict.insert(vm, &key, value)?; + dict.insert(vm, key, value)?; } } else if let Some(keys) = vm.get_method(dict_obj.clone(), "keys") { let keys = objiter::get_iter(vm, &vm.invoke(&keys?, vec![])?)?; while let Some(key) = objiter::get_next_object(vm, &keys)? { let val = dict_obj.get_item(&key, vm)?; - dict.insert(vm, &key, val)?; + dict.insert(vm, key, val)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -94,13 +94,13 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.insert(vm, &key, value)?; + dict.insert(vm, key, value)?; } } } for (key, value) in kwargs.into_iter() { - dict.insert(vm, &vm.new_str(key), value)?; + dict.insert(vm, vm.new_str(key), value)?; } Ok(()) } @@ -111,7 +111,7 @@ impl PyDictRef { vm: &VirtualMachine, ) -> PyResult<()> { for (key, value) in dict_other { - dict.insert(vm, &key, value)?; + dict.insert(vm, key, value)?; } Ok(()) } @@ -127,7 +127,7 @@ impl PyDictRef { let value = value.unwrap_or_else(|| vm.ctx.none()); for elem in iterable.iter(vm)? { let elem = elem?; - dict.insert(vm, &elem, value.clone())?; + dict.insert(vm, elem, value.clone())?; } PyDict { entries: dict }.into_ref_with_type(vm, class) } @@ -246,12 +246,12 @@ impl PyDictRef { #[pymethod(magic)] fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner_setitem_fast(&key, value, vm) + self.inner_setitem_fast(key, value, vm) } /// Set item variant which can be called with multiple /// key types, such as str to name a notable one. - fn inner_setitem_fast( + fn inner_setitem_fast( &self, key: K, value: PyObjectRef, @@ -263,20 +263,20 @@ impl PyDictRef { #[pymethod(magic)] #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(value) = self.inner_getitem_option(&key, vm)? { + if let Some(value) = self.inner_getitem_option(key.clone(), vm)? { Ok(value) } else { - Err(vm.new_key_error(key.clone())) + Err(vm.new_key_error(key)) } } /// Return an optional inner item, or an error (can be key error as well) - fn inner_getitem_option( + fn inner_getitem_option( &self, key: K, vm: &VirtualMachine, ) -> PyResult> { - if let Some(value) = self.entries.get(vm, key)? { + if let Some(value) = self.entries.get(vm, &key)? { return Ok(Some(value)); } @@ -312,7 +312,7 @@ impl PyDictRef { Some(value) => Ok(value), None => { let set_value = default.unwrap_or_else(|| vm.ctx.none()); - self.entries.insert(vm, &key, set_value.clone())?; + self.entries.insert(vm, key, set_value.clone())?; Ok(set_value) } } @@ -378,7 +378,7 @@ impl PyDictRef { Some(value) => Ok(value), None => match default { OptionalArg::Present(default) => Ok(default), - OptionalArg::Missing => Err(vm.new_key_error(key.clone())), + OptionalArg::Missing => Err(vm.new_key_error(key)), }, } } @@ -407,7 +407,7 @@ impl PyDictRef { let dict = DictContentType::default(); for (key, value) in attrs { - dict.insert(vm, &vm.ctx.new_str(key), value)?; + dict.insert(vm, vm.ctx.new_str(key), value)?; } Ok(PyDict { entries: dict }.into_ref(vm)) @@ -432,7 +432,7 @@ impl PyDictRef { /// python value, or None. /// Note that we can pass any type which implements the DictKey /// trait. Notable examples are String and PyObjectRef. - pub fn get_item_option( + pub fn get_item_option( &self, key: T, vm: &VirtualMachine, @@ -471,17 +471,15 @@ impl PyDictRef { } } -impl ItemProtocol for PyDictRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { +impl ItemProtocol for PyDictRef +where + T: DictKey, +{ + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { self.as_object().get_item(key, vm) } - fn set_item( - &self, - key: T, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { if self.lease_class().is(&vm.ctx.dict_type()) { self.inner_setitem_fast(key, value, vm) .map(|_| vm.ctx.none()) @@ -491,7 +489,7 @@ impl ItemProtocol for PyDictRef { } } - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { self.as_object().del_item(key, vm) } } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 6eb42442f..e17f20b7f 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -91,7 +91,7 @@ impl PyFunction { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &func_args.args[i]; - locals.set_item(arg_name, arg.clone(), vm)?; + locals.set_item(arg_name.as_str(), arg.clone(), vm)?; } // Pack other positional arguments in to *args: @@ -103,7 +103,7 @@ impl PyFunction { } let vararg_value = vm.ctx.new_tuple(last_args); - locals.set_item(vararg_name, vararg_value, vm)?; + locals.set_item(vararg_name.as_str(), vararg_value, vm)?; } else { // Check the number of positional arguments if nargs > nexpected_args { @@ -121,7 +121,7 @@ impl PyFunction { { let d = vm.ctx.new_dict(); if let Some(ref kwargs_name) = code_object.varkeywords_name { - locals.set_item(kwargs_name, d.as_object().clone(), vm)?; + locals.set_item(kwargs_name.as_str(), d.as_object().clone(), vm)?; } Some(d) } else { @@ -143,9 +143,9 @@ impl PyFunction { ); } - locals.set_item(&name, value, vm)?; + locals.set_item(name.as_str(), value, vm)?; } else if let Some(d) = &kwargs { - d.set_item(&name, value, vm)?; + d.set_item(name.as_str(), value, vm)?; } else { return Err( vm.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -189,7 +189,7 @@ impl PyFunction { for (default_index, i) in (required_args..nexpected_args).enumerate() { let arg_name = &code_object.arg_names[i]; if !locals.contains_key(arg_name, vm) { - locals.set_item(arg_name, defaults[default_index].clone(), vm)?; + locals.set_item(arg_name.as_str(), defaults[default_index].clone(), vm)?; } } } @@ -199,8 +199,10 @@ impl PyFunction { for arg_name in &code_object.kwonlyarg_names { if !locals.contains_key(arg_name, vm) { if let Some(kw_only_defaults) = &self.kw_only_defaults { - if let Some(default) = kw_only_defaults.get_item_option(arg_name, vm)? { - locals.set_item(arg_name, default, vm)?; + if let Some(default) = + kw_only_defaults.get_item_option(arg_name.as_str(), vm)? + { + locals.set_item(arg_name.as_str(), default, vm)?; continue; } } diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 019f00b9a..b68373d5c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -73,7 +73,7 @@ impl PySetInner { fn new(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let set = PySetInner::default(); for item in iterable.iter(vm)? { - set.add(&item?, vm)?; + set.add(item?, vm)?; } Ok(set) } @@ -177,7 +177,7 @@ impl PySetInner { fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let set = self.clone(); for item in other.iter(vm)? { - set.add(&item?, vm)?; + set.add(item?, vm)?; } Ok(set) @@ -188,7 +188,7 @@ impl PySetInner { for item in other.iter(vm)? { let obj = item?; if self.contains(&obj, vm)? { - set.add(&obj, vm)?; + set.add(obj, vm)?; } } Ok(set) @@ -256,16 +256,16 @@ impl PySetInner { Ok(format!("{{{}}}", str_parts.join(", "))) } - fn add(&self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.content.insert(vm, item, ()) } - fn remove(&self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.content.delete(vm, item) + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.delete(vm, &item) } fn discard(&self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.content.delete_if_exists(vm, item) + self.content.delete_if_exists(vm, &item) } fn clear(&self) { @@ -284,7 +284,7 @@ impl PySetInner { fn update(&self, others: Args, vm: &VirtualMachine) -> PyResult<()> { for iterable in others { for item in iterable.iter(vm)? { - self.add(&item?, vm)?; + self.add(item?, vm)?; } } Ok(()) @@ -297,7 +297,7 @@ impl PySetInner { for item in iterable.iter(vm)? { let obj = item?; if temp_inner.contains(&obj, vm)? { - self.add(&obj, vm)?; + self.add(obj, vm)?; } } temp_inner = self.copy() @@ -518,13 +518,13 @@ impl PySet { #[pymethod] pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner.add(&item, vm)?; + self.inner.add(item, vm)?; Ok(()) } #[pymethod] fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner.remove(&item, vm) + self.inner.remove(item, vm) } #[pymethod] @@ -620,7 +620,7 @@ impl PyFrozenSet { ) -> PyResult { let inner = PySetInner::default(); for elem in it { - inner.add(&elem, vm)?; + inner.add(elem, vm)?; } Ok(Self { inner }) } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index e4e1e7d1c..8de7e1ff5 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -993,11 +993,11 @@ impl PyString { Ok(from_str) => { if to_str.len() == from_str.len() { for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { - new_dict.set_item(&vm.new_int(c1 as u32), vm.new_int(c2 as u32), vm)?; + new_dict.set_item(vm.new_int(c1 as u32), vm.new_int(c2 as u32), vm)?; } if let OptionalArg::Present(none_str) = none_str { for c in none_str.value.chars() { - new_dict.set_item(&vm.new_int(c as u32), vm.get_none(), vm)?; + new_dict.set_item(vm.new_int(c as u32), vm.get_none(), vm)?; } } new_dict.into_pyobject(vm) @@ -1019,14 +1019,14 @@ impl PyString { for (key, val) in dict { if let Some(num) = key.payload::() { new_dict.set_item( - &num.as_bigint().to_i32().into_pyobject(vm)?, + num.as_bigint().to_i32().into_pyobject(vm)?, val, vm, )?; } else if let Some(string) = key.payload::() { if string.len() == 1 { let num_value = string.value.chars().next().unwrap() as u32; - new_dict.set_item(&num_value.into_pyobject(vm)?, val, vm)?; + new_dict.set_item(num_value.into_pyobject(vm)?, val, vm)?; } else { return Err(vm.new_value_error( "string keys in translate table must be of length 1".to_owned(), diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index b2bc51257..8f57f6362 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -114,7 +114,11 @@ impl PySuper { } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match frame.scope.get_locals().get_item_option(first_arg, vm)? { + match frame + .scope + .get_locals() + .get_item_option(first_arg.as_str(), vm)? + { Some(obj) => obj, _ => { return Err(vm.new_type_error(format!( diff --git a/vm/src/py_serde.rs b/vm/src/py_serde.rs index 33cb4dc76..d4079b441 100644 --- a/vm/src/py_serde.rs +++ b/vm/src/py_serde.rs @@ -209,7 +209,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { // Although JSON keys must be strings, implementation accepts any keys // and can be reused by other deserializers without such limit while let Some((key_obj, value)) = access.next_entry_seed(self.clone(), self.clone())? { - dict.set_item(&key_obj, value, self.vm).unwrap(); + dict.set_item(key_obj, value, self.vm).unwrap(); } Ok(dict.into_object()) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d6e20f959..2cc60f9aa 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -10,7 +10,6 @@ use num_complex::Complex64; use num_traits::{One, ToPrimitive, Zero}; use crate::bytecode; -use crate::dictdatatype::DictKey; use crate::exceptions::{self, PyBaseExceptionRef}; use crate::function::{IntoPyNativeFunc, PyFuncArgs}; use crate::obj::objbuiltinfunc::{PyBuiltinFunction, PyBuiltinMethod}; @@ -952,32 +951,28 @@ impl TypeProtocol for &'_ T { /// The python item protocol. Mostly applies to dictionaries. /// Allows getting, setting and deletion of keys-value pairs. -pub trait ItemProtocol { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; - fn set_item( - &self, - key: T, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult; - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; +pub trait ItemProtocol +where + T: IntoPyObject + ?Sized, +{ + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; } -impl ItemProtocol for PyObjectRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { +impl ItemProtocol for PyObjectRef +where + T: IntoPyObject, +{ + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } - fn set_item( - &self, - key: T, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index eec0ca7bd..26fe0a1fe 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -33,7 +33,7 @@ fn dis_compiler_flag_names(vm: &VirtualMachine) -> PyObjectRef { let dict = vm.ctx.new_dict(); for (name, flag) in CodeFlags::NAME_MAPPING { dict.set_item( - &vm.ctx.new_int(flag.bits()), + vm.ctx.new_int(flag.bits()), vm.ctx.new_str((*name).to_owned()), vm, ) diff --git a/vm/src/stdlib/errno.rs b/vm/src/stdlib/errno.rs index 28934a1bd..c41c380cc 100644 --- a/vm/src/stdlib/errno.rs +++ b/vm/src/stdlib/errno.rs @@ -9,7 +9,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { for (name, code) in ERROR_CODES { let name = vm.new_str((*name).to_owned()); let code = vm.ctx.new_int(*code); - errorcode.set_item(&code, name.clone(), vm).unwrap(); + errorcode.set_item(code.clone(), name.clone(), vm).unwrap(); vm.set_attr(&module, name, code).unwrap(); } module diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a279df994..8a4ac13ab 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1131,7 +1131,7 @@ mod posix { for (key, value) in env::vars_os() { environ .set_item( - &vm.ctx.new_bytes(key.into_vec()), + vm.ctx.new_bytes(key.into_vec()), vm.ctx.new_bytes(value.into_vec()), vm, ) @@ -2254,7 +2254,7 @@ mod nt { for (key, value) in env::vars() { environ - .set_item(&vm.new_str(key), vm.new_str(value), vm) + .set_item(vm.new_str(key), vm.new_str(value), vm) .unwrap(); } environ diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index 69da935b1..197127595 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -330,7 +330,7 @@ impl PyLocal { zelf.as_object() ))) } else { - zelf.ldict(vm).set_item(attr.as_object(), value, vm)?; + zelf.ldict(vm).set_item(attr.into_object(), value, vm)?; Ok(()) } } @@ -343,7 +343,7 @@ impl PyLocal { zelf.as_object() ))) } else { - zelf.ldict(vm).del_item(attr.as_object(), vm)?; + zelf.ldict(vm).del_item(attr.into_object(), vm)?; Ok(()) } } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 8d114b9ac..7626434fa 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -213,8 +213,12 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm) - .unwrap(); + dict.set_item( + String::from(js_sys::JsString::from(key)).as_str(), + py_val, + vm, + ) + .unwrap(); } dict.into_object() } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 7e20a2bff..d5c6a683a 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -267,7 +267,7 @@ impl WASMVirtualMachine { let (key, value) = entry?; let key: String = Object::from(key).to_string().into(); attrs - .set_item(&key, convert::js_to_py(vm, value), vm) + .set_item(key.as_str(), convert::js_to_py(vm, value), vm) .to_js(vm)?; } } @@ -280,7 +280,7 @@ impl WASMVirtualMachine { let sys_modules = vm .get_attribute(vm.sys_module.clone(), "modules") .to_js(vm)?; - sys_modules.set_item(&name, module, vm).to_js(vm)?; + sys_modules.set_item(name, module, vm).to_js(vm)?; Ok(()) })? @@ -301,7 +301,7 @@ impl WASMVirtualMachine { let sys_modules = vm .get_attribute(vm.sys_module.clone(), "modules") .to_js(vm)?; - sys_modules.set_item(&name, py_module, vm).to_js(vm)?; + sys_modules.set_item(name, py_module, vm).to_js(vm)?; Ok(()) })? From d7bc12114d8ef73fddee2cfd0e47a895588ee82a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 23:58:07 -0500 Subject: [PATCH 189/209] Get rid of proc-macro-hacky stuff for py_compile_bytecode --- derive/src/compile_bytecode.rs | 17 +++++++++-------- derive/src/lib.rs | 12 +----------- vm/src/lib.rs | 15 --------------- 3 files changed, 10 insertions(+), 34 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 3cb925b72..cec3cb605 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -274,6 +274,8 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result Result ::#crate_name::bytecode::FrozenModule { + m.insert(#module_name.into(), ::#crate_name::bytecode::FrozenModule { code: ::#crate_name::bytecode::CodeObject::from_bytes( #bytes ).expect("Deserializing CodeObject failed"), package: #package, - } + }); } }); let output = quote! { - ({ - use ::#crate_name::__exports::hashmap; - hashmap! { - #(#modules),* - } - }) + { + let mut m = ::std::collections::HashMap::with_capacity(#modules_len); + #(#modules)* + m + } }; Ok(output) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 46547071e..da5a7dd42 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -59,17 +59,7 @@ pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream { result_to_tokens(pyclass::impl_pystruct_sequence(attr, item)) } -fn result_to_tokens_expr(result: Result) -> TokenStream { - let tokens2 = result.unwrap_or_else(ToTokens::into_token_stream); - let ret = quote::quote! { - macro_rules! __proc_macro_call { - () => {{ #tokens2 }} - } - }; - ret.into() -} - #[proc_macro] pub fn py_compile_bytecode(input: TokenStream) -> TokenStream { - result_to_tokens_expr(compile_bytecode::impl_py_compile_bytecode(input.into())) + result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into())) } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 238f664bd..0ab03beac 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -33,20 +33,6 @@ extern crate self as rustpython_vm; pub use rustpython_derive::*; -#[doc(hidden)] -pub use rustpython_derive::py_compile_bytecode as _py_compile_bytecode; - -#[macro_export] -macro_rules! py_compile_bytecode { - ($($arg:tt)*) => {{ - #[macro_use] - mod __m { - $crate::_py_compile_bytecode!($($arg)*); - } - __proc_macro_call!() - }}; -} - //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; @@ -90,6 +76,5 @@ pub use rustpython_common as common; #[doc(hidden)] pub mod __exports { - pub use maplit::hashmap; pub use smallbox::smallbox; } From dd8e9a65083ad21ae4a61ead906d88bc064d8798 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 23:58:54 -0500 Subject: [PATCH 190/209] Use pylib as a dependency of the rustpython binary in order to get a Lib path --- Cargo.lock | 2 +- Cargo.toml | 4 +++- src/main.rs | 8 ++++---- vm/pylib-crate/Cargo.toml | 9 ++++++--- vm/pylib-crate/src/lib.rs | 27 ++++++++------------------- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da5ef537f..efe6c395c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1294,6 +1294,7 @@ dependencies = [ "num-traits", "rustpython-compiler", "rustpython-parser", + "rustpython-pylib", "rustpython-vm", "rustyline", ] @@ -1368,7 +1369,6 @@ dependencies = [ name = "rustpython-pylib" version = "0.1.0" dependencies = [ - "maplit", "rustpython-bytecode", "rustpython-derive", ] diff --git a/Cargo.toml b/Cargo.toml index 2c6bdefe9..d99b23c33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" description = "A python interpreter written in rust." repository = "https://github.com/RustPython/RustPython" license = "MIT" +include = ["LICENSE", "Cargo.toml", "src/**/*.rs"] [workspace] members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common"] @@ -15,7 +16,7 @@ name = "bench" path = "./benchmarks/bench.rs" [features] -default = ["threading"] +default = ["threading", "pylib"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["rustpython-vm/freeze-stdlib"] threading = ["rustpython-vm/threading"] @@ -29,6 +30,7 @@ clap = "2.33" rustpython-compiler = { path = "compiler", version = "0.1.1" } rustpython-parser = { path = "parser", version = "0.1.1" } rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] } +pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true } dirs = { package = "dirs-next", version = "1.0" } num-traits = "0.2.8" cfg-if = "0.1" diff --git a/src/main.rs b/src/main.rs index 48bf381db..e9f202ae6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -198,14 +198,14 @@ fn create_settings(matches: &ArgMatches) -> PySettings { // add the current directory to sys.path settings.path_list.push("".to_owned()); + // BUILDTIME_RUSTPYTHONPATH should be set when distributing if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { settings.path_list.extend( std::env::split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()), ) - } else if option_env!("RUSTPYTHONPATH").is_none() { - settings - .path_list - .push(concat!(env!("CARGO_MANIFEST_DIR"), "/Lib").to_owned()); + } else { + #[cfg(all(feature = "pylib", not(feature = "freeze-stdlib")))] + settings.path_list.push(pylib::LIB_PATH.to_owned()); } if !ignore_environment { diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index 21e41087e..4f02ef35a 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -4,7 +4,10 @@ version = "0.1.0" authors = ["RustPython Team"] edition = "2018" +[features] +default = ["compiled-bytecode"] +compiled-bytecode = ["rustpython-derive", "rustpython-bytecode"] + [dependencies] -rustpython-derive = { version = "0.1.2", path = "../../derive" } -rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" } -maplit = "1.0" +rustpython-derive = { version = "0.1.2", path = "../../derive", optional = true } +rustpython-bytecode = { version = "0.1.2", path = "../../bytecode", optional = true } diff --git a/vm/pylib-crate/src/lib.rs b/vm/pylib-crate/src/lib.rs index 6adec5c74..98059df0b 100644 --- a/vm/pylib-crate/src/lib.rs +++ b/vm/pylib-crate/src/lib.rs @@ -4,25 +4,14 @@ extern crate self as rustpython_pylib; -use rustpython_bytecode::bytecode::{self, FrozenModule}; -use std::collections::HashMap; - -use rustpython_derive::py_compile_bytecode as _py_compile_bytecode; -#[macro_export] -macro_rules! py_compile_bytecode { - ($($arg:tt)*) => {{ - #[macro_use] - mod __m { - $crate::_py_compile_bytecode!($($arg)*); - } - __proc_macro_call!() - }}; -} - -mod __exports { - pub use maplit::hashmap; -} +pub const LIB_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Lib"); +#[cfg(feature = "compiled-bytecode")] +use { + rustpython_bytecode::bytecode::{self, FrozenModule}, + std::collections::HashMap, +}; +#[cfg(feature = "compiled-bytecode")] pub fn frozen_stdlib() -> HashMap { - py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib") + rustpython_derive::py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib") } From 0876c19c04b9bb22cc8a68b8f07466895a5fccfc Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 2 Aug 2020 23:59:05 -0500 Subject: [PATCH 191/209] Unskip test_json.test_tool --- Lib/test/test_json/test_tool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index d8cc54050..f362f1b13 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -7,7 +7,6 @@ from test import support from test.support.script_helper import assert_python_ok -@unittest.skip("TODO: RUSTPYTHON, rustpython -X faulthandler") class TestTool(unittest.TestCase): data = """ From f56e83a6d385e9279882bc603f19759ae7cf4603 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 00:31:35 -0500 Subject: [PATCH 192/209] Make sure Lib gets included to rustpython-pylib --- vm/pylib-crate/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index 4f02ef35a..3c42ed57a 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -3,6 +3,7 @@ name = "rustpython-pylib" version = "0.1.0" authors = ["RustPython Team"] edition = "2018" +include = ["Cargo.toml", "src/**/*.rs", "Lib/**"] [features] default = ["compiled-bytecode"] From 6ec2374633ebfc6481d3dab1fdfb3d95cdfb2a17 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 22:21:45 -0500 Subject: [PATCH 193/209] Don't include test files or pyc into rustpython-pylib --- vm/pylib-crate/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index 3c42ed57a..554e1ae5c 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -3,7 +3,7 @@ name = "rustpython-pylib" version = "0.1.0" authors = ["RustPython Team"] edition = "2018" -include = ["Cargo.toml", "src/**/*.rs", "Lib/**"] +include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] [features] default = ["compiled-bytecode"] From 3ad2f17a8bb4cf223a2f30f45c274e45b88afb87 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 22:26:01 -0500 Subject: [PATCH 194/209] Add the license for rustpython-pylib as the psf license --- vm/pylib-crate/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index 554e1ae5c..ea58f076b 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["RustPython Team"] edition = "2018" include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] +license-file = "Lib/PSF-LICENSE" [features] default = ["compiled-bytecode"] From c5807bc0737ffc0b6333e2e8e75276ef96aa51bc Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 22:28:47 -0500 Subject: [PATCH 195/209] Add a description+other metadata to rustpython-pylib --- vm/pylib-crate/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index ea58f076b..1fea89844 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -2,9 +2,11 @@ name = "rustpython-pylib" version = "0.1.0" authors = ["RustPython Team"] +description = "A subset of the Python standard library for use with RustPython" +repository = "https://github.com/RustPython/RustPython" +license-file = "Lib/PSF-LICENSE" edition = "2018" include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] -license-file = "Lib/PSF-LICENSE" [features] default = ["compiled-bytecode"] From 78c6b0b24bcd8e0c97e46167f75c123bfd2ca67d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 3 Aug 2020 03:17:01 +0900 Subject: [PATCH 196/209] BufferedIO OptionalSize & truncate --- vm/src/stdlib/io.rs | 71 ++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index a2acff91d..7dde91a74 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,7 +1,6 @@ /* * I/O core tools. */ -use std::convert::TryInto; use std::fs; use std::io::{self, prelude::*, Cursor, SeekFrom}; @@ -25,6 +24,32 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; +#[derive(FromArgs)] +struct OptionalSize { + // In a few functions, the default value is -1 rather than None. + // Make sure the default value doesn't affect compatibility. + #[pyarg(positional_only, default = "None")] + size: Option, +} + +impl OptionalSize { + fn to_usize(self) -> Option { + self.size.and_then(|v| v.to_usize()) + } + + fn try_usize(self, vm: &VirtualMachine) -> PyResult> { + self.size + .map(|v| { + if v >= 0 { + Ok(v as usize) + } else { + Err(vm.new_value_error(format!("Negative size value {}", v))) + } + }) + .transpose() + } +} + fn byte_count(bytes: OptionalOption) -> i64 { bytes.flatten().unwrap_or(-1) } @@ -83,9 +108,9 @@ impl BufferedIO { } //Read k bytes from the object and return. - fn read(&mut self, bytes: Option) -> Option> { + fn read(&mut self, bytes: Option) -> Option> { //for a defined number of bytes, i.e. bytes != -1 - match bytes.and_then(|v| v.to_usize()) { + match bytes { Some(bytes) => { let mut buffer = unsafe { // Do not move or edit any part of this block without a safety validation. @@ -115,13 +140,9 @@ impl BufferedIO { self.cursor.position() } - fn readline(&mut self, size: OptionalOption, vm: &VirtualMachine) -> PyResult> { + fn readline(&mut self, size: Option, vm: &VirtualMachine) -> PyResult> { let mut buf = Vec::new(); - let mut left = size - .flatten() - .filter(|s| *s >= 0) - .and_then(|s| s.try_into().ok()) - .unwrap_or(usize::MAX); + let mut left = size.unwrap_or(usize::MAX); if left == 0 { return Ok(buf); } @@ -150,6 +171,12 @@ impl BufferedIO { } } } + + fn truncate(&mut self, pos: Option) -> PyResult<()> { + let pos = pos.unwrap_or_else(|| self.tell() as usize); + self.cursor.get_mut().truncate(pos); + Ok(()) + } } #[derive(Debug)] @@ -212,8 +239,8 @@ impl PyStringIORef { //Read k bytes from the object and return. //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k - fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - let data = match self.buffer(vm)?.read(bytes.flatten()) { + fn read(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + let data = match self.buffer(vm)?.read(size.to_usize()) { Some(value) => value, None => Vec::new(), }; @@ -228,19 +255,18 @@ impl PyStringIORef { Ok(self.buffer(vm)?.tell()) } - fn readline(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult { + fn readline(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { // TODO size should correspond to the number of characters, at the moments its the number of // bytes. - match String::from_utf8(self.buffer(vm)?.readline(size, vm)?) { + match String::from_utf8(self.buffer(vm)?.readline(size.to_usize(), vm)?) { Ok(value) => Ok(value), Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_owned())), } } - fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { + fn truncate(self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<()> { let mut buffer = self.buffer(vm)?; - let size = size.flatten().unwrap_or_else(|| buffer.tell() as usize); - buffer.cursor.get_mut().truncate(size); + buffer.truncate(pos.try_usize(vm)?)?; Ok(()) } @@ -316,8 +342,8 @@ impl PyBytesIORef { //Takes an integer k (bytes) and returns them from the underlying buffer //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k - fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - match self.buffer(vm)?.read(bytes.flatten()) { + fn read(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + match self.buffer(vm)?.read(size.to_usize()) { Some(value) => Ok(vm.ctx.new_bytes(value)), None => Err(vm.new_value_error("Error Retrieving Value".to_owned())), } @@ -343,14 +369,13 @@ impl PyBytesIORef { Ok(self.buffer(vm)?.tell()) } - fn readline(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult> { - self.buffer(vm)?.readline(size, vm) + fn readline(self, size: OptionalSize, vm: &VirtualMachine) -> PyResult> { + self.buffer(vm)?.readline(size.to_usize(), vm) } - fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { + fn truncate(self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<()> { let mut buffer = self.buffer(vm)?; - let size = size.flatten().unwrap_or_else(|| buffer.tell() as usize); - buffer.cursor.get_mut().truncate(size); + buffer.truncate(pos.try_usize(vm)?)?; Ok(()) } From eabbd917a7d351a0abf79631bafd92ecc0eb6614 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 3 Aug 2020 19:57:37 +0900 Subject: [PATCH 197/209] Remove loop from BufferedIO.readline --- vm/src/stdlib/io.rs | 63 +++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 7dde91a74..2fc9c6ee3 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -141,35 +141,36 @@ impl BufferedIO { } fn readline(&mut self, size: Option, vm: &VirtualMachine) -> PyResult> { - let mut buf = Vec::new(); - let mut left = size.unwrap_or(usize::MAX); - if left == 0 { - return Ok(buf); - } - loop { - let (done, used) = { - let mut available = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; - if left < available.len() { - available = &available[..left]; - } - - match available.find_byte(b'\n') { - Some(i) => { - buf.extend_from_slice(&available[..=i]); - (true, i + 1) - } - _ => { - buf.extend_from_slice(available); - (false, available.len()) - } - } - }; - self.cursor.consume(used); - left -= used; - if done || used == 0 || left == 0 { - return Ok(buf); + let size = match size { + None => { + let mut buf = String::new(); + self.cursor + .read_line(&mut buf) + .map_err(|err| os_err(vm, err))?; + return Ok(buf.into_bytes()); } - } + Some(0) => { + return Ok(Vec::new()); + } + Some(size) => size, + }; + + let available = { + // For Cursor, fill_buf returns all of the remaining data unlike other BufReads which have outer reading source. + // Unless we add other data by write, there will be no more data. + let buf = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; + if size < buf.len() { + &buf[..size] + } else { + buf + } + }; + let buf = match available.find_byte(b'\n') { + Some(i) => (available[..=i].to_vec()), + _ => (available.to_vec()), + }; + self.cursor.consume(buf.len()); + Ok(buf) } fn truncate(&mut self, pos: Option) -> PyResult<()> { @@ -1470,12 +1471,12 @@ mod tests { #[test] fn test_buffered_read() { let data = vec![1, 2, 3, 4]; - let bytes: i64 = -1; + let bytes = None; let mut buffered = BufferedIO { cursor: Cursor::new(data.clone()), }; - assert_eq!(buffered.read(Some(bytes)).unwrap(), data); + assert_eq!(buffered.read(bytes).unwrap(), data); } #[test] @@ -1487,7 +1488,7 @@ mod tests { }; assert_eq!(buffered.seek(SeekFrom::Start(count)).unwrap(), count); - assert_eq!(buffered.read(Some(count as i64)).unwrap(), vec![3, 4]); + assert_eq!(buffered.read(Some(count as usize)).unwrap(), vec![3, 4]); } #[test] From 9646bca3ccd746bd52ce0dd7fc1409c1744c618e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 4 Aug 2020 21:10:37 +0900 Subject: [PATCH 198/209] Remove byte_count --- vm/src/stdlib/io.rs | 69 +++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 2fc9c6ee3..6cb199b36 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,7 +20,7 @@ use crate::obj::objiter; use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; use crate::pyobject::{ - BufferProtocol, Either, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + BufferProtocol, Either, IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; use crate::vm::VirtualMachine; @@ -50,9 +50,6 @@ impl OptionalSize { } } -fn byte_count(bytes: OptionalOption) -> i64 { - bytes.flatten().unwrap_or(-1) -} fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { @@ -443,13 +440,13 @@ fn io_base_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { fn io_base_readline( instance: PyObjectRef, - size: OptionalOption, + size: OptionalSize, vm: &VirtualMachine, ) -> PyResult> { - let size = byte_count(size); - let mut res = Vec::::new(); + let size = size.to_usize(); let read = vm.get_attribute(instance, "read")?; - while size < 0 || res.len() < size as usize { + let mut res = Vec::new(); + while size.map_or(true, |s| res.len() < s) { let read_res = PyBytesLike::try_from_object(vm, vm.invoke(&read, vec![vm.new_int(1)])?)?; if read_res.with_ref(|b| b.is_empty()) { break; @@ -537,26 +534,21 @@ fn io_base_readlines(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_list(vm.extract_elements(&instance)?)) } -fn raw_io_base_read( - instance: PyObjectRef, - size: OptionalOption, - vm: &VirtualMachine, -) -> PyResult { - let size = byte_count(size); - if size < 0 { - return vm.call_method(&instance, "readall", vec![]); - } - let b = PyByteArray::new(vec![0; size as usize]).into_ref(vm); - let n = >::try_from_object( - vm, - vm.call_method(&instance, "readinto", vec![b.as_object().clone()])?, - )?; - if let Some(n) = n { - let bytes = &mut b.borrow_value_mut().elements; - bytes.truncate(n); - Ok(vm.ctx.new_bytes(bytes.clone())) +fn raw_io_base_read(instance: PyObjectRef, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + if let Some(size) = size.to_usize() { + let b = PyByteArray::new(vec![0; size]).into_ref(vm); + let n = >::try_from_object( + vm, + vm.call_method(&instance, "readinto", vec![b.as_object().clone()])?, + )?; + n.map(|n| { + let bytes = &mut b.borrow_value_mut().elements; + bytes.truncate(n); + bytes.clone() + }) + .into_pyobject(vm) } else { - Ok(vm.get_none()) + vm.call_method(&instance, "readall", vec![]) } } @@ -592,13 +584,13 @@ fn buffered_io_base_name(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult fn buffered_reader_read( instance: PyObjectRef, - size: OptionalOption, + size: OptionalSize, vm: &VirtualMachine, ) -> PyResult { vm.call_method( &vm.get_attribute(instance.clone(), "raw")?, "read", - vec![vm.new_int(byte_count(size))], + vec![size.to_usize().into_pyobject(vm).unwrap()], ) } @@ -718,26 +710,23 @@ mod fileio { fn file_io_read( instance: PyObjectRef, - read_byte: OptionalOption, + read_byte: OptionalSize, vm: &VirtualMachine, ) -> PyResult> { - let read_byte = byte_count(read_byte); - let mut handle = fio_get_fileno(&instance, vm)?; - - let bytes = if read_byte < 0 { - let mut bytes = vec![]; - handle - .read_to_end(&mut bytes) - .map_err(|e| os::convert_io_error(vm, e))?; - bytes - } else { + let bytes = if let Some(read_byte) = read_byte.to_usize() { let mut bytes = vec![0; read_byte as usize]; let n = handle .read(&mut bytes) .map_err(|e| os::convert_io_error(vm, e))?; bytes.truncate(n); bytes + } else { + let mut bytes = vec![]; + handle + .read_to_end(&mut bytes) + .map_err(|e| os::convert_io_error(vm, e))?; + bytes }; fio_set_fileno(&instance, handle, vm)?; From 61455ce26125e24ac6699ee4eabdb447a6dc6421 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 4 Aug 2020 22:57:19 +0900 Subject: [PATCH 199/209] Separate IntoPyObject and IntoPyResult --- derive/src/pyclass.rs | 2 +- vm/src/dictdatatype.rs | 2 +- vm/src/format.rs | 3 +- vm/src/function.rs | 14 ++++----- vm/src/obj/objbool.rs | 10 +++--- vm/src/obj/objbytes.rs | 4 +-- vm/src/obj/objcomplex.rs | 10 +++--- vm/src/obj/objdict.rs | 4 +-- vm/src/obj/objfloat.rs | 20 ++++++------ vm/src/obj/objgetset.rs | 14 ++++----- vm/src/obj/objint.rs | 14 ++++----- vm/src/obj/objnone.rs | 8 ++--- vm/src/obj/objstr.rs | 22 ++++++------- vm/src/obj/objtuple.rs | 4 +-- vm/src/pyobject.rs | 67 ++++++++++++++++++++++++---------------- vm/src/stdlib/array.rs | 30 +++++++++--------- vm/src/stdlib/csv.rs | 6 ++-- vm/src/stdlib/socket.rs | 2 +- vm/src/stdlib/ssl.rs | 2 +- vm/src/stdlib/string.rs | 16 +++++----- vm/src/sysmodule.rs | 8 ++--- 21 files changed, 135 insertions(+), 127 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index bbcec3a3f..7c1d4a3b9 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -524,7 +524,7 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result Dict { hash_value, } => { // New key: - self.unchecked_push(hash_index, hash_value, key.into_pyobject(vm)?, value); + self.unchecked_push(hash_index, hash_value, key.into_pyobject(vm), value); break Ok(()); } } diff --git a/vm/src/format.rs b/vm/src/format.rs index 133cc9409..0f88af928 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -564,6 +564,7 @@ pub(crate) enum FormatParseError { } impl FormatParseError { + // TODO: IntoPyException pub fn into_pyobject(self, vm: &VirtualMachine) -> PyBaseExceptionRef { match self { FormatParseError::UnmatchedBracket => { @@ -823,7 +824,7 @@ impl FormatString { } FieldNamePart::Index(index) => { // TODO Implement DictKey for usize so we can pass index directly - argument = argument.get_item(&index.into_pyobject(vm)?, vm)?; + argument = argument.get_item(&index.into_pyobject(vm), vm)?; } FieldNamePart::StringIndex(index) => { argument = argument.get_item(&index, vm)?; diff --git a/vm/src/function.rs b/vm/src/function.rs index b26318ced..9d2f156b0 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -10,7 +10,7 @@ use crate::exceptions::PyBaseExceptionRef; use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype::{isinstance, PyClassRef}; use crate::pyobject::{ - IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IntoPyResult, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -524,13 +524,13 @@ macro_rules! into_py_native_func_tuple { where F: Fn($($T,)* &VirtualMachine) -> R + 'static + Send + Sync, $($T: FromArgs,)* - R: IntoPyObject, + R: IntoPyResult, { fn into_func(self) -> PyNativeFunc { smallbox!(move |vm: &VirtualMachine, args: PyFuncArgs| { let ($($n,)*) = args.bind::<($($T,)*)>(vm)?; - (self)($($n,)* vm).into_pyobject(vm) + (self)($($n,)* vm).into_pyresult(vm) }) } } @@ -540,13 +540,13 @@ macro_rules! into_py_native_func_tuple { F: Fn(&S, $($T,)* &VirtualMachine) -> R + 'static + Send + Sync, S: PyValue, $($T: FromArgs,)* - R: IntoPyObject, + R: IntoPyResult, { fn into_func(self) -> PyNativeFunc { smallbox!(move |vm: &VirtualMachine, args: PyFuncArgs| { let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; - (self)(&zelf, $($n,)* vm).into_pyobject(vm) + (self)(&zelf, $($n,)* vm).into_pyresult(vm) }) } } @@ -555,7 +555,7 @@ macro_rules! into_py_native_func_tuple { where F: Fn($($T,)*) -> R + 'static + Send + Sync, $($T: FromArgs,)* - R: IntoPyObject, + R: IntoPyResult, { fn into_func(self) -> PyNativeFunc { IntoPyNativeFunc::into_func(move |$($n,)* _vm: &VirtualMachine| (self)($($n,)*)) @@ -567,7 +567,7 @@ macro_rules! into_py_native_func_tuple { F: Fn(&S, $($T,)*) -> R + 'static + Send + Sync, S: PyValue, $($T: FromArgs,)* - R: IntoPyObject, + R: IntoPyResult, { fn into_func(self) -> PyNativeFunc { IntoPyNativeFunc::into_func(move |zelf: &S, $($n,)* _vm: &VirtualMachine| (self)(zelf, $($n,)*)) diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index 0ce748d15..16b168c78 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -13,8 +13,8 @@ use super::objstr::PyStringRef; use super::objtype; impl IntoPyObject for bool { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_bool(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_bool(self) } } @@ -112,7 +112,7 @@ impl PyBool { #[pymethod(name = "__ror__")] #[pymethod(magic)] - fn or(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn or(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&lhs, &vm.ctx.types.bool_type) && objtype::isinstance(&rhs, &vm.ctx.types.bool_type) { @@ -126,7 +126,7 @@ impl PyBool { #[pymethod(name = "__rand__")] #[pymethod(magic)] - fn and(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn and(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&lhs, &vm.ctx.types.bool_type) && objtype::isinstance(&rhs, &vm.ctx.types.bool_type) { @@ -140,7 +140,7 @@ impl PyBool { #[pymethod(name = "__rxor__")] #[pymethod(magic)] - fn xor(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn xor(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&lhs, &vm.ctx.types.bool_type) && objtype::isinstance(&rhs, &vm.ctx.types.bool_type) { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index d1c0d87ab..46003f1f3 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -59,8 +59,8 @@ impl From> for PyBytes { } impl IntoPyObject for Vec { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_bytes(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_bytes(self) } } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 367db543b..bfb01bcf3 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -32,8 +32,8 @@ impl PyValue for PyComplex { } impl IntoPyObject for Complex64 { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_complex(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_complex(self) } } @@ -81,10 +81,10 @@ impl PyComplex { where F: Fn(Complex64, Complex64) -> Complex64, { - try_complex(&other, vm)?.map_or_else( - || Ok(vm.ctx.not_implemented()), + Ok(try_complex(&other, vm)?.map_or_else( + || vm.ctx.not_implemented(), |other| op(self.value, other).into_pyobject(vm), - ) + )) } #[pymethod(name = "__add__")] diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index c792c5dfc..a31e66a31 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -282,7 +282,7 @@ impl PyDictRef { if let Some(method_or_err) = vm.get_method(self.clone().into_object(), "__missing__") { let method = method_or_err?; - Ok(Some(vm.invoke(&method, vec![key.into_pyobject(vm)?])?)) + Ok(Some(vm.invoke(&method, vec![key.into_pyobject(vm)])?)) } else { Ok(None) } @@ -419,7 +419,7 @@ impl PyDictRef { } pub fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { - let key = key.into_pyobject(vm).unwrap(); + let key = key.into_pyobject(vm); self.entries.contains(vm, &key).unwrap() } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 894d241f4..f0c0a239d 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -10,8 +10,8 @@ use super::objtype::PyClassRef; use crate::format::FormatSpec; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ - IntoPyObject, PyArithmaticValue::*, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, - PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IntoPyObject, IntoPyResult, PyArithmaticValue::*, PyClassImpl, PyComparisonValue, PyContext, + PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; use rustpython_common::{float_ops, hash}; @@ -36,13 +36,13 @@ impl PyValue for PyFloat { } impl IntoPyObject for f64 { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_float(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_float(self) } } impl IntoPyObject for f32 { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_float(f64::from(self))) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_float(f64::from(self)) } } @@ -142,9 +142,9 @@ pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { let v1 = Complex64::new(v1, 0.); let v2 = Complex64::new(v2, 0.); - v1.powc(v2).into_pyobject(vm) + Ok(v1.powc(v2).into_pyobject(vm)) } else { - v1.powf(v2).into_pyobject(vm) + Ok(v1.powf(v2).into_pyobject(vm)) } } @@ -259,7 +259,7 @@ impl PyFloat { { try_float(&other, vm)?.map_or_else( || Ok(vm.ctx.not_implemented()), - |other| op(self.value, other).into_pyobject(vm), + |other| op(self.value, other).into_pyresult(vm), ) } @@ -270,7 +270,7 @@ impl PyFloat { { try_float(&other, vm)?.map_or_else( || Ok(vm.ctx.not_implemented()), - |other| op(self.value, other).into_pyobject(vm), + |other| op(self.value, other).into_pyresult(vm), ) } diff --git a/vm/src/obj/objgetset.rs b/vm/src/obj/objgetset.rs index 1f0cb4214..b97adf1d3 100644 --- a/vm/src/obj/objgetset.rs +++ b/vm/src/obj/objgetset.rs @@ -4,7 +4,7 @@ use super::objtype::PyClassRef; use crate::function::{FunctionBox, OptionalArg, OwnedParam, RefParam}; use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + IntoPyResult, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::slots::SlotDescriptor; @@ -22,12 +22,12 @@ impl IntoPyGetterFunc<(OwnedParam, R, VirtualMachine)> for F where F: Fn(T, &VirtualMachine) -> R + 'static + Send + Sync, T: TryFromObject, - R: IntoPyObject, + R: IntoPyResult, { fn into_getter(self) -> PyGetterFunc { smallbox::smallbox!(move |vm: &VirtualMachine, obj| { let obj = T::try_from_object(vm, obj)?; - (self)(obj, vm).into_pyobject(vm) + (self)(obj, vm).into_pyresult(vm) }) } } @@ -36,12 +36,12 @@ impl IntoPyGetterFunc<(RefParam, R, VirtualMachine)> for F where F: Fn(&S, &VirtualMachine) -> R + 'static + Send + Sync, S: PyValue, - R: IntoPyObject, + R: IntoPyResult, { fn into_getter(self) -> PyGetterFunc { smallbox::smallbox!(move |vm: &VirtualMachine, obj| { let zelf = PyRef::::try_from_object(vm, obj)?; - (self)(&zelf, vm).into_pyobject(vm) + (self)(&zelf, vm).into_pyresult(vm) }) } } @@ -50,7 +50,7 @@ impl IntoPyGetterFunc<(OwnedParam, R)> for F where F: Fn(T) -> R + 'static + Send + Sync, T: TryFromObject, - R: IntoPyObject, + R: IntoPyResult, { fn into_getter(self) -> PyGetterFunc { IntoPyGetterFunc::into_getter(move |obj, _vm: &VirtualMachine| (self)(obj)) @@ -61,7 +61,7 @@ impl IntoPyGetterFunc<(RefParam, R)> for F where F: Fn(&S) -> R + 'static + Send + Sync, S: PyValue, - R: IntoPyObject, + R: IntoPyResult, { fn into_getter(self) -> PyGetterFunc { IntoPyGetterFunc::into_getter(move |zelf: &S, _vm: &VirtualMachine| (self)(zelf)) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index f846c9f76..97be9bd6f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -16,8 +16,8 @@ use crate::bytesinner::PyBytesInner; use crate::format::FormatSpec; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyArithmaticValue, PyClassImpl, PyComparisonValue, PyContext, - PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IdProtocol, IntoPyObject, IntoPyResult, PyArithmaticValue, PyClassImpl, PyComparisonValue, + PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::stdlib::array::PyArray; use crate::vm::VirtualMachine; @@ -62,8 +62,8 @@ impl PyInt { } impl IntoPyObject for BigInt { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_int(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(self) } } @@ -76,8 +76,8 @@ impl PyValue for PyInt { macro_rules! impl_into_pyobject_int { ($($t:ty)*) => {$( impl IntoPyObject for $t { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_int(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(self) } } )*}; @@ -121,7 +121,7 @@ fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { if int2.is_negative() { let v1 = try_float(int1, vm)?; let v2 = try_float(int2, vm)?; - objfloat::float_pow(v1, v2, vm).into_pyobject(vm) + objfloat::float_pow(v1, v2, vm).into_pyresult(vm) } else { Ok(if let Some(v2) = int2.to_u64() { vm.ctx.new_int(Pow::pow(int1, v2)) diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index f618927e1..09ea8ce64 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -18,16 +18,16 @@ impl PyValue for PyNone { // This allows a built-in function to not return a value, mapping to // Python's behavior of returning `None` in this situation. impl IntoPyObject for () { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.none()) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() } } impl IntoPyObject for Option { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { match self { Some(x) => x.into_pyobject(vm), - None => Ok(vm.ctx.none()), + None => vm.ctx.none(), } } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 8de7e1ff5..1ec987096 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -953,7 +953,7 @@ impl PyString { let mut translated = String::new(); for c in self.value.chars() { - match table.get_item(&(c as u32).into_pyobject(vm)?, vm) { + match table.get_item(&(c as u32).into_pyobject(vm), vm) { Ok(value) => { if let Some(text) = value.payload::() { translated.push_str(&text.value); @@ -1000,7 +1000,7 @@ impl PyString { new_dict.set_item(vm.new_int(c as u32), vm.get_none(), vm)?; } } - new_dict.into_pyobject(vm) + Ok(new_dict.into_pyobject(vm)) } else { Err(vm.new_value_error( "the first two maketrans arguments must have equal length".to_owned(), @@ -1019,14 +1019,14 @@ impl PyString { for (key, val) in dict { if let Some(num) = key.payload::() { new_dict.set_item( - num.as_bigint().to_i32().into_pyobject(vm)?, + num.as_bigint().to_i32().into_pyobject(vm), val, vm, )?; } else if let Some(string) = key.payload::() { if string.len() == 1 { let num_value = string.value.chars().next().unwrap() as u32; - new_dict.set_item(num_value.into_pyobject(vm)?, val, vm)?; + new_dict.set_item(num_value.into_pyobject(vm), val, vm)?; } else { return Err(vm.new_value_error( "string keys in translate table must be of length 1".to_owned(), @@ -1034,7 +1034,7 @@ impl PyString { } } } - new_dict.into_pyobject(vm) + Ok(new_dict.into_pyobject(vm)) } _ => Err(vm.new_value_error( "if you give only one argument to maketrans it must be a dict".to_owned(), @@ -1100,20 +1100,20 @@ impl PyValue for PyString { } impl IntoPyObject for String { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_str(self)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self) } } impl IntoPyObject for &str { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_str(self.to_owned())) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self.to_owned()) } } impl IntoPyObject for &String { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_str(self.clone())) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self.clone()) } } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 92e591f46..6fca5bb9c 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -45,8 +45,8 @@ impl PyValue for PyTuple { macro_rules! impl_intopyobj_tuple { ($(($T:ident, $idx:tt)),+) => { impl<$($T: IntoPyObject),*> IntoPyObject for ($($T,)*) { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_tuple(vec![$(self.$idx.into_pyobject(vm)?),*])) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_tuple(vec![$(self.$idx.into_pyobject(vm)),*]) } } }; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 2cc60f9aa..6f295918c 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -965,15 +965,15 @@ where T: IntoPyObject, { fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) + vm.call_method(self, "__getitem__", key.into_pyobject(vm)) } fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) + vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm), value]) } fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) + vm.call_method(self, "__delitem__", key.into_pyobject(vm)) } } @@ -1151,45 +1151,36 @@ pub trait TryFromObject: Sized { /// and should be implemented by many primitive Rust types, allowing a built-in /// function to simply return a `bool` or a `usize` for example. pub trait IntoPyObject { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult; + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef; } impl IntoPyObject for PyRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { - Ok(self.obj) + fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.obj } } impl IntoPyObject for &PyRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { - Ok(self.obj.clone()) + fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.obj.clone() } } impl IntoPyObject for PyCallable { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { - Ok(self.into_object()) + fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.into_object() } } impl IntoPyObject for PyObjectRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { - Ok(self) + fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self } } impl IntoPyObject for &PyObjectRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { - Ok(self.clone()) - } -} - -impl IntoPyObject for PyResult -where - T: IntoPyObject, -{ - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - self.and_then(|res| T::into_pyobject(res, vm)) + fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.clone() } } @@ -1199,8 +1190,30 @@ impl IntoPyObject for T where T: PyValue + Sized, { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new(self, T::class(vm), None)) + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + PyObject::new(self, T::class(vm), None) + } +} + +pub trait IntoPyResult { + fn into_pyresult(self, vm: &VirtualMachine) -> PyResult; +} + +impl IntoPyResult for T +where + T: IntoPyObject, +{ + fn into_pyresult(self, vm: &VirtualMachine) -> PyResult { + Ok(self.into_pyobject(vm)) + } +} + +impl IntoPyResult for PyResult +where + T: IntoPyObject, +{ + fn into_pyresult(self, vm: &VirtualMachine) -> PyResult { + self.map(|res| T::into_pyobject(res, vm)) } } @@ -1434,10 +1447,10 @@ impl IntoPyObject for PyArithmaticValue where T: IntoPyObject, { - fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { match self { PyArithmaticValue::Implemented(v) => v.into_pyobject(vm), - PyArithmaticValue::NotImplemented => Ok(vm.ctx.not_implemented()), + PyArithmaticValue::NotImplemented => vm.ctx.not_implemented(), } } } diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 4e397d1f4..1a79b591f 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -78,7 +78,7 @@ macro_rules! def_array_enum { Ok(()) } - fn pop(&mut self, i: usize, vm: &VirtualMachine) -> PyResult { + fn pop(&mut self, i: usize, vm: &VirtualMachine) -> PyObjectRef { match self { $(ArrayContentType::$n(v) => { v.remove(i).into_pyobject(vm) @@ -182,7 +182,7 @@ macro_rules! def_array_enum { Ok(i) } - fn getitem_by_idx(&self, i: usize, vm: &VirtualMachine) -> Option { + fn getitem_by_idx(&self, i: usize, vm: &VirtualMachine) -> Option { match self { $(ArrayContentType::$n(v) => v.get(i).map(|x| x.into_pyobject(vm)),)* } @@ -191,11 +191,9 @@ macro_rules! def_array_enum { fn getitem(&self, needle: Either, vm: &VirtualMachine) -> PyResult { match needle { Either::A(i) => { - let i = match self.idx(i, "array", vm) { - Ok(v) => v, - Err(e) => return Err(e), - }; - self.getitem_by_idx(i, vm).unwrap() + self.idx(i, "array", vm).map(|i| { + self.getitem_by_idx(i, vm).unwrap() + }) } Either::B(_slice) => Err(vm.new_not_implemented_error("array slice is not implemented".to_owned())), } @@ -214,7 +212,7 @@ macro_rules! def_array_enum { } } - fn iter<'a>(&'a self, vm: &'a VirtualMachine) -> impl Iterator + 'a { + fn iter<'a>(&'a self, vm: &'a VirtualMachine) -> impl Iterator + 'a { let mut i = 0; std::iter::from_fn(move || { let ret = self.getitem_by_idx(i, vm); @@ -379,7 +377,7 @@ impl PyArray { Err(vm.new_index_error("pop from empty array".to_owned())) } else { let i = self.borrow_value().idx(i.unwrap_or(-1), "pop", vm)?; - self.borrow_value_mut().pop(i, vm) + Ok(self.borrow_value_mut().pop(i, vm)) } } @@ -393,7 +391,7 @@ impl PyArray { let array = self.borrow_value(); let mut v = Vec::with_capacity(array.len()); for obj in array.iter(vm) { - v.push(obj?); + v.push(obj); } Ok(vm.ctx.new_list(v)) } @@ -428,7 +426,7 @@ impl PyArray { Ok(vm.new_bool(false)) } else { for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { - let ne = objbool::boolval(vm, vm._ne(a?, b?)?)?; + let ne = objbool::boolval(vm, vm._ne(a, b)?)?; if ne { return Ok(vm.new_bool(false)); } @@ -445,7 +443,7 @@ impl PyArray { let rhs = rhs.borrow_value(); for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { - let lt = objbool::boolval(vm, vm._lt(a?, b?)?)?; + let lt = objbool::boolval(vm, vm._lt(a, b)?)?; if lt { return Ok(vm.new_bool(true)); @@ -463,7 +461,7 @@ impl PyArray { let rhs = rhs.borrow_value(); for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { - let le = objbool::boolval(vm, vm._le(a?, b?)?)?; + let le = objbool::boolval(vm, vm._le(a, b)?)?; if le { return Ok(vm.new_bool(true)); @@ -481,7 +479,7 @@ impl PyArray { let rhs = rhs.borrow_value(); for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { - let gt = objbool::boolval(vm, vm._gt(a?, b?)?)?; + let gt = objbool::boolval(vm, vm._gt(a, b)?)?; if gt { return Ok(vm.new_bool(true)); @@ -499,7 +497,7 @@ impl PyArray { let rhs = rhs.borrow_value(); for (a, b) in lhs.iter(vm).zip(rhs.iter(vm)) { - let ge = objbool::boolval(vm, vm._ge(a?, b?)?)?; + let ge = objbool::boolval(vm, vm._ge(a, b)?)?; if ge { return Ok(vm.new_bool(true)); @@ -542,7 +540,7 @@ impl PyArrayIter { fn next(&self, vm: &VirtualMachine) -> PyResult { let pos = self.position.fetch_add(1); if let Some(item) = self.array.borrow_value().getitem_by_idx(pos, vm) { - Ok(item?) + Ok(item) } else { Err(objiter::new_stop_iteration(vm)) } diff --git a/vm/src/stdlib/csv.rs b/vm/src/stdlib/csv.rs index 452650c75..7db15d984 100644 --- a/vm/src/stdlib/csv.rs +++ b/vm/src/stdlib/csv.rs @@ -68,7 +68,7 @@ pub fn build_reader( ) -> PyResult { let config = ReaderOption::new(args, vm)?; - Reader::new(iterable, config).into_ref(vm).into_pyobject(vm) + Ok(Reader::new(iterable, config).into_ref(vm).into_pyobject(vm)) } fn into_strings(iterable: &PyIterable, vm: &VirtualMachine) -> PyResult> { @@ -152,7 +152,7 @@ impl Reader { #[pymethod(name = "__iter__")] fn iter(this: PyRef, vm: &VirtualMachine) -> PyResult { this.state.write().cast_to_reader(vm)?; - this.into_pyobject(vm) + Ok(this.into_pyobject(vm)) } #[pymethod(name = "__next__")] @@ -167,7 +167,7 @@ impl Reader { let iter = records .into_iter() .map(|bytes| bytes.into_pyobject(vm)) - .collect::>>()?; + .collect::>(); Ok(vm.ctx.new_list(iter)) } Err(_err) => { diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 8238c4d64..2ae76905e 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -551,7 +551,7 @@ fn socket_getaddrinfo(opts: GAIOptions, vm: &VirtualMachine) -> PyResult { Some(s) => vm.new_str(s), None => vm.get_none(), }, - get_addr_tuple(ai.sockaddr).into_pyobject(vm).unwrap(), + get_addr_tuple(ai.sockaddr).into_pyobject(vm), ]) }) }) diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index 0263f748c..a862cb8e0 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -770,7 +770,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { // Constants "OPENSSL_VERSION" => ctx.new_str(openssl::version::version().to_owned()), "OPENSSL_VERSION_NUMBER" => ctx.new_int(openssl::version::number()), - "OPENSSL_VERSION_INFO" => parse_version_info(openssl::version::number()).into_pyobject(vm).unwrap(), + "OPENSSL_VERSION_INFO" => parse_version_info(openssl::version::number()).into_pyobject(vm), "PROTOCOL_SSLv2" => ctx.new_int(SslVersion::Ssl2 as u32), "PROTOCOL_SSLv3" => ctx.new_int(SslVersion::Ssl3 as u32), "PROTOCOL_SSLv23" => ctx.new_int(SslVersion::Tls as u32), diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 8319f04b8..8e73815c8 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -21,7 +21,7 @@ mod _string { format_spec: Option, preconversion_spec: Option, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyObjectRef { let tuple = ( literal, field_name, @@ -51,7 +51,7 @@ mod _string { Some(format_spec), preconversion_spec, vm, - )?); + )); } FormatPart::Literal(text) => literal.push_str(&text), } @@ -63,7 +63,7 @@ mod _string { None, None, vm, - )?); + )); } Ok(result.into()) } @@ -77,17 +77,17 @@ mod _string { let first = match field_name.field_type { FieldType::Auto => vm.new_str("".to_owned()), - FieldType::Index(index) => index.into_pyobject(vm)?, - FieldType::Keyword(attribute) => attribute.into_pyobject(vm)?, + FieldType::Index(index) => index.into_pyobject(vm), + FieldType::Keyword(attribute) => attribute.into_pyobject(vm), }; let rest = field_name .parts .iter() .map(|p| match p { - FieldNamePart::Attribute(attribute) => (true, attribute).into_pyobject(vm).unwrap(), - FieldNamePart::StringIndex(index) => (false, index).into_pyobject(vm).unwrap(), - FieldNamePart::Index(index) => (false, *index).into_pyobject(vm).unwrap(), + FieldNamePart::Attribute(attribute) => (true, attribute).into_pyobject(vm), + FieldNamePart::StringIndex(index) => (false, index).into_pyobject(vm), + FieldNamePart::Index(index) => (false, *index).into_pyobject(vm), }) .collect(); diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index b9f74875d..590e91f65 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -576,12 +576,8 @@ settrace() -- set the global debug tracing function module_names.push("sys".to_owned()); module_names.push("builtins".to_owned()); module_names.sort(); - let builtin_module_names = ctx.new_tuple( - module_names - .iter() - .map(|v| v.into_pyobject(vm).unwrap()) - .collect(), - ); + let builtin_module_names = + ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm)).collect()); let modules = ctx.new_dict(); let prefix = option_env!("RUSTPYTHON_PREFIX").unwrap_or("/usr/local"); From a995f973ead14b7fef20b27f0e7a5741a11a30f5 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Aug 2020 22:19:16 -0500 Subject: [PATCH 200/209] Fix os module on wasi --- vm/src/stdlib/os.rs | 46 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 8a4ac13ab..7b3822760 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -598,18 +598,6 @@ mod _os { ) } - #[cfg(all(not(unix), not(windows)))] - #[pyfunction] - pub(super) fn symlink( - src: PyPathLike, - dst: PyPathLike, - _target_is_directory: TargetIsDirectory, - dir_fd: DirFd, - vm: &VirtualMachine, - ) -> PyResult<()> { - unimplemented!(); - } - #[pyfunction] fn getcwd(vm: &VirtualMachine) -> PyResult { Ok(env::current_dir() @@ -2464,22 +2452,21 @@ use nt as platform; pub use nt::{_set_thread_local_invalid_parameter_handler, silent_iph_handler}; #[cfg(not(any(unix, windows)))] +#[pymodule(name = "_os")] mod minor { use super::*; - pub const MODULE_NAME: &str = "posix"; - #[cfg(target_os = "wasi")] pub(super) type OpenFlags = u16; #[cfg(target_os = "wasi")] pub(crate) fn raw_file_number(handle: File) -> i64 { // This should be safe, since the wasi api is pretty well defined, but once - // `wasi_ext` get's stabilized, we should use that instead. + // `wasi_ext` gets stabilized we should use that instead. unsafe { std::mem::transmute::<_, u32>(handle).into() } } #[cfg(not(target_os = "wasi"))] - pub(crate) fn raw_file_number(handle: File) -> i64 { + pub(crate) fn raw_file_number(_handle: File) -> i64 { unimplemented!(); } @@ -2489,32 +2476,43 @@ mod minor { } #[cfg(not(target_os = "wasi"))] - pub(crate) fn rust_file(raw_fileno: i64) -> File { + pub(crate) fn rust_file(_raw_fileno: i64) -> File { unimplemented!(); } #[pyfunction] - pub(super) fn access(path: PyStringRef, mode: u8, vm: &VirtualMachine) -> PyResult { + pub(super) fn access(_path: PyStringRef, _mode: u8, _vm: &VirtualMachine) -> PyResult { unimplemented!() } #[pyfunction] - fn stat( - file: Either, + pub(super) fn stat( + _file: Either, _dir_fd: DirFd, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, + _follow_symlinks: FollowSymlinks, + _vm: &VirtualMachine, ) -> PyResult { unimplemented!(); } + #[pyfunction] + pub(super) fn symlink( + _src: PyPathLike, + _dst: PyPathLike, + _target_is_directory: TargetIsDirectory, + _dir_fd: DirFd, + _vm: &VirtualMachine, + ) -> PyResult<()> { + unimplemented!(); + } + pub(super) fn _environ(vm: &VirtualMachine) -> PyDictRef { vm.ctx.new_dict() } - fn extend_module_platform_specific(_vm: &VirtualMachine, _module: &PyObjectRef) {} + pub fn extend_module_platform_specific(_vm: &VirtualMachine, _module: &PyObjectRef) {} - pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + pub(super) fn support_funcs(_vm: &VirtualMachine) -> Vec { Vec::new() } } From 477c968455dc99ce175d607a8df579646c5ab022 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 4 Aug 2020 22:19:43 -0500 Subject: [PATCH 201/209] Call the _os module posix on minor platforms --- vm/src/stdlib/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 7b3822760..c383f316d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -927,7 +927,7 @@ fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { } #[cfg(unix)] -#[pymodule(name = "posix")] +#[pymodule] mod posix { use super::*; @@ -2452,7 +2452,7 @@ use nt as platform; pub use nt::{_set_thread_local_invalid_parameter_handler, silent_iph_handler}; #[cfg(not(any(unix, windows)))] -#[pymodule(name = "_os")] +#[pymodule(name = "posix")] mod minor { use super::*; From ea25947b533b4d771d8a08991c645ffaf8f1e8de Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 4 Aug 2020 23:07:53 +0900 Subject: [PATCH 202/209] Remove impl IntoPyObject for &PyObjectRef and &PyRef they doesn't give any benefit but just hide clone --- vm/src/format.rs | 5 ++--- vm/src/frame.rs | 6 +++--- vm/src/obj/objdict.rs | 2 +- vm/src/obj/objmappingproxy.rs | 2 +- vm/src/obj/objmemory.rs | 2 +- vm/src/obj/objstr.rs | 2 +- vm/src/pyobject.rs | 12 ------------ 7 files changed, 9 insertions(+), 22 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index 0f88af928..c70a7873a 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,7 +1,7 @@ use crate::exceptions::PyBaseExceptionRef; use crate::function::PyFuncArgs; use crate::obj::{objstr, objtype}; -use crate::pyobject::{IntoPyObject, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; use itertools::{Itertools, PeekingNext}; use num_bigint::{BigInt, Sign}; @@ -823,8 +823,7 @@ impl FormatString { argument = vm.get_attribute(argument, attribute.as_str())?; } FieldNamePart::Index(index) => { - // TODO Implement DictKey for usize so we can pass index directly - argument = argument.get_item(&index.into_pyobject(vm), vm)?; + argument = argument.get_item(index, vm)?; } FieldNamePart::StringIndex(index) => { argument = argument.get_item(&index, vm)?; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 337f23241..38606b3fb 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -947,7 +947,7 @@ impl ExecutingFrame<'_> { fn execute_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { let b_ref = self.pop_value(); let a_ref = self.pop_value(); - let value = a_ref.get_item(&b_ref, vm)?; + let value = a_ref.get_item(b_ref, vm)?; self.push_value(value); Ok(None) } @@ -963,7 +963,7 @@ impl ExecutingFrame<'_> { fn execute_delete_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); - obj.del_item(&idx, vm)?; + obj.del_item(idx, vm)?; Ok(None) } @@ -987,7 +987,7 @@ impl ExecutingFrame<'_> { })?; for (key, value) in dict { if for_call { - if map_obj.contains_key(&key, vm) { + if map_obj.contains_key(key.clone(), vm) { let key_repr = vm.to_repr(&key)?; let msg = format!( "got multiple values for keyword argument {}", diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index a31e66a31..90b522485 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -75,7 +75,7 @@ impl PyDictRef { } else if let Some(keys) = vm.get_method(dict_obj.clone(), "keys") { let keys = objiter::get_iter(vm, &vm.invoke(&keys?, vec![])?)?; while let Some(key) = objiter::get_next_object(vm, &keys)? { - let val = dict_obj.get_item(&key, vm)?; + let val = dict_obj.get_item(key.clone(), vm)?; dict.insert(vm, key, val)?; } } else { diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs index 2f31b1a18..67256eca0 100644 --- a/vm/src/obj/objmappingproxy.rs +++ b/vm/src/obj/objmappingproxy.rs @@ -50,7 +50,7 @@ impl PyMappingProxy { let key = PyStringRef::try_from_object(vm, key)?; class.get_attr(key.as_str()) } - MappingProxyInner::Dict(obj) => obj.get_item(&key, vm).ok(), + MappingProxyInner::Dict(obj) => obj.get_item(key, vm).ok(), }; Ok(opt) } diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index acdf09ad3..75b9aff13 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -60,7 +60,7 @@ impl PyMemoryView { #[pymethod(name = "__getitem__")] fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.obj_ref.get_item(&needle, vm) + self.obj_ref.get_item(needle, vm) } #[pymethod(magic)] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 1ec987096..093e6e494 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -953,7 +953,7 @@ impl PyString { let mut translated = String::new(); for c in self.value.chars() { - match table.get_item(&(c as u32).into_pyobject(vm), vm) { + match table.get_item((c as u32).into_pyobject(vm), vm) { Ok(value) => { if let Some(text) = value.payload::() { translated.push_str(&text.value); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 6f295918c..4a4118aaa 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1160,12 +1160,6 @@ impl IntoPyObject for PyRef { } } -impl IntoPyObject for &PyRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.obj.clone() - } -} - impl IntoPyObject for PyCallable { fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.into_object() @@ -1178,12 +1172,6 @@ impl IntoPyObject for PyObjectRef { } } -impl IntoPyObject for &PyObjectRef { - fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.clone() - } -} - // Allows a built-in function to return any built-in object payload without // explicitly implementing `IntoPyObject`. impl IntoPyObject for T From 276180e4c34af9cd0f66d9c6f5bee08e7ac3f0c5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 5 Aug 2020 15:45:09 +0900 Subject: [PATCH 203/209] Fix broken build --- vm/src/stdlib/io.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 6cb199b36..62ae92dbe 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -541,12 +541,12 @@ fn raw_io_base_read(instance: PyObjectRef, size: OptionalSize, vm: &VirtualMachi vm, vm.call_method(&instance, "readinto", vec![b.as_object().clone()])?, )?; - n.map(|n| { + Ok(n.map(|n| { let bytes = &mut b.borrow_value_mut().elements; bytes.truncate(n); bytes.clone() }) - .into_pyobject(vm) + .into_pyobject(vm)) } else { vm.call_method(&instance, "readall", vec![]) } @@ -590,7 +590,7 @@ fn buffered_reader_read( vm.call_method( &vm.get_attribute(instance.clone(), "raw")?, "read", - vec![size.to_usize().into_pyobject(vm).unwrap()], + vec![size.to_usize().into_pyobject(vm)], ) } From ee90f7cfbd08d6521a33003f69540db867ff5550 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 1 Aug 2020 04:35:30 +0900 Subject: [PATCH 204/209] IntoPyException --- vm/src/exceptions.rs | 4 + vm/src/stdlib/io.rs | 14 +- vm/src/stdlib/os.rs | 329 +++++++++++++++---------------- vm/src/stdlib/posixsubprocess.rs | 15 +- vm/src/stdlib/pwd.rs | 6 +- vm/src/stdlib/select.rs | 3 +- vm/src/stdlib/socket.rs | 9 +- vm/src/stdlib/ssl.rs | 15 +- vm/src/stdlib/thread.rs | 4 +- vm/src/stdlib/winreg.rs | 17 +- 10 files changed, 205 insertions(+), 211 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index a20af790b..02ad6e7ae 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -40,6 +40,10 @@ impl fmt::Debug for PyBaseException { pub type PyBaseExceptionRef = PyRef; +pub trait IntoPyException { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef; +} + impl PyValue for PyBaseException { const HAVE_DICT: bool = true; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 62ae92dbe..bab8ab4d3 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -10,7 +10,7 @@ use num_traits::ToPrimitive; use crate::byteslike::PyBytesLike; use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; -use crate::exceptions::PyBaseExceptionRef; +use crate::exceptions::{IntoPyException, PyBaseExceptionRef}; use crate::function::{Args, KwArgs, OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objbytearray::PyByteArray; @@ -53,7 +53,7 @@ impl OptionalSize { fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { - super::os::convert_io_error(vm, err) + err.into_pyexception(vm) } #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] { @@ -718,14 +718,14 @@ mod fileio { let mut bytes = vec![0; read_byte as usize]; let n = handle .read(&mut bytes) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; bytes.truncate(n); bytes } else { let mut bytes = vec![]; handle .read_to_end(&mut bytes) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; bytes }; fio_set_fileno(&instance, handle, vm)?; @@ -776,7 +776,7 @@ mod fileio { let len = obj .with_ref(|b| handle.write(b)) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; fio_set_fileno(&instance, handle, vm)?; @@ -809,7 +809,7 @@ mod fileio { let new_pos = handle .seek(seekfrom(vm, offset, how)?) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; fio_set_fileno(&instance, handle, vm)?; @@ -821,7 +821,7 @@ mod fileio { let pos = handle .seek(SeekFrom::Current(0)) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; fio_set_fileno(&instance, handle, vm)?; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c383f316d..755f49b1f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -2,7 +2,7 @@ use std::ffi; use std::fs::File; use std::fs::OpenOptions; use std::io::{self, ErrorKind, Read, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use std::{env, fs}; @@ -12,7 +12,7 @@ use num_traits::ToPrimitive; use super::errno::errors; use crate::byteslike::PyBytesLike; use crate::common::cell::PyRwLock; -use crate::exceptions::PyBaseExceptionRef; +use crate::exceptions::{IntoPyException, PyBaseExceptionRef}; use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objdict::PyDictRef; @@ -82,6 +82,14 @@ impl PyPathLike { } } +fn fs_metadata>(path: P, follow_symlink: bool) -> io::Result { + if follow_symlink { + fs::metadata(path.as_ref()) + } else { + fs::symlink_metadata(path.as_ref()) + } +} + impl TryFromObject for PyPathLike { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let match1 = |obj: &PyObjectRef| { @@ -126,34 +134,64 @@ fn make_path<'a>(_vm: &VirtualMachine, path: &'a PyPathLike, dir_fd: &DirFd) -> } } -pub fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { - #[allow(unreachable_patterns)] // some errors are just aliases of each other - let exc_type = match err.kind() { - ErrorKind::NotFound => vm.ctx.exceptions.file_not_found_error.clone(), - ErrorKind::PermissionDenied => vm.ctx.exceptions.permission_error.clone(), - ErrorKind::AlreadyExists => vm.ctx.exceptions.file_exists_error.clone(), - ErrorKind::WouldBlock => vm.ctx.exceptions.blocking_io_error.clone(), - _ => match err.raw_os_error() { - Some(errors::EAGAIN) - | Some(errors::EALREADY) - | Some(errors::EWOULDBLOCK) - | Some(errors::EINPROGRESS) => vm.ctx.exceptions.blocking_io_error.clone(), - _ => vm.ctx.exceptions.os_error.clone(), - }, - }; - let os_error = vm.new_exception_msg(exc_type, err.to_string()); - let errno = match err.raw_os_error() { - Some(errno) => vm.new_int(errno), - None => vm.get_none(), - }; - vm.set_attr(os_error.as_object(), "errno", errno).unwrap(); - os_error +impl IntoPyException for io::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + #[allow(unreachable_patterns)] // some errors are just aliases of each other + let exc_type = match self.kind() { + ErrorKind::NotFound => vm.ctx.exceptions.file_not_found_error.clone(), + ErrorKind::PermissionDenied => vm.ctx.exceptions.permission_error.clone(), + ErrorKind::AlreadyExists => vm.ctx.exceptions.file_exists_error.clone(), + ErrorKind::WouldBlock => vm.ctx.exceptions.blocking_io_error.clone(), + _ => match self.raw_os_error() { + Some(errors::EAGAIN) + | Some(errors::EALREADY) + | Some(errors::EWOULDBLOCK) + | Some(errors::EINPROGRESS) => vm.ctx.exceptions.blocking_io_error.clone(), + _ => vm.ctx.exceptions.os_error.clone(), + }, + }; + let os_error = vm.new_exception_msg(exc_type, self.to_string()); + let errno = match self.raw_os_error() { + Some(errno) => vm.new_int(errno), + None => vm.get_none(), + }; + vm.set_attr(os_error.as_object(), "errno", errno).unwrap(); + os_error + } +} + +#[cfg(unix)] +impl IntoPyException for nix::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + let nix_error = match self { + nix::Error::InvalidPath => { + let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); + vm.new_exception_msg(exc_type, self.to_string()) + } + nix::Error::InvalidUtf8 => { + let exc_type = vm.ctx.exceptions.unicode_error.clone(); + vm.new_exception_msg(exc_type, self.to_string()) + } + nix::Error::UnsupportedOperation => vm.new_runtime_error(self.to_string()), + nix::Error::Sys(errno) => { + let exc_type = posix::convert_nix_errno(vm, errno); + vm.new_exception_msg(exc_type, self.to_string()) + } + }; + + if let nix::Error::Sys(errno) = self { + vm.set_attr(nix_error.as_object(), "errno", vm.ctx.new_int(errno as i32)) + .unwrap(); + } + + nix_error + } } /// Convert the error stored in the `errno` variable into an Exception #[inline] pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { - convert_io_error(vm, io::Error::last_os_error()) + io::Error::last_os_error().into_pyexception(vm) } #[allow(dead_code)] @@ -269,7 +307,7 @@ mod _os { } let handle = options .open(fname) - .map_err(|err| convert_io_error(vm, err))?; + .map_err(|err| err.into_pyexception(vm))?; Ok(raw_file_number(handle)) } @@ -290,7 +328,7 @@ mod _os { #[pyfunction] fn fsync(fd: i64, vm: &VirtualMachine) -> PyResult<()> { let file = rust_file(fd); - file.sync_all().map_err(|err| convert_io_error(vm, err))?; + file.sync_all().map_err(|err| err.into_pyexception(vm))?; // Avoid closing the fd raw_file_number(file); Ok(()) @@ -302,7 +340,7 @@ mod _os { let mut file = rust_file(fd); let n = file .read(&mut buffer) - .map_err(|err| convert_io_error(vm, err))?; + .map_err(|err| err.into_pyexception(vm))?; buffer.truncate(n); // Avoid closing the fd @@ -315,7 +353,7 @@ mod _os { let mut file = rust_file(fd); let written = data .with_ref(|b| file.write(b)) - .map_err(|err| convert_io_error(vm, err))?; + .map_err(|err| err.into_pyexception(vm))?; // Avoid closing the fd raw_file_number(file); @@ -325,7 +363,7 @@ mod _os { #[pyfunction] fn remove(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { let path = make_path(vm, &path, &dir_fd); - fs::remove_file(path).map_err(|err| convert_io_error(vm, err)) + fs::remove_file(path).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -336,27 +374,27 @@ mod _os { vm: &VirtualMachine, ) -> PyResult<()> { let path = make_path(vm, &path, &dir_fd); - fs::create_dir(path).map_err(|err| convert_io_error(vm, err)) + fs::create_dir(path).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] fn mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - fs::create_dir_all(path.as_str()).map_err(|err| convert_io_error(vm, err)) + fs::create_dir_all(path.as_str()).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] fn rmdir(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { let path = make_path(vm, &path, &dir_fd); - fs::remove_dir(path).map_err(|err| convert_io_error(vm, err)) + fs::remove_dir(path).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] fn listdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult { - let res: PyResult> = fs::read_dir(&path.path) - .map_err(|err| convert_io_error(vm, err))? + let dir_iter = fs::read_dir(&path.path).map_err(|err| err.into_pyexception(vm))?; + let res: PyResult> = dir_iter .map(|entry| match entry { Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), - Err(s) => Err(convert_io_error(vm, s)), + Err(err) => Err(err.into_pyexception(vm)), }) .collect(); Ok(vm.ctx.new_list(res?)) @@ -394,7 +432,7 @@ mod _os { fn readlink(path: PyPathLike, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { let mode = path.mode; let path = make_path(vm, &path, &dir_fd); - let path = fs::read_link(path).map_err(|err| convert_io_error(vm, err))?; + let path = fs::read_link(path).map_err(|err| err.into_pyexception(vm))?; mode.process_path(path, vm) } @@ -430,11 +468,8 @@ mod _os { action: fn(fs::Metadata) -> bool, vm: &VirtualMachine, ) -> PyResult { - let metadata = match follow_symlinks.follow_symlinks { - true => fs::metadata(self.entry.path()), - false => fs::symlink_metadata(self.entry.path()), - }; - let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + let meta = fs_metadata(self.entry.path(), follow_symlinks.follow_symlinks) + .map_err(|err| err.into_pyexception(vm))?; Ok(action(meta)) } @@ -461,7 +496,7 @@ mod _os { Ok(self .entry .file_type() - .map_err(|err| convert_io_error(vm, err))? + .map_err(|err| err.into_pyexception(vm))? .is_symlink()) } @@ -514,7 +549,7 @@ mod _os { } .into_ref(vm) .into_object()), - Err(s) => Err(convert_io_error(vm, s)), + Err(err) => Err(err.into_pyexception(vm)), }, None => { self.exhausted.store(true); @@ -551,16 +586,14 @@ mod _os { OptionalArg::Missing => PyPathLike::new_str(".".to_owned()), }; - match fs::read_dir(path.path) { - Ok(iter) => Ok(ScandirIterator { - entries: PyRwLock::new(iter), - exhausted: AtomicCell::new(false), - mode: path.mode, - } - .into_ref(vm) - .into_object()), - Err(s) => Err(convert_io_error(vm, s)), + let entries = fs::read_dir(path.path).map_err(|err| err.into_pyexception(vm))?; + Ok(ScandirIterator { + entries: PyRwLock::new(entries), + exhausted: AtomicCell::new(false), + mode: path.mode, } + .into_ref(vm) + .into_object()) } #[pystruct_sequence(module = "os", name = "stat_result")] @@ -601,7 +634,7 @@ mod _os { #[pyfunction] fn getcwd(vm: &VirtualMachine) -> PyResult { Ok(env::current_dir() - .map_err(|err| convert_io_error(vm, err))? + .map_err(|err| err.into_pyexception(vm))? .as_path() .to_str() .unwrap() @@ -610,7 +643,7 @@ mod _os { #[pyfunction] fn chdir(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - env::set_current_dir(&path.path).map_err(|err| convert_io_error(vm, err)) + env::set_current_dir(&path.path).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -620,7 +653,7 @@ mod _os { #[pyfunction] fn rename(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - fs::rename(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) + fs::rename(src.path, dst.path).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -643,13 +676,11 @@ mod _os { #[pyfunction] fn urandom(size: usize, vm: &VirtualMachine) -> PyResult> { let mut buf = vec![0u8; size]; - match getrandom::getrandom(&mut buf) { - Ok(()) => Ok(buf), - Err(e) => match e.raw_os_error() { - Some(errno) => Err(convert_io_error(vm, io::Error::from_raw_os_error(errno))), - None => Err(vm.new_os_error("Getting random failed".to_owned())), - }, - } + getrandom::getrandom(&mut buf).map_err(|e| match e.raw_os_error() { + Some(errno) => io::Error::from_raw_os_error(errno).into_pyexception(vm), + None => vm.new_os_error("Getting random failed".to_owned()), + })?; + Ok(buf) } // this is basically what CPython has for Py_off_t; windows uses long long @@ -696,7 +727,7 @@ mod _os { #[pyfunction] fn link(src: PyPathLike, dst: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - fs::hard_link(src.path, dst.path).map_err(|err| convert_io_error(vm, err)) + fs::hard_link(src.path, dst.path).map_err(|err| err.into_pyexception(vm)) } #[derive(FromArgs)] @@ -751,7 +782,7 @@ mod _os { )) } }; - utime::set_file_times(&args.path.path, acc, modif).map_err(|e| convert_io_error(vm, e)) + utime::set_file_times(&args.path.path, acc, modif).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -962,32 +993,7 @@ mod posix { unsafe { File::from_raw_fd(raw_fileno as i32) } } - pub fn convert_nix_error(vm: &VirtualMachine, err: nix::Error) -> PyBaseExceptionRef { - let nix_error = match err { - nix::Error::InvalidPath => { - let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); - vm.new_exception_msg(exc_type, err.to_string()) - } - nix::Error::InvalidUtf8 => { - let exc_type = vm.ctx.exceptions.unicode_error.clone(); - vm.new_exception_msg(exc_type, err.to_string()) - } - nix::Error::UnsupportedOperation => vm.new_runtime_error(err.to_string()), - nix::Error::Sys(errno) => { - let exc_type = convert_nix_errno(vm, errno); - vm.new_exception_msg(exc_type, err.to_string()) - } - }; - - if let nix::Error::Sys(errno) = err { - vm.set_attr(nix_error.as_object(), "errno", vm.ctx.new_int(errno as i32)) - .unwrap(); - } - - nix_error - } - - fn convert_nix_errno(vm: &VirtualMachine, errno: Errno) -> PyClassRef { + pub(super) fn convert_nix_errno(vm: &VirtualMachine, errno: Errno) -> PyClassRef { match errno { Errno::EPERM => vm.ctx.exceptions.permission_error.clone(), _ => vm.ctx.exceptions.os_error.clone(), @@ -1089,14 +1095,14 @@ mod posix { return Ok(metadata.is_ok()); } - let metadata = metadata.map_err(|err| convert_io_error(vm, err))?; + let metadata = metadata.map_err(|err| err.into_pyexception(vm))?; let user_id = metadata.uid(); let group_id = metadata.gid(); let mode = metadata.mode(); let perm = get_right_permission(mode, Uid::from_raw(user_id), Gid::from_raw(group_id)) - .map_err(|err| convert_nix_error(vm, err))?; + .map_err(|err| err.into_pyexception(vm))?; let r_ok = !flags.contains(AccessFlags::R_OK) || perm.is_readable; let w_ok = !flags.contains(AccessFlags::W_OK) || perm.is_writable; @@ -1154,14 +1160,10 @@ mod posix { let get_stats = move || -> io::Result { let meta = match file { - Either::A(path) => { - let path = make_path(vm, &path, &dir_fd); - if follow_symlinks.follow_symlinks { - fs::metadata(path)? - } else { - fs::symlink_metadata(path)? - } - } + Either::A(path) => fs_metadata( + make_path(vm, &path, &dir_fd), + follow_symlinks.follow_symlinks, + )?, Either::B(fno) => { let file = rust_file(fno); let meta = file.metadata()?; @@ -1185,7 +1187,7 @@ mod posix { .into_obj(vm)) }; - get_stats().map_err(|err| convert_io_error(vm, err)) + get_stats().map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -1198,13 +1200,13 @@ mod posix { ) -> PyResult<()> { use std::os::unix::fs as unix_fs; let dst = make_path(vm, &dst, &dir_fd); - unix_fs::symlink(src.path, dst).map_err(|err| convert_io_error(vm, err)) + unix_fs::symlink(src.path, dst).map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[pyfunction] fn chroot(path: PyPathLike, vm: &VirtualMachine) -> PyResult<()> { - nix::unistd::chroot(&*path.path).map_err(|err| convert_nix_error(vm, err)) + nix::unistd::chroot(&*path.path).map_err(|err| err.into_pyexception(vm)) } // As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) @@ -1249,16 +1251,15 @@ mod posix { }; match path { - Either::A(p) => nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) - .map_err(|err| convert_nix_error(vm, err)), + Either::A(p) => nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag), Either::B(fd) => { let path = fs::read_link(format!("/proc/self/fd/{}", fd)).map_err(|_| { vm.new_os_error(String::from("Cannot find path for specified fd")) })?; nix::unistd::fchownat(dir_fd, &path, uid, gid, flag) - .map_err(|err| convert_nix_error(vm, err)) } } + .map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] @@ -1298,7 +1299,7 @@ mod posix { let flags = fcntl(fd, FcntlArg::F_GETFD); match flags { Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), - Err(err) => Err(convert_nix_error(vm, err)), + Err(err) => Err(err.into_pyexception(vm)), } } @@ -1315,7 +1316,7 @@ mod posix { #[pyfunction] fn set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - raw_set_inheritable(fd as RawFd, inheritable).map_err(|err| convert_nix_error(vm, err)) + raw_set_inheritable(fd as RawFd, inheritable).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -1325,7 +1326,7 @@ mod posix { let flags = fcntl(fd, FcntlArg::F_GETFL); match flags { Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), - Err(err) => Err(convert_nix_error(vm, err)), + Err(err) => Err(err.into_pyexception(vm)), } } @@ -1344,14 +1345,14 @@ mod posix { } Ok(()) }; - _set_flag().map_err(|err| convert_nix_error(vm, err)) + _set_flag().map_err(|err: nix::Error| err.into_pyexception(vm)) } #[pyfunction] fn pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { use nix::unistd::close; use nix::unistd::pipe; - let (rfd, wfd) = pipe().map_err(|err| convert_nix_error(vm, err))?; + let (rfd, wfd) = pipe().map_err(|err| err.into_pyexception(vm))?; set_inheritable(rfd.into(), false, vm) .and_then(|_| set_inheritable(wfd.into(), false, vm)) .map_err(|err| { @@ -1377,7 +1378,7 @@ mod posix { use nix::fcntl::OFlag; use nix::unistd::pipe2; let oflags = OFlag::from_bits_truncate(flags); - pipe2(oflags).map_err(|err| convert_nix_error(vm, err)) + pipe2(oflags).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -1398,19 +1399,15 @@ mod posix { follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult<()> { - use std::os::unix::fs::PermissionsExt; - let path = make_path(vm, &path, &dir_fd); - let metadata = if follow_symlinks.follow_symlinks { - fs::metadata(path) - } else { - fs::symlink_metadata(path) + let body = move || { + use std::os::unix::fs::PermissionsExt; + let meta = fs_metadata(path, follow_symlinks.follow_symlinks)?; + let mut permissions = meta.permissions(); + permissions.set_mode(mode); + fs::set_permissions(path, permissions) }; - let meta = metadata.map_err(|err| convert_io_error(vm, err))?; - let mut permissions = meta.permissions(); - permissions.set_mode(mode); - fs::set_permissions(path, permissions).map_err(|err| convert_io_error(vm, err))?; - Ok(()) + body().map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -1439,7 +1436,7 @@ mod posix { unistd::execv(&path, &argv) .map(|_ok| ()) - .map_err(|err| convert_nix_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -1464,7 +1461,7 @@ mod posix { fn getpgid(pid: u32, vm: &VirtualMachine) -> PyResult { match unistd::getpgid(Some(Pid::from_raw(pid as i32))) { Ok(pgid) => Ok(vm.new_int(pgid.as_raw())), - Err(err) => Err(convert_nix_error(vm, err)), + Err(err) => Err(err.into_pyexception(vm)), } } @@ -1478,7 +1475,7 @@ mod posix { fn getsid(pid: u32, vm: &VirtualMachine) -> PyResult { match unistd::getsid(Some(Pid::from_raw(pid as i32))) { Ok(sid) => Ok(vm.new_int(sid.as_raw())), - Err(err) => Err(convert_nix_error(vm, err)), + Err(err) => Err(err.into_pyexception(vm)), } } @@ -1496,19 +1493,19 @@ mod posix { #[pyfunction] fn setgid(gid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setgid(Gid::from_raw(gid)).map_err(|err| convert_nix_error(vm, err)) + unistd::setgid(Gid::from_raw(gid)).map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[pyfunction] fn setegid(egid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setegid(Gid::from_raw(egid)).map_err(|err| convert_nix_error(vm, err)) + unistd::setegid(Gid::from_raw(egid)).map_err(|err| err.into_pyexception(vm)) } #[pyfunction] fn setpgid(pid: u32, pgid: u32, vm: &VirtualMachine) -> PyResult<()> { unistd::setpgid(Pid::from_raw(pid as i32), Pid::from_raw(pgid as i32)) - .map_err(|err| convert_nix_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] @@ -1516,25 +1513,25 @@ mod posix { fn setsid(vm: &VirtualMachine) -> PyResult<()> { unistd::setsid() .map(|_ok| ()) - .map_err(|err| convert_nix_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } #[pyfunction] fn setuid(uid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setuid(Uid::from_raw(uid)).map_err(|err| convert_nix_error(vm, err)) + unistd::setuid(Uid::from_raw(uid)).map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[pyfunction] fn seteuid(euid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) + unistd::seteuid(Uid::from_raw(euid)).map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[pyfunction] fn setreuid(ruid: u32, euid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setuid(Uid::from_raw(ruid)).map_err(|err| convert_nix_error(vm, err))?; - unistd::seteuid(Uid::from_raw(euid)).map_err(|err| convert_nix_error(vm, err)) + unistd::setuid(Uid::from_raw(ruid)).map_err(|err| err.into_pyexception(vm))?; + unistd::seteuid(Uid::from_raw(euid)).map_err(|err| err.into_pyexception(vm)) } // cfg from nix @@ -1551,24 +1548,21 @@ mod posix { Uid::from_raw(euid), Uid::from_raw(suid), ) - .map_err(|err| convert_nix_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[pyfunction] fn openpty(vm: &VirtualMachine) -> PyResult { - match nix::pty::openpty(None, None) { - Ok(r) => Ok(vm - .ctx - .new_tuple(vec![vm.new_int(r.master), vm.new_int(r.slave)])), - Err(err) => Err(convert_nix_error(vm, err)), - } + let r = nix::pty::openpty(None, None).map_err(|err| err.into_pyexception(vm))?; + Ok(vm + .ctx + .new_tuple(vec![vm.new_int(r.master), vm.new_int(r.slave)])) } #[pyfunction] fn ttyname(fd: i32, vm: &VirtualMachine) -> PyResult { - use libc::ttyname; - let name = unsafe { ttyname(fd) }; + let name = unsafe { libc::ttyname(fd) }; if name.is_null() { Err(errno_err(vm)) } else { @@ -1619,7 +1613,7 @@ mod posix { #[pyfunction] fn uname(vm: &VirtualMachine) -> PyResult { - let info = uname::uname().map_err(|err| convert_io_error(vm, err))?; + let info = uname::uname().map_err(|err| err.into_pyexception(vm))?; Ok(UnameResult { sysname: info.sysname, nodename: info.nodename, @@ -1693,7 +1687,7 @@ mod posix { Gid::from_raw(egid), Gid::from_raw(sgid), ) - .map_err(|err| convert_nix_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } // cfg from nix @@ -1724,7 +1718,7 @@ mod posix { fn initgroups(user_name: PyStringRef, gid: u32, vm: &VirtualMachine) -> PyResult<()> { let user = ffi::CString::new(user_name.as_str()).unwrap(); let gid = Gid::from_raw(gid); - unistd::initgroups(&user, gid).map_err(|err| convert_nix_error(vm, err)) + unistd::initgroups(&user, gid).map_err(|err| err.into_pyexception(vm)) } // cfg from nix @@ -1744,7 +1738,7 @@ mod posix { }) .collect::, _>>()?; let ret = unistd::setgroups(&gids); - ret.map_err(|err| convert_nix_error(vm, err)) + ret.map_err(|err| err.into_pyexception(vm)) } fn envp_from_dict(dict: PyDictRef, vm: &VirtualMachine) -> PyResult> { @@ -1972,7 +1966,7 @@ mod posix { fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { let mut status = 0; let pid = unsafe { libc::waitpid(pid, &mut status, opt) }; - let pid = Errno::result(pid).map_err(|e| convert_nix_error(vm, e))?; + let pid = Errno::result(pid).map_err(|err| err.into_pyexception(vm))?; Ok((pid, status)) } #[pyfunction] @@ -2005,7 +1999,7 @@ mod posix { ws_ypixel: 0, }; unsafe { winsz(fd.unwrap_or(libc::STDOUT_FILENO), &mut w) } - .map_err(|e| convert_nix_error(vm, e))?; + .map_err(|err| err.into_pyexception(vm))?; (w.ws_col.into(), w.ws_row.into()) } }; @@ -2122,7 +2116,7 @@ mod posix { #[cfg(unix)] use posix as platform; #[cfg(unix)] -pub(crate) use posix::{convert_nix_error, raw_set_inheritable as set_inheritable}; +pub(crate) use posix::raw_set_inheritable; #[cfg(windows)] #[pymodule] @@ -2184,16 +2178,18 @@ mod nt { _dir_fd: DirFd, vm: &VirtualMachine, ) -> PyResult<()> { - use std::os::windows::fs as win_fs; - let meta = fs::metadata(src.path.clone()).map_err(|err| convert_io_error(vm, err))?; - let ret = if meta.is_file() { - win_fs::symlink_file(src.path, dst.path) - } else if meta.is_dir() { - win_fs::symlink_dir(src.path, dst.path) - } else { - panic!("Unknown file type"); + let body = move || { + use std::os::windows::fs as win_fs; + let meta = fs::metadata(src.path.clone())?; + if meta.is_file() { + win_fs::symlink_file(src.path, dst.path) + } else if meta.is_dir() { + win_fs::symlink_dir(src.path, dst.path) + } else { + panic!("Unknown file type"); + } }; - ret.map_err(|err| convert_io_error(vm, err)) + body().map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -2259,10 +2255,7 @@ mod nt { let get_stats = move || -> io::Result { let meta = match file { - Either::A(path) => match follow_symlinks.follow_symlinks { - true => fs::metadata(path.path)?, - false => fs::symlink_metadata(path.path)?, - }, + Either::A(path) => fs_metadata(path.path, follow_symlinks.follow_symlinks)?, Either::B(fno) => { let f = rust_file(fno); let meta = f.metadata()?; @@ -2286,7 +2279,7 @@ mod nt { .into_obj(vm)) }; - get_stats().map_err(|e| convert_io_error(vm, e)) + get_stats().map_err(|err| err.into_pyexception(vm)) } #[pyfunction] @@ -2304,10 +2297,10 @@ mod nt { } else { fs::symlink_metadata(path) }; - let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + let meta = metadata.map_err(|err| err.into_pyexception(vm))?; let mut permissions = meta.permissions(); permissions.set_readonly(mode & S_IWRITE != 0); - fs::set_permissions(path, permissions).map_err(|err| convert_io_error(vm, err)) + fs::set_permissions(path, permissions).map_err(|err| err.into_pyexception(vm)) } // cwait is available on MSVC only (according to CPython) @@ -2442,7 +2435,7 @@ mod nt { }); } - pub(super) fn support_funcs(vm: &VirtualMachine) -> Vec { + pub(super) fn support_funcs(_vm: &VirtualMachine) -> Vec { Vec::new() } } diff --git a/vm/src/stdlib/posixsubprocess.rs b/vm/src/stdlib/posixsubprocess.rs index 41928ddda..363b317d3 100644 --- a/vm/src/stdlib/posixsubprocess.rs +++ b/vm/src/stdlib/posixsubprocess.rs @@ -3,10 +3,11 @@ use std::convert::Infallible as Never; use std::ffi::{CStr, CString}; use std::io::{self, prelude::*}; +use crate::exceptions::IntoPyException; use crate::pyobject::{PyObjectRef, PyResult, PySequence, TryFromObject}; use crate::VirtualMachine; -use super::os::{convert_nix_error, set_inheritable, PyPathLike}; +use super::os; macro_rules! gen_args { ($($field:ident: $t:ty),*$(,)?) => { @@ -22,7 +23,7 @@ struct CStrPathLike { } impl TryFromObject for CStrPathLike { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - let s = PyPathLike::try_from_object(vm, obj)?.into_bytes(); + let s = os::PyPathLike::try_from_object(vm, obj)?.into_bytes(); let s = CString::new(s) .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; Ok(CStrPathLike { s }) @@ -53,7 +54,7 @@ fn _posixsubprocess_fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResu let argv = &argv; let envp = args.env_list.as_ref().map(|s| cstrs_to_ptrs(s.as_slice())); let envp = envp.as_deref(); - match nix::unistd::fork().map_err(|e| convert_nix_error(vm, e))? { + match nix::unistd::fork().map_err(|err| err.into_pyexception(vm))? { nix::unistd::ForkResult::Child => exec(&args, ProcArgs { argv, envp }), nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()), } @@ -84,7 +85,7 @@ fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! { fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result { for &fd in args.fds_to_keep.as_slice() { if fd != args.errpipe_write { - set_inheritable(fd, true)? + os::raw_set_inheritable(fd, true)? } } @@ -97,7 +98,7 @@ fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result { let c2pwrite = if args.c2pwrite == 0 { let fd = unistd::dup(args.c2pwrite)?; - set_inheritable(fd, true)?; + os::raw_set_inheritable(fd, true)?; fd } else { args.c2pwrite @@ -106,12 +107,12 @@ fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result { let mut errwrite = args.errwrite; while errwrite == 0 || errwrite == 1 { errwrite = unistd::dup(errwrite)?; - set_inheritable(errwrite, true)?; + os::raw_set_inheritable(errwrite, true)?; } let dup_into_stdio = |fd, io_fd| { if fd == io_fd { - set_inheritable(fd, true) + os::raw_set_inheritable(fd, true) } else if fd != -1 { unistd::dup2(fd, io_fd).map(drop) } else { diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index 24c4bd5a7..b73286635 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -1,4 +1,4 @@ -use super::os::convert_nix_error; +use crate::exceptions::IntoPyException; use crate::obj::objint::PyIntRef; use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult}; @@ -44,7 +44,7 @@ impl From for Passwd { } fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { - match User::from_name(name.as_str()).map_err(|e| convert_nix_error(vm, e))? { + match User::from_name(name.as_str()).map_err(|err| err.into_pyexception(vm))? { Some(user) => Ok(Passwd::from(user) .into_struct_sequence(vm, vm.try_class("pwd", "struct_passwd")?)? .into_object()), @@ -59,7 +59,7 @@ fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { fn pwd_getpwuid(uid: PyIntRef, vm: &VirtualMachine) -> PyResult { let uid_t = libc::uid_t::try_from(uid.as_bigint()).map(unistd::Uid::from_raw); let user = match uid_t { - Ok(uid) => User::from_uid(uid).map_err(|e| convert_nix_error(vm, e))?, + Ok(uid) => User::from_uid(uid).map_err(|err| err.into_pyexception(vm))?, Err(_) => None, }; match user { diff --git a/vm/src/stdlib/select.rs b/vm/src/stdlib/select.rs index ed4d0b8fa..a7dc34bdf 100644 --- a/vm/src/stdlib/select.rs +++ b/vm/src/stdlib/select.rs @@ -1,3 +1,4 @@ +use crate::exceptions::IntoPyException; use crate::function::OptionalOption; use crate::pyobject::{Either, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; @@ -177,7 +178,7 @@ fn select_select( }; if select_res < 0 { - return Err(super::os::convert_io_error(vm, err)); + return Err(err.into_pyexception(vm)); } let set2list = |list: Vec, mut set: FdSet| { diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 2ae76905e..4c2fa8cec 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,11 +10,8 @@ use gethostname::gethostname; use nix::unistd::sethostname; use socket2::{Domain, Protocol, Socket, Type as SocketType}; -use super::os::convert_io_error; -#[cfg(unix)] -use super::os::convert_nix_error; use crate::byteslike::PyBytesLike; -use crate::exceptions::PyBaseExceptionRef; +use crate::exceptions::{IntoPyException, PyBaseExceptionRef}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytearray::PyByteArrayRef; use crate::obj::objbytes::PyBytesRef; @@ -481,7 +478,7 @@ fn socket_gethostname(vm: &VirtualMachine) -> PyResult { #[cfg(all(unix, not(target_os = "redox")))] fn socket_sethostname(hostname: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - sethostname(hostname.as_str()).map_err(|err| convert_nix_error(vm, err)) + sethostname(hostname.as_str()).map_err(|err| err.into_pyexception(vm)) } fn socket_inet_aton(ip_string: PyStringRef, vm: &VirtualMachine) -> PyResult { @@ -648,7 +645,7 @@ fn convert_sock_error(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef let socket_timeout = vm.class("_socket", "timeout"); vm.new_exception_msg(socket_timeout, "Timed out".to_owned()) } else { - convert_io_error(vm, err) + err.into_pyexception(vm) } } diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index a862cb8e0..a86506102 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -1,6 +1,7 @@ use super::socket::PySocketRef; use crate::byteslike::PyBytesLike; -use crate::exceptions::PyBaseExceptionRef; +use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; +use crate::exceptions::{IntoPyException, PyBaseExceptionRef}; use crate::function::OptionalArg; use crate::obj::objbytearray::PyByteArrayRef; use crate::obj::objstr::PyStringRef; @@ -11,11 +12,6 @@ use crate::pyobject::{ use crate::types::create_type; use crate::VirtualMachine; -use crate::common::cell::{PyRwLock, PyRwLockWriteGuard}; -use std::convert::TryFrom; -use std::ffi::{CStr, CString}; -use std::fmt; - use foreign_types_shared::{ForeignType, ForeignTypeRef}; use openssl::{ asn1::{Asn1Object, Asn1ObjectRef}, @@ -24,6 +20,9 @@ use openssl::{ ssl::{self, SslContextBuilder, SslOptions, SslVerifyMode}, x509::{self, X509Object, X509Ref, X509}, }; +use std::convert::TryFrom; +use std::ffi::{CStr, CString}; +use std::fmt; mod sys { #![allow(non_camel_case_types, unused)] @@ -147,7 +146,7 @@ fn ssl_enum_certificates(store_name: PyStringRef, vm: &VirtualMachine) -> PyResu }); let certs = certs .collect::, _>>() - .map_err(|e| super::os::convert_io_error(vm, e))?; + .map_err(|e: std::io::Error| e.into_pyexception(vm))?; Ok(vm.ctx.new_list(certs)) } @@ -652,7 +651,7 @@ fn convert_openssl_error(vm: &VirtualMachine, err: ErrorStack) -> PyBaseExceptio } fn convert_ssl_error(vm: &VirtualMachine, e: ssl::Error) -> PyBaseExceptionRef { match e.into_io_error() { - Ok(io_err) => super::os::convert_io_error(vm, io_err), + Ok(io_err) => io_err.into_pyexception(vm), Err(e) => convert_openssl_error(vm, e.ssl_error().unwrap().clone()), } } diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index 197127595..7d21b7e4f 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -1,5 +1,5 @@ /// Implementation of the _thread module -use crate::exceptions; +use crate::exceptions::{self, IntoPyException}; use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::obj::objdict::PyDictRef; use crate::obj::objstr::PyStringRef; @@ -256,7 +256,7 @@ fn thread_start_new_thread( vm.state.thread_count.fetch_add(1); thread_to_id(&handle.thread()) }) - .map_err(|err| super::os::convert_io_error(vm, err)) + .map_err(|err| err.into_pyexception(vm)) } thread_local!(static SENTINELS: RefCell> = RefCell::default()); diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 550b99d05..79c3098ea 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -1,15 +1,14 @@ #![allow(non_snake_case)] use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; -use std::convert::TryInto; -use std::io; - -use super::os; +use crate::exceptions::IntoPyException; use crate::function::OptionalArg; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::VirtualMachine; +use std::convert::TryInto; +use std::io; use winapi::shared::winerror; use winreg::{enums::RegType, RegKey, RegValue}; @@ -116,7 +115,7 @@ fn winreg_OpenKey( let subkey = subkey.as_ref().map_or("", |s| s.as_str()); let key = key .with_key(|k| k.open_subkey_with_flags(subkey, access)) - .map_err(|e| os::convert_io_error(vm, e))?; + .map_err(|e| e.into_pyexception(vm))?; Ok(PyHKEY::new(key)) } @@ -128,7 +127,7 @@ fn winreg_QueryValue( ) -> PyResult { let subkey = subkey.as_ref().map_or("", |s| s.as_str()); key.with_key(|k| k.get_value(subkey)) - .map_err(|e| os::convert_io_error(vm, e)) + .map_err(|e| e.into_pyexception(vm)) } fn winreg_QueryValueEx( @@ -138,7 +137,7 @@ fn winreg_QueryValueEx( ) -> PyResult<(PyObjectRef, usize)> { let subkey = subkey.as_ref().map_or("", |s| s.as_str()); key.with_key(|k| k.get_raw_value(subkey)) - .map_err(|e| os::convert_io_error(vm, e)) + .map_err(|e| e.into_pyexception(vm)) .and_then(|regval| { let ty = regval.vtype.clone() as usize; Ok((reg_to_py(regval, vm)?, ty)) @@ -152,7 +151,7 @@ fn winreg_EnumKey(key: Hkey, index: u32, vm: &VirtualMachine) -> PyResult Date: Wed, 5 Aug 2020 23:15:35 +0900 Subject: [PATCH 205/209] impl IntoPyException for FormatParseError --- vm/src/format.rs | 13 ++++++------- vm/src/obj/objstr.rs | 5 +++-- vm/src/stdlib/string.rs | 5 +++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/vm/src/format.rs b/vm/src/format.rs index c70a7873a..42552b131 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,4 +1,4 @@ -use crate::exceptions::PyBaseExceptionRef; +use crate::exceptions::{IntoPyException, PyBaseExceptionRef}; use crate::function::PyFuncArgs; use crate::obj::{objstr, objtype}; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; @@ -563,9 +563,8 @@ pub(crate) enum FormatParseError { InvalidCharacterAfterRightBracket, } -impl FormatParseError { - // TODO: IntoPyException - pub fn into_pyobject(self, vm: &VirtualMachine) -> PyBaseExceptionRef { +impl IntoPyException for FormatParseError { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { match self { FormatParseError::UnmatchedBracket => { vm.new_value_error("expected '}' before end of string".to_owned()) @@ -812,8 +811,8 @@ impl FormatString { preconversion_spec, format_spec, } => { - let FieldName { field_type, parts } = - FieldName::parse(field_name.as_str()).map_err(|e| e.into_pyobject(vm))?; + let FieldName { field_type, parts } = FieldName::parse(field_name.as_str()) + .map_err(|e| e.into_pyexception(vm))?; let mut argument = field_func(&field_type)?; @@ -832,7 +831,7 @@ impl FormatString { } let nested_format = - FormatString::from_str(&format_spec).map_err(|e| e.into_pyobject(vm))?; + FormatString::from_str(&format_spec).map_err(|e| e.into_pyexception(vm))?; let format_spec = nested_format.format_internal(vm, field_func)?; let value = diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 093e6e494..cf71dccb2 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -19,6 +19,7 @@ use super::objiter; use super::objnone::PyNone; use super::objsequence::{PySliceableSequence, SequenceIndex}; use super::objtype::{self, PyClassRef}; +use crate::exceptions::IntoPyException; use crate::format::{FormatSpec, FormatString, FromTemplate}; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::pyobject::{ @@ -590,7 +591,7 @@ impl PyString { fn format(&self, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { match FormatString::from_str(&self.value) { Ok(format_string) => format_string.format(&args, vm), - Err(err) => Err(err.into_pyobject(vm)), + Err(err) => Err(err.into_pyexception(vm)), } } @@ -602,7 +603,7 @@ impl PyString { fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult { match FormatString::from_str(&self.value) { Ok(format_string) => format_string.format_map(&mapping, vm), - Err(err) => Err(err.into_pyobject(vm)), + Err(err) => Err(err.into_pyexception(vm)), } } diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 8e73815c8..6b9314c38 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -7,6 +7,7 @@ pub(crate) use _string::make_module; mod _string { use std::mem; + use crate::exceptions::IntoPyException; use crate::format::{ FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, }; @@ -34,7 +35,7 @@ mod _string { #[pyfunction] fn formatter_parser(text: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_string = - FormatString::from_str(text.as_str()).map_err(|e| e.into_pyobject(vm))?; + FormatString::from_str(text.as_str()).map_err(|e| e.into_pyexception(vm))?; let mut result = Vec::new(); let mut literal = String::new(); @@ -73,7 +74,7 @@ mod _string { text: PyStringRef, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, PyList)> { - let field_name = FieldName::parse(text.as_str()).map_err(|e| e.into_pyobject(vm))?; + let field_name = FieldName::parse(text.as_str()).map_err(|e| e.into_pyexception(vm))?; let first = match field_name.field_type { FieldType::Auto => vm.new_str("".to_owned()), From 310989a4d11fc22e78cd8d4033a79c237cb9dcdc Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 6 Aug 2020 01:50:55 +0900 Subject: [PATCH 206/209] Reuse std::convert::Infallible as Never --- vm/src/pyobject.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4a4118aaa..bf41a4ef1 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -834,8 +834,7 @@ pub trait IdProtocol { } } -#[derive(Debug)] -enum Never {} +type Never = std::convert::Infallible; impl PyValue for Never { fn class(_vm: &VirtualMachine) -> PyClassRef { From bc8a0b9f6e7ba15708fcce7be1870ad39d91a77a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 31 Jul 2020 03:37:59 +0900 Subject: [PATCH 207/209] DictKey for str shares same hash with PyString --- common/src/hash.rs | 37 +++++++++++++++-------- vm/src/dictdatatype.rs | 66 +++++++++++++++++++----------------------- vm/src/obj/objint.rs | 2 +- vm/src/obj/objstr.rs | 2 +- vm/src/vm.rs | 8 ++--- 5 files changed, 61 insertions(+), 54 deletions(-) diff --git a/common/src/hash.rs b/common/src/hash.rs index 65a11f224..58c554f43 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -23,6 +23,17 @@ pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8; // pub const CUTOFF: usize = 7; +#[inline] +pub fn mod_int(value: i64) -> PyHash { + value % MODULUS as i64 +} + +pub fn hash_value(data: &T) -> PyHash { + let mut hasher = DefaultHasher::new(); + data.hash(&mut hasher); + mod_int(hasher.finish() as PyHash) +} + pub fn hash_float(value: f64) -> PyHash { // cpython _Py_HashDouble if !value.is_finite() { @@ -68,12 +79,6 @@ pub fn hash_float(value: f64) -> PyHash { x as PyHash * value.signum() as PyHash } -pub fn hash_value(data: &T) -> PyHash { - let mut hasher = DefaultHasher::new(); - data.hash(&mut hasher); - hasher.finish() as PyHash -} - pub fn hash_iter<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result where I: IntoIterator, @@ -84,7 +89,7 @@ where let item_hash = hashf(element)?; item_hash.hash(&mut hasher); } - Ok(hasher.finish() as PyHash) + Ok(mod_int(hasher.finish() as PyHash)) } pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result @@ -98,12 +103,20 @@ where // xor is commutative and hash should be independent of order hash ^= item_hash; } - Ok(hash) + Ok(mod_int(hash)) } pub fn hash_bigint(value: &BigInt) -> PyHash { - match value.to_i64() { - Some(i64_value) => (i64_value % MODULUS as i64), - None => (value % MODULUS).to_i64().unwrap(), - } + value.to_i64().map_or_else( + || { + (value % MODULUS).to_i64().unwrap_or_else(|| + // guaranteed to be safe by mod + unsafe { std::hint::unreachable_unchecked() }) + }, + mod_int, + ) +} + +pub fn hash_str(value: &str) -> PyHash { + hash_value(value) } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 912455410..5a64c875f 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,17 +1,18 @@ -use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; -use crate::obj::objstr::{PyString, PyStringRef}; -use crate::pyobject::{IdProtocol, IntoPyObject, PyObjectRef, PyResult}; -use crate::vm::VirtualMachine; -use num_bigint::ToBigInt; -use rustpython_common::hash; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 /// And: http://code.activestate.com/recipes/578375/ -use std::collections::{hash_map::DefaultHasher, HashMap}; -use std::hash::{Hash, Hasher}; +use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; +use crate::obj::objstr::{PyString, PyStringRef}; +use crate::pyobject::{IdProtocol, IntoPyObject, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; +use rustpython_common::hash; +use std::collections::HashMap; use std::mem::size_of; +// HashIndex is intended to be same size with hash::PyHash +// but it doesn't mean the values are compatible with actual pyhash value + /// hash value of an object returned by __hash__ type HashValue = hash::PyHash; /// index calculated by resolving collision @@ -303,7 +304,7 @@ impl Dict { /// Lookup the index for the given key. #[cfg_attr(feature = "flame-it", flame("Dict"))] fn lookup(&self, vm: &VirtualMachine, key: &K) -> PyResult { - let hash_value = key.do_hash(vm)?; + let hash_value = key.key_hash(vm)?; let perturb = hash_value; let mut hash_index: HashIndex = hash_value; 'outer: loop { @@ -314,7 +315,7 @@ impl Dict { let index = inner.indices[&hash_index]; if let Some(entry) = &inner.entries[index] { // Okay, we have an entry at this place - if key.do_is(&entry.key) { + if key.key_is(&entry.key) { // Literally the same object break 'outer Ok(LookupResult::Existing(index)); } else if entry.hash == hash_value { @@ -337,7 +338,7 @@ impl Dict { // warn!("Perturb value: {}", i); }; // This comparison needs to be done outside the lock. - if key.do_eq(vm, &entry.key)? { + if key.key_eq(vm, &entry.key)? { break Ok(LookupResult::Existing(index)); } else { // entry mismatch. @@ -412,40 +413,37 @@ enum LookupResult { /// - PyObjectRef -> arbitrary python type used as key /// - str -> string reference used as key, this is often used internally pub trait DictKey: IntoPyObject { - fn do_hash(&self, vm: &VirtualMachine) -> PyResult; - fn do_is(&self, other: &PyObjectRef) -> bool; - fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult; + fn key_hash(&self, vm: &VirtualMachine) -> PyResult; + fn key_is(&self, other: &PyObjectRef) -> bool; + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult; } /// Implement trait for PyObjectRef such that we can use python objects /// to index dictionaries. impl DictKey for PyObjectRef { - fn do_hash(&self, vm: &VirtualMachine) -> PyResult { - let raw_hash = vm._hash(self)?; - let mut hasher = DefaultHasher::new(); - raw_hash.hash(&mut hasher); - Ok(hasher.finish() as HashValue) + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + vm._hash(self) } - fn do_is(&self, other: &PyObjectRef) -> bool { + fn key_is(&self, other: &PyObjectRef) -> bool { self.is(other) } - fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { vm.identical_or_equal(self, other_key) } } impl DictKey for PyStringRef { - fn do_hash(&self, _vm: &VirtualMachine) -> PyResult { + fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { Ok(self.hash()) } - fn do_is(&self, other: &PyObjectRef) -> bool { + fn key_is(&self, other: &PyObjectRef) -> bool { self.is(other) } - fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { if self.is(other_key) { Ok(true) } else if let Some(py_str_value) = other_key.payload::() { @@ -461,28 +459,24 @@ impl DictKey for PyStringRef { /// Implement trait for the str type, so that we can use strings /// to index dictionaries. impl DictKey for &str { - fn do_hash(&self, _vm: &VirtualMachine) -> PyResult { + fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { // follow a similar route as the hashing of PyStringRef - let raw_hash = hash::hash_value(*self).to_bigint().unwrap(); - let raw_hash = hash::hash_bigint(&raw_hash); - let mut hasher = DefaultHasher::new(); - raw_hash.hash(&mut hasher); - Ok(hasher.finish() as HashValue) + Ok(hash::hash_str(*self)) } - fn do_is(&self, _other: &PyObjectRef) -> bool { + fn key_is(&self, _other: &PyObjectRef) -> bool { // No matter who the other pyobject is, we are never the same thing, since // we are a str, not a pyobject. false } - fn do_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObjectRef) -> PyResult { if let Some(py_str_value) = other_key.payload::() { Ok(py_str_value.as_str() == *self) } else { - // Fall back to PyString implementation. + // Fall back to PyObjectRef implementation. let s = vm.ctx.new_str(*self); - s.do_eq(vm, other_key) + s.key_eq(vm, other_key) } } } @@ -545,8 +539,8 @@ mod tests { let value1 = text; let value2 = vm.new_str(value1.to_owned()); - let hash1 = value1.do_hash(&vm).expect("Hash should not fail."); - let hash2 = value2.do_hash(&vm).expect("Hash should not fail."); + let hash1 = value1.key_hash(&vm).expect("Hash should not fail."); + let hash2 = value2.key_hash(&vm).expect("Hash should not fail."); assert_eq!(hash1, hash2); } } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 97be9bd6f..376979ea8 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -419,7 +419,7 @@ impl PyInt { } #[pymethod(name = "__hash__")] - pub fn hash(&self) -> hash::PyHash { + fn hash(&self) -> hash::PyHash { hash::hash_bigint(&self.value) } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index cf71dccb2..8e35d3a44 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -283,7 +283,7 @@ impl PyString { #[pymethod(name = "__hash__")] pub(crate) fn hash(&self) -> hash::PyHash { self.hash.load().unwrap_or_else(|| { - let hash = hash::hash_value(&self.value); + let hash = hash::hash_str(&self.value); self.hash.store(Some(hash)); hash }) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 99234ff4d..a7f235ae2 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1407,11 +1407,11 @@ impl VirtualMachine { } pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { + // TODO: tp_hash let hash_obj = self.call_method(obj, "__hash__", vec![])?; - if let Some(hash_value) = hash_obj.payload_if_subclass::(self) { - Ok(hash_value.hash()) - } else { - Err(self.new_type_error("__hash__ method should return an integer".to_owned())) + match hash_obj.payload_if_subclass::(self) { + Some(py_int) => Ok(rustpython_common::hash::hash_bigint(py_int.as_bigint())), + None => Err(self.new_type_error("__hash__ method should return an integer".to_owned())), } } From 3f6ebd3385d9b7a841bed4071438376e5f5c1e66 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 6 Aug 2020 20:18:56 +0900 Subject: [PATCH 208/209] Use From::from for PyValues instead of new --- vm/src/macros.rs | 4 +-- vm/src/obj/objbuiltinfunc.rs | 15 ++++++---- vm/src/obj/objbytearray.rs | 10 ++----- vm/src/obj/objbytes.rs | 10 ++----- vm/src/obj/objclassmethod.rs | 4 +-- vm/src/obj/objint.rs | 36 +++++++++++++++++------ vm/src/obj/objrange.rs | 22 +++++++------- vm/src/obj/objstaticmethod.rs | 8 +++-- vm/src/obj/objtype.rs | 4 +-- vm/src/pyobject.rs | 55 ++++++++++++++++++++++------------- vm/src/stdlib/hashlib.rs | 3 +- vm/src/stdlib/io.rs | 3 +- vm/src/stdlib/itertools.rs | 8 ++--- vm/src/stdlib/marshal.rs | 2 +- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/zlib.rs | 14 ++++----- 16 files changed, 115 insertions(+), 85 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index a879bdd3c..8ce00a1f1 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -94,7 +94,7 @@ macro_rules! py_namespace { /// use rustpython_vm::pyobject::PyValue; /// /// let vm: VirtualMachine = Default::default(); -/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// let obj = PyInt::from(0).into_ref(&vm).into_object(); /// assert_eq!( /// "int", /// match_class!(match obj.clone() { @@ -119,7 +119,7 @@ macro_rules! py_namespace { /// use rustpython_vm::pyobject::PyValue; /// /// let vm: VirtualMachine = Default::default(); -/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// let obj = PyInt::from(0).into_ref(&vm).into_object(); /// /// let int_value = match_class!(match obj { /// i @ PyInt => i.as_bigint().clone(), diff --git a/vm/src/obj/objbuiltinfunc.rs b/vm/src/obj/objbuiltinfunc.rs index deae985f5..28638ea8c 100644 --- a/vm/src/obj/objbuiltinfunc.rs +++ b/vm/src/obj/objbuiltinfunc.rs @@ -29,15 +29,17 @@ impl fmt::Debug for PyBuiltinFunction { } } -impl PyBuiltinFunction { - pub fn new(value: PyNativeFunc) -> Self { +impl From for PyBuiltinFunction { + fn from(value: PyNativeFunc) -> Self { Self { value, module: None, name: None, } } +} +impl PyBuiltinFunction { pub fn new_with_name(value: PyNativeFunc, module: PyStringRef, name: PyStringRef) -> Self { Self { value, @@ -86,12 +88,15 @@ impl fmt::Debug for PyBuiltinMethod { } } -impl PyBuiltinMethod { - pub fn new(value: PyNativeFunc) -> Self { +impl From for PyBuiltinMethod { + fn from(value: PyNativeFunc) -> Self { Self { - function: PyBuiltinFunction::new(value), + function: value.into(), } } +} + +impl PyBuiltinMethod { pub fn new_with_name(value: PyNativeFunc, module: PyStringRef, name: PyStringRef) -> Self { Self { function: PyBuiltinFunction::new_with_name(value, module, name), diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 49fad9223..47cb1ce74 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -42,12 +42,6 @@ pub struct PyByteArray { pub type PyByteArrayRef = PyRef; impl PyByteArray { - pub fn new(data: Vec) -> Self { - PyByteArray { - inner: PyRwLock::new(PyBytesInner { elements: data }), - } - } - fn from_inner(inner: PyBytesInner) -> Self { PyByteArray { inner: PyRwLock::new(inner), @@ -65,7 +59,9 @@ impl PyByteArray { impl From> for PyByteArray { fn from(elements: Vec) -> Self { - Self::new(elements) + Self { + inner: PyRwLock::new(PyBytesInner { elements }), + } } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 46003f1f3..372ccb5ee 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -41,12 +41,6 @@ pub struct PyBytes { pub type PyBytesRef = PyRef; impl PyBytes { - pub fn new(elements: Vec) -> Self { - PyBytes { - inner: PyBytesInner { elements }, - } - } - pub fn get_value(&self) -> &[u8] { &self.inner.elements } @@ -54,7 +48,9 @@ impl PyBytes { impl From> for PyBytes { fn from(elements: Vec) -> Self { - Self::new(elements) + Self { + inner: PyBytesInner { elements }, + } } } diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index edc579698..5b833aec0 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -33,8 +33,8 @@ pub struct PyClassMethod { } pub type PyClassMethodRef = PyRef; -impl PyClassMethod { - pub fn new(value: PyObjectRef) -> Self { +impl From for PyClassMethod { + fn from(value: PyObjectRef) -> Self { Self { callable: value } } } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 97be9bd6f..9d3b3ac68 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -16,8 +16,9 @@ use crate::bytesinner::PyBytesInner; use crate::format::FormatSpec; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, IntoPyResult, PyArithmaticValue, PyClassImpl, PyComparisonValue, - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IdProtocol, IntoPyObject, IntoPyRef, IntoPyResult, PyArithmaticValue, PyClassImpl, + PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::stdlib::array::PyArray; use crate::vm::VirtualMachine; @@ -52,15 +53,20 @@ impl fmt::Display for PyInt { pub type PyIntRef = PyRef; impl PyInt { - pub fn new>(i: T) -> Self { - PyInt { value: i.into() } - } - pub fn as_bigint(&self) -> &BigInt { &self.value } } +impl From for PyInt +where + T: Into, +{ + fn from(v: T) -> Self { + Self { value: v.into() } + } +} + impl IntoPyObject for BigInt { fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx.new_int(self) @@ -214,9 +220,21 @@ fn inner_truediv(i1: &BigInt, i2: &BigInt, vm: &VirtualMachine) -> PyResult { #[pyimpl(flags(BASETYPE))] impl PyInt { + fn raw_new(cls: PyClassRef, value: T, vm: &VirtualMachine) -> PyResult + where + T: Into, + { + if cls.is(&vm.ctx.int_type()) { + Ok(value.into().into_pyref(vm)) + } else { + PyInt::from(value).into_ref_with_type(vm, cls) + } + } + #[pyslot] fn tp_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResult { - PyInt::new(options.get_int_value(vm)?).into_ref_with_type(vm, cls) + let value = options.get_int_value(vm)?; + Self::raw_new(cls, value, vm) } #[inline] @@ -568,7 +586,7 @@ impl PyInt { ) } }; - PyInt::new(x).into_ref_with_type(vm, cls) + Self::raw_new(cls, x, vm) } #[pymethod] @@ -631,7 +649,7 @@ impl PyInt { } _ => (), } - Ok(PyBytes::new(bytes)) + Ok(bytes.into()) } #[pyproperty] fn real(&self, vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index ac8668f9c..53742f1d5 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -11,7 +11,7 @@ use super::objtype::PyClassRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - self, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + self, IntoPyRef, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -132,9 +132,9 @@ type PyRangeRef = PyRef; impl PyRange { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { - start: PyInt::new(BigInt::zero()).into_ref(vm), + start: (0).into_pyref(vm), stop, - step: PyInt::new(BigInt::one()).into_ref(vm), + step: (1).into_pyref(vm), } .into_ref_with_type(vm, cls) } @@ -146,7 +146,7 @@ impl PyRange { step: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let step = step.unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)); + let step = step.unwrap_or_else(|| (1).into_pyref(vm)); if step.as_bigint().is_zero() { return Err(vm.new_value_error("range() arg 3 must not be zero".to_owned())); } @@ -198,9 +198,9 @@ impl PyRange { }; let reversed = PyRange { - start: PyInt::new(new_start).into_ref(vm), - stop: PyInt::new(new_stop).into_ref(vm), - step: PyInt::new(-step).into_ref(vm), + start: new_start.into_pyref(vm), + stop: new_stop.into_pyref(vm), + step: (-step).into_pyref(vm), }; PyRangeIterator { @@ -349,15 +349,15 @@ impl PyRange { substop = (substop * range_step.as_bigint()) + range_start.as_bigint(); Ok(PyRange { - start: PyInt::new(substart).into_ref(vm), - stop: PyInt::new(substop).into_ref(vm), - step: PyInt::new(substep).into_ref(vm), + start: substart.into_pyref(vm), + stop: substop.into_pyref(vm), + step: substep.into_pyref(vm), } .into_ref(vm) .into_object()) } RangeIndex::Int(index) => match self.get(index.as_bigint()) { - Some(value) => Ok(PyInt::new(value).into_ref(vm).into_object()), + Some(value) => Ok(vm.ctx.new_int(value)), None => Err(vm.new_index_error("range object index out of range".to_owned())), }, } diff --git a/vm/src/obj/objstaticmethod.rs b/vm/src/obj/objstaticmethod.rs index f78842b2c..e9c3715e9 100644 --- a/vm/src/obj/objstaticmethod.rs +++ b/vm/src/obj/objstaticmethod.rs @@ -29,12 +29,14 @@ impl SlotDescriptor for PyStaticMethod { } } -#[pyimpl(with(SlotDescriptor), flags(BASETYPE))] -impl PyStaticMethod { - pub fn new(callable: PyObjectRef) -> Self { +impl From for PyStaticMethod { + fn from(callable: PyObjectRef) -> Self { Self { callable } } +} +#[pyimpl(with(SlotDescriptor), flags(BASETYPE))] +impl PyStaticMethod { #[pyslot] fn tp_new( cls: PyClassRef, diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 236c76216..1de26d7f2 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -298,13 +298,13 @@ impl PyClassRef { let mut attributes = dict.to_attributes(); if let Some(f) = attributes.get_mut("__new__") { if f.class().is(&vm.ctx.function_type()) { - *f = PyStaticMethod::new(f.clone()).into_ref(vm).into_object(); + *f = PyStaticMethod::from(f.clone()).into_ref(vm).into_object(); } } if let Some(f) = attributes.get_mut("__init_subclass__") { if f.class().is(&vm.ctx.function_type()) { - *f = PyClassMethod::new(f.clone()).into_ref(vm).into_object(); + *f = PyClassMethod::from(f.clone()).into_ref(vm).into_object(); } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index bf41a4ef1..056038eef 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use indexmap::IndexMap; use num_bigint::BigInt; use num_complex::Complex64; -use num_traits::{One, ToPrimitive, Zero}; +use num_traits::ToPrimitive; use crate::bytecode; use crate::exceptions::{self, PyBaseExceptionRef}; @@ -89,9 +89,6 @@ impl fmt::Display for PyObject { } } -const INT_CACHE_POOL_MIN: i32 = -5; -const INT_CACHE_POOL_MAX: i32 = 256; - #[derive(Debug)] pub struct PyContext { pub true_value: PyIntRef, @@ -132,6 +129,9 @@ impl PyValue for PyEllipsis { // Basic objects: impl PyContext { + pub const INT_CACHE_POOL_MIN: i32 = -5; + pub const INT_CACHE_POOL_MAX: i32 = 256; + pub fn new() -> Self { flame_guard!("init PyContext"); let types = TypeZoo::new(); @@ -151,17 +151,17 @@ impl PyContext { create_type("NotImplementedType", &types.type_type, &types.object_type); let not_implemented = create_object(PyNotImplemented, ¬_implemented_type); - let int_cache_pool = (INT_CACHE_POOL_MIN..=INT_CACHE_POOL_MAX) - .map(|v| create_object(PyInt::new(BigInt::from(v)), &types.int_type).into_object()) + let int_cache_pool = (Self::INT_CACHE_POOL_MIN..=Self::INT_CACHE_POOL_MAX) + .map(|v| PyObject::new(PyInt::from(BigInt::from(v)), types.int_type.clone(), None)) .collect(); - let true_value = create_object(PyInt::new(BigInt::one()), &types.bool_type); - let false_value = create_object(PyInt::new(BigInt::zero()), &types.bool_type); + let true_value = create_object(PyInt::from(1), &types.bool_type); + let false_value = create_object(PyInt::from(0), &types.bool_type); let empty_tuple = create_object(PyTuple::from(vec![]), &types.tuple_type); let tp_new_wrapper = create_object( - PyBuiltinFunction::new(objtype::tp_new_wrapper.into_func()), + PyBuiltinFunction::from(objtype::tp_new_wrapper.into_func()), &types.builtin_function_or_method_type, ) .into_object(); @@ -393,23 +393,23 @@ impl PyContext { #[inline] pub fn new_int + ToPrimitive>(&self, i: T) -> PyObjectRef { if let Some(i) = i.to_i32() { - if i >= INT_CACHE_POOL_MIN && i <= INT_CACHE_POOL_MAX { - let inner_idx = (i - INT_CACHE_POOL_MIN) as usize; + if i >= Self::INT_CACHE_POOL_MIN && i <= Self::INT_CACHE_POOL_MAX { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; return self.int_cache_pool[inner_idx].clone(); } } - PyObject::new(PyInt::new(i), self.int_type(), None) + PyObject::new(PyInt::from(i), self.int_type(), None) } #[inline] pub fn new_bigint(&self, i: &BigInt) -> PyObjectRef { if let Some(i) = i.to_i32() { - if i >= INT_CACHE_POOL_MIN && i <= INT_CACHE_POOL_MAX { - let inner_idx = (i - INT_CACHE_POOL_MIN) as usize; + if i >= Self::INT_CACHE_POOL_MIN && i <= Self::INT_CACHE_POOL_MAX { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; return self.int_cache_pool[inner_idx].clone(); } } - PyObject::new(PyInt::new(i.clone()), self.int_type(), None) + PyObject::new(PyInt::from(i.clone()), self.int_type(), None) } pub fn new_float(&self, value: f64) -> PyObjectRef { @@ -428,12 +428,12 @@ impl PyContext { } pub fn new_bytes(&self, data: Vec) -> PyObjectRef { - PyObject::new(objbytes::PyBytes::new(data), self.bytes_type(), None) + PyObject::new(objbytes::PyBytes::from(data), self.bytes_type(), None) } pub fn new_bytearray(&self, data: Vec) -> PyObjectRef { PyObject::new( - objbytearray::PyByteArray::new(data), + objbytearray::PyByteArray::from(data), self.bytearray_type(), None, ) @@ -486,7 +486,7 @@ impl PyContext { F: IntoPyNativeFunc, { PyObject::new( - PyBuiltinFunction::new(f.into_func()), + PyBuiltinFunction::from(f.into_func()), self.builtin_function_or_method_type(), None, ) @@ -509,7 +509,7 @@ impl PyContext { F: IntoPyNativeFunc, { PyObject::new( - PyBuiltinMethod::new(f.into_func()), + PyBuiltinMethod::from(f.into_func()), self.method_descriptor_type(), None, ) @@ -520,7 +520,7 @@ impl PyContext { F: IntoPyNativeFunc, { PyObject::new( - PyClassMethod::new(self.new_method(f)), + PyClassMethod::from(self.new_method(f)), self.classmethod_type(), None, ) @@ -530,7 +530,7 @@ impl PyContext { F: IntoPyNativeFunc, { PyObject::new( - PyStaticMethod::new(self.new_method(f)), + PyStaticMethod::from(self.new_method(f)), self.staticmethod_type(), None, ) @@ -1144,6 +1144,19 @@ pub trait TryFromObject: Sized { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult; } +pub trait IntoPyRef { + fn into_pyref(self, vm: &VirtualMachine) -> PyRef; +} + +impl IntoPyRef

for T +where + P: PyValue + IntoPyObject + From, +{ + fn into_pyref(self, vm: &VirtualMachine) -> PyRef

{ + PyRef::from_obj_unchecked(P::from(self).into_pyobject(vm)) + } +} + /// Implemented by any type that can be returned from a built-in Python function. /// /// `IntoPyObject` has a blanket implementation for any built-in object payload, diff --git a/vm/src/stdlib/hashlib.rs b/vm/src/stdlib/hashlib.rs index 262755279..cf85ce26a 100644 --- a/vm/src/stdlib/hashlib.rs +++ b/vm/src/stdlib/hashlib.rs @@ -74,8 +74,7 @@ impl PyHasher { #[pymethod(name = "digest")] fn digest(&self) -> PyBytes { - let result = self.get_digest(); - PyBytes::new(result) + self.get_digest().into() } #[pymethod(name = "hexdigest")] diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index bab8ab4d3..9e56a1184 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -536,7 +536,8 @@ fn io_base_readlines(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { fn raw_io_base_read(instance: PyObjectRef, size: OptionalSize, vm: &VirtualMachine) -> PyResult { if let Some(size) = size.to_usize() { - let b = PyByteArray::new(vec![0; size]).into_ref(vm); + // FIXME: unnessessary zero-init + let b = PyByteArray::from(vec![0; size]).into_ref(vm); let n = >::try_from_object( vm, vm.call_method(&instance, "readinto", vec![b.as_object().clone()])?, diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 98b57f0f0..24889a2ce 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -17,8 +17,8 @@ mod decl { use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{self, PyClassRef}; use crate::pyobject::{ - IdProtocol, PyCallable, PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue, - TypeProtocol, + IdProtocol, IntoPyRef, PyCallable, PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, + PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -197,11 +197,11 @@ mod decl { } #[pymethod(name = "__next__")] - fn next(&self) -> PyResult { + fn next(&self, vm: &VirtualMachine) -> PyResult { let mut cur = self.cur.write(); let result = cur.clone(); *cur += &self.step; - Ok(PyInt::new(result)) + Ok(result.into_pyref(vm)) } #[pymethod(name = "__iter__")] diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 1be96e919..b84cd90a9 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -5,7 +5,7 @@ use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn marshal_dumps(co: PyCodeRef) -> PyBytes { - PyBytes::new(co.code.to_bytes()) + PyBytes::from(co.code.to_bytes()) } fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 755f49b1f..457e1747b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -75,7 +75,7 @@ pub struct PyPathLike { impl PyPathLike { pub fn new_str(path: String) -> Self { - PyPathLike { + Self { path: PathBuf::from(path), mode: OutputMode::String, } diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index 6d6d8064c..ccf8b1e9e 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -4,7 +4,7 @@ use crate::exceptions::PyBaseExceptionRef; use crate::function::OptionalArg; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue}; +use crate::pyobject::{IntoPyRef, PyClassImpl, PyObjectRef, PyResult, PyValue}; use crate::types::create_type; use crate::vm::VirtualMachine; @@ -205,8 +205,8 @@ fn zlib_decompressobj( PyDecompress { decompress: PyMutex::new(decompress), eof: AtomicCell::new(false), - unused_data: PyMutex::new(PyBytes::new(vec![]).into_ref(vm)), - unconsumed_tail: PyMutex::new(PyBytes::new(vec![]).into_ref(vm)), + unused_data: PyMutex::new(PyBytes::from(vec![]).into_ref(vm)), + unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(vm)), } } #[pyclass(name = "Decompress")] @@ -249,13 +249,13 @@ impl PyDecompress { if stream_end && !leftover.is_empty() { let mut unused_data = self.unused_data.lock(); - let unused = unused_data + let unused: Vec<_> = unused_data .get_value() .iter() .chain(leftover) .copied() .collect(); - *unused_data = PyBytes::new(unused).into_ref(vm); + *unused_data = unused.into_pyref(vm); } } @@ -288,7 +288,7 @@ impl PyDecompress { }; let mut unconsumed_tail = self.unconsumed_tail.lock(); if !leftover.is_empty() || unconsumed_tail.len() > 0 { - *unconsumed_tail = PyBytes::new(leftover.to_owned()).into_ref(vm); + *unconsumed_tail = PyBytes::from(leftover.to_owned()).into_ref(vm); } ret @@ -318,7 +318,7 @@ impl PyDecompress { }; self.save_unused_input(&mut d, &data, stream_end, orig_in, vm); - *data = PyBytes::new(Vec::new()).into_ref(vm); + *data = PyBytes::from(Vec::new()).into_ref(vm); // TODO: drop the inner decompressor, somehow // if stream_end { From 6edc82c1927c1f60c63da99bd7c43749f6639e7a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 6 Aug 2020 20:19:13 +0900 Subject: [PATCH 209/209] new_bigint for &BigInt --- vm/src/stdlib/ast.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index e28f1e48f..c2a76bf5b 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -407,7 +407,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes }), Number { value } => { let py_n = match value { - ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), + ast::Number::Integer { value } => vm.ctx.new_bigint(value), ast::Number::Float { value } => vm.ctx.new_float(*value), ast::Number::Complex { real, imag } => { vm.ctx.new_complex(Complex64::new(*real, *imag))