From 430bc4bac2217c3fe5dc6c06e6eda28272cb93d6 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 09:35:53 -0500 Subject: [PATCH 001/658] Fix vm.import when there are no frames on stack --- vm/src/builtins.rs | 11 ++++++++--- vm/src/obj/objsuper.rs | 8 +++++--- vm/src/vm.rs | 30 +++++++++++++++++++----------- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 6d3d512a7..a81273e60 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -700,9 +700,14 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { ] ); let current_path = { - let mut source_pathbuf = PathBuf::from(&vm.current_frame().code.source_path); - source_pathbuf.pop(); - source_pathbuf + match vm.current_frame() { + Some(frame) => { + let mut source_pathbuf = PathBuf::from(&frame.code.source_path); + source_pathbuf.pop(); + source_pathbuf + } + None => PathBuf::new(), + } }; import_module(vm, current_path, &objstr::get_value(name)) diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 2b2be0241..43161ae6b 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -129,13 +129,15 @@ fn super_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let py_obj = if let Some(obj) = py_obj { obj.clone() } else { - let frame = vm.current_frame(); + let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { match vm.get_locals().get_item(first_arg) { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 029588d13..05a115732 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -91,19 +91,28 @@ impl VirtualMachine { result } - pub fn current_frame(&self) -> Ref { - Ref::map(self.frames.borrow(), |frames| { - let index = frames.len() - 1; - let current_frame = &frames[index]; - objframe::get_value(current_frame) - }) + pub fn current_frame(&self) -> Option> { + let frames = self.frames.borrow(); + if frames.is_empty() { + None + } else { + Some(Ref::map(self.frames.borrow(), |frames| { + objframe::get_value(frames.last().unwrap()) + })) + } } pub fn current_scope(&self) -> Ref { - let frame = self.current_frame(); + let frame = self + .current_frame() + .expect("called current_scope but no frames on the stack"); Ref::map(frame, |f| &f.scope) } + pub fn try_class(&self, module: &str, class: &str) -> PyResult { + self.get_attribute(self.import(module)?, class) + } + pub fn class(&self, module: &str, class: &str) -> PyObjectRef { let module = self .import(module) @@ -246,10 +255,9 @@ impl VirtualMachine { } pub fn import(&self, module: &str) -> PyResult { - let builtins_import = self.builtins.get_item("__import__"); - match builtins_import { - Some(func) => self.invoke(func, vec![self.ctx.new_str(module.to_string())]), - None => Err(self.new_exception( + match self.get_attribute(self.builtins.clone(), "__import__") { + Ok(func) => self.invoke(func, vec![self.ctx.new_str(module.to_string())]), + Err(_) => Err(self.new_exception( self.ctx.exceptions.import_error.clone(), "__import__ not found".to_string(), )), From a830d8e131bf5d4f5debbc054814bbbbe4777535 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 09:36:15 -0500 Subject: [PATCH 002/658] Add PyDocument and PyElement --- vm/src/function.rs | 1 + wasm/lib/src/browser_module.rs | 113 ++++++++++++++++++++++++++------- wasm/lib/src/convert.rs | 4 +- wasm/lib/src/vm_class.rs | 1 - 4 files changed, 94 insertions(+), 25 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index 7159427e5..3093b8c5f 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -304,6 +304,7 @@ pub enum OptionalArg { } impl OptionalArg { + #[inline] pub fn into_option(self) -> Option { match self { Present(value) => Some(value), diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 7fc2a2476..13f1404ab 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use futures::Future; use js_sys::Promise; use num_traits::cast::ToPrimitive; @@ -7,11 +5,13 @@ use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::{future_to_promise, JsFuture}; -use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::import::import_module; -use rustpython_vm::obj::{objint, objstr}; +use rustpython_vm::function::{OptionalArg, PyFuncArgs}; +use rustpython_vm::obj::{ + objint, + objstr::{self, PyStringRef}, +}; use rustpython_vm::pyobject::{ - AttributeProtocol, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, + PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use rustpython_vm::VirtualMachine; @@ -44,7 +44,7 @@ impl FetchResponseFormat { fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); - let promise_type = import_promise_type(vm)?; + let promise_type = vm.try_class("browser", "Promise")?; let response_format = args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; @@ -165,7 +165,7 @@ pub struct PyPromise { impl PyValue for PyPromise { fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.class(BROWSER_NAME, "Promise") + vm.class("browser", "Promise") } } @@ -182,15 +182,8 @@ pub fn get_promise_value(obj: &PyObjectRef) -> Promise { panic!("Inner error getting promise") } -pub fn import_promise_type(vm: &VirtualMachine) -> PyResult { - match import_module(vm, PathBuf::default(), BROWSER_NAME)?.get_attr("Promise") { - Some(promise) => Ok(promise), - None => Err(vm.new_not_implemented_error("No Promise".to_string())), - } -} - fn promise_then(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = import_promise_type(vm)?; + let promise_type = vm.try_class("browser", "Promise")?; arg_check!( vm, args, @@ -236,7 +229,7 @@ fn promise_then(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn promise_catch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = import_promise_type(vm)?; + let promise_type = vm.try_class("browser", "Promise")?; arg_check!( vm, args, @@ -270,6 +263,65 @@ fn promise_catch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(PyPromise::new_obj(promise_type, ret_promise)) } +#[derive(Debug)] +struct PyDocument { + doc: web_sys::Document, +} +type PyDocumentRef = PyRef; + +impl PyValue for PyDocument { + fn class(vm: &VirtualMachine) -> PyObjectRef { + vm.class("browser", "Document") + } +} + +fn document_query(zelf: PyDocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = zelf + .doc + .query_selector(&query.value) + .map_err(|err| convert::js_py_typeerror(vm, err))?; + let elem = match elem { + Some(elem) => PyElement { elem }.into_ref(vm).into_object(), + None => vm.get_none(), + }; + Ok(elem) +} + +#[derive(Debug)] +struct PyElement { + elem: web_sys::Element, +} +type PyElementRef = PyRef; + +impl PyValue for PyElement { + fn class(vm: &VirtualMachine) -> PyObjectRef { + vm.class("browser", "Element") + } +} + +fn elem_get_attr( + zelf: PyElementRef, + attr: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, +) -> PyObjectRef { + match zelf.elem.get_attribute(&attr.value) { + Some(s) => vm.new_str(s), + None => default.into_option().unwrap_or_else(|| vm.get_none()), + } +} + +fn elem_set_attr( + zelf: PyElementRef, + attr: PyStringRef, + value: PyStringRef, + vm: &VirtualMachine, +) -> PyResult<()> { + zelf.elem + .set_attribute(&attr.value, &value.value) + .map_err(|err| convert::js_to_py(vm, err)) +} + fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); @@ -315,19 +367,36 @@ fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(result) } -const BROWSER_NAME: &str = "browser"; - pub fn make_module(ctx: &PyContext) -> PyObjectRef { let promise = py_class!(ctx, "Promise", ctx.object(), { "then" => ctx.new_rustfunc(promise_then), - "catch" => ctx.new_rustfunc(promise_catch) + "catch" => ctx.new_rustfunc(promise_catch), }); - py_module!(ctx, BROWSER_NAME, { + let document_class = py_class!(ctx, "Document", ctx.object(), { + "query" => ctx.new_rustfunc(document_query), + }); + + let document = PyObject::new( + PyDocument { + doc: window().document().expect("Document missing from window"), + }, + document_class.clone(), + ); + + let element = py_class!(ctx, "Element", ctx.object(), { + "get_attr" => ctx.new_rustfunc(elem_get_attr), + "set_attr" => ctx.new_rustfunc(elem_set_attr), + }); + + py_module!(ctx, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), "request_animation_frame" => ctx.new_rustfunc(browser_request_animation_frame), "cancel_animation_frame" => ctx.new_rustfunc(browser_cancel_animation_frame), "Promise" => promise, + "Document" => document_class, + "document" => document, + "Element" => element, "alert" => ctx.new_rustfunc(browser_alert), "confirm" => ctx.new_rustfunc(browser_confirm), "prompt" => ctx.new_rustfunc(browser_prompt), @@ -337,5 +406,5 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { pub fn setup_browser_module(vm: &VirtualMachine) { vm.stdlib_inits .borrow_mut() - .insert(BROWSER_NAME.to_string(), Box::new(make_module)); + .insert("browser".to_string(), Box::new(make_module)); } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 122ff1763..b7f9048e9 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -114,7 +114,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { } } // the browser module might not be injected - if let Ok(promise_type) = browser_module::import_promise_type(vm) { + if let Ok(promise_type) = vm.try_class("browser", "Promise") { if objtype::isinstance(&py_obj, &promise_type) { return browser_module::get_promise_value(&py_obj).into(); } @@ -158,7 +158,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { if js_val.is_object() { if let Some(promise) = js_val.dyn_ref::() { // the browser module might not be injected - if let Ok(promise_type) = browser_module::import_promise_type(vm) { + if let Ok(promise_type) = vm.try_class("browser", "Promise") { return browser_module::PyPromise::new_obj(promise_type, promise.clone()); } } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 9d80854f7..77822147a 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -49,7 +49,6 @@ impl StoredVirtualMachine { thread_local! { static STORED_VMS: RefCell>> = RefCell::default(); - static ACTIVE_VMS: RefCell> = RefCell::default(); } #[wasm_bindgen(js_name = vmStore)] From 2626a800ab486c74ae3a28e19a0ac1757e19fe64 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 23 Mar 2019 11:04:40 +0200 Subject: [PATCH 003/658] Support frozenset --- tests/snippets/set.py | 85 ++++++++++++++++++ vm/src/obj/objset.rs | 204 ++++++++++++++++++++++++++++++------------ 2 files changed, 233 insertions(+), 56 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a7a20b0ef..9e49dff89 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -144,3 +144,88 @@ a ^= set([3,4,5]) assert a == set([1,2,4,5]) with assertRaises(TypeError): a ^= 1 + +# frozen set + +assert frozenset([1,2]) == frozenset([1,2]) +assert not frozenset([1,2,3]) == frozenset([1,2]) + +assert frozenset([1,2,3]) >= frozenset([1,2]) +assert frozenset([1,2]) >= frozenset([1,2]) +assert not frozenset([1,3]) >= frozenset([1,2]) + +assert frozenset([1,2,3]).issuperset(frozenset([1,2])) +assert frozenset([1,2]).issuperset(frozenset([1,2])) +assert not frozenset([1,3]).issuperset(frozenset([1,2])) + +assert frozenset([1,2,3]) > frozenset([1,2]) +assert not frozenset([1,2]) > frozenset([1,2]) +assert not frozenset([1,3]) > frozenset([1,2]) + +assert frozenset([1,2]) <= frozenset([1,2,3]) +assert frozenset([1,2]) <= frozenset([1,2]) +assert not frozenset([1,3]) <= frozenset([1,2]) + +assert frozenset([1,2]).issubset(frozenset([1,2,3])) +assert frozenset([1,2]).issubset(frozenset([1,2])) +assert not frozenset([1,3]).issubset(frozenset([1,2])) + +assert frozenset([1,2]) < frozenset([1,2,3]) +assert not frozenset([1,2]) < frozenset([1,2]) +assert not frozenset([1,3]) < frozenset([1,2]) + +a = frozenset([1, 2, 3]) +assert len(a) == 3 +b = a.copy() +assert b == a + +assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) + +assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) + +assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) +assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) + +assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) +assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) + +assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) +assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) + +assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) +assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) + +assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) +assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) + +assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) + +assert_raises(TypeError, lambda: frozenset([[]])) + +# set and frozen set +assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) +assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) + +assert frozenset([1,2,3]) | set([4,5]) == frozenset([1,2,3,4,5]) +assert set([1,2,3]) | frozenset([4,5]) == set([1,2,3,4,5]) + +assert frozenset([1,2,3]).intersection(set([5,6])) == frozenset([]) +assert set([1,2,3]).intersection(frozenset([5,6])) == set([]) + +assert frozenset([1,2,3]) & set([1,2,3,4,5]) == frozenset([1,2,3]) +assert set([1,2,3]) & frozenset([1,2,3,4,5]) == set([1,2,3]) + +assert frozenset([1,2,3]).difference(set([5,6])) == frozenset([1,2,3]) +assert set([1,2,3]).difference(frozenset([5,6])) == set([1,2,3]) + +assert frozenset([1,2,3]) - set([4,5]) == frozenset([1,2,3]) +assert set([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) + +assert frozenset([1,2,3]).symmetric_difference(set([1,2])) == frozenset([3]) +assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) + +assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) +assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 33a445531..8bb338d40 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -9,13 +9,14 @@ use std::hash::{Hash, Hasher}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyContext, PyIteratorValue, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; use super::objiter; +use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] @@ -24,6 +25,12 @@ pub struct PySet { } pub type PySetRef = PyRef; +#[derive(Default)] +pub struct PyFrozenSet { + elements: RefCell>, +} +pub type PyFrozenSetRef = PyRef; + impl fmt::Debug for PySet { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter @@ -31,14 +38,69 @@ impl fmt::Debug for PySet { } } +impl fmt::Debug for PyFrozenSet { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("frozenset") + } +} + impl PyValue for PySet { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.set_type() } } +impl PyValue for PyFrozenSet { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.frozenset_type() + } +} + pub fn get_elements(obj: &PyObjectRef) -> HashMap { - obj.payload::().unwrap().elements.borrow().clone() + if let Some(set) = obj.payload::() { + return set.elements.borrow().clone(); + } else if let Some(frozenset) = obj.payload::() { + return frozenset.elements.borrow().clone(); + } + panic!("Not frozenset or set"); +} + +macro_rules! validate_set_or_frozenset { + ( $vm:ident, $obj:expr ) => { + if !(objtype::issubclass(&$obj, &$vm.ctx.set_type()) + || objtype::issubclass(&$obj, &$vm.ctx.frozenset_type())) + { + return Err($vm.new_type_error(format!( + "{} is not a subtype of set or frozenset a", + $obj.typ() + ))); + } + }; +} + +fn create_set( + vm: &VirtualMachine, + elements: HashMap, + cls: PyClassRef, +) -> PyResult { + if objtype::issubclass(&cls, &vm.ctx.set_type()) { + Ok(PyObject::new( + PySet { + elements: RefCell::new(elements), + }, + PySet::class(vm).into_object(), + )) + } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { + Ok(PyObject::new( + PyFrozenSet { + elements: RefCell::new(elements), + }, + PyFrozenSet::class(vm).into_object(), + )) + } else { + Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) + } } fn perform_action_with_hash( @@ -152,11 +214,9 @@ fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } /* Create a new object of sub-type of set */ -fn set_new( - cls: PyClassRef, - iterable: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { +fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + validate_set_or_frozenset!(vm, cls); + let elements: HashMap = match iterable { OptionalArg::Missing => HashMap::new(), OptionalArg::Present(iterable) => { @@ -169,25 +229,22 @@ fn set_new( } }; - PySet { - elements: RefCell::new(elements), - } - .into_ref_with_type(vm, cls) + create_set(vm, elements, cls) } fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { trace!("set.len called with: {:?}", args); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); + arg_check!(vm, args, required = [(s, None)]); + validate_set_or_frozenset!(vm, s.type_pyref()); let elements = get_elements(s); Ok(vm.context().new_int(elements.len())) } -fn set_copy(obj: PySetRef, _vm: &VirtualMachine) -> PySet { +fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { trace!("set.copy called with: {:?}", obj); - let elements = obj.elements.borrow().clone(); - PySet { - elements: RefCell::new(elements), - } + validate_set_or_frozenset!(vm, obj.type_pyref()); + let elements = get_elements(&obj).clone(); + create_set(vm, elements, obj.type_pyref()) } fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -211,11 +268,8 @@ fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(set, Some(vm.ctx.set_type())), (needle, None)] - ); + arg_check!(vm, args, required = [(set, None), (needle, None)]); + validate_set_or_frozenset!(vm, &set.type_pyref()); for element in get_elements(set).iter() { match vm._eq(needle.clone(), element.1.clone()) { Ok(value) => { @@ -281,14 +335,10 @@ fn set_compare_inner( size_func: &Fn(usize, usize) -> bool, swap: bool, ) -> PyResult { - arg_check!( - vm, - args, - required = [ - (zelf, Some(vm.ctx.set_type())), - (other, Some(vm.ctx.set_type())) - ] - ); + arg_check!(vm, args, required = [(zelf, None), (other, None)]); + + validate_set_or_frozenset!(vm, zelf.type_pyref()); + validate_set_or_frozenset!(vm, other.type_pyref()); let get_zelf = |swap: bool| -> &PyObjectRef { if swap { @@ -323,47 +373,48 @@ fn set_compare_inner( Ok(vm.new_bool(true)) } -fn set_union(zelf: PySetRef, other: PySetRef, _vm: &VirtualMachine) -> PySet { - let mut elements = zelf.elements.borrow().clone(); - elements.extend(other.elements.borrow().clone()); +fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + validate_set_or_frozenset!(vm, zelf.type_pyref()); + validate_set_or_frozenset!(vm, other.type_pyref()); - PySet { - elements: RefCell::new(elements), - } + let mut elements = get_elements(&zelf).clone(); + elements.extend(get_elements(&other).clone()); + + create_set(vm, elements, zelf.type_pyref()) } -fn set_intersection(zelf: PySetRef, other: PySetRef, vm: &VirtualMachine) -> PyResult { +fn set_intersection(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { set_combine_inner(zelf, other, vm, SetCombineOperation::Intersection) } -fn set_difference(zelf: PySetRef, other: PySetRef, vm: &VirtualMachine) -> PyResult { +fn set_difference(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { set_combine_inner(zelf, other, vm, SetCombineOperation::Difference) } fn set_symmetric_difference( - zelf: PySetRef, - other: PySetRef, + zelf: PyObjectRef, + other: PyObjectRef, vm: &VirtualMachine, -) -> PyResult { +) -> PyResult { + validate_set_or_frozenset!(vm, zelf.type_pyref()); + validate_set_or_frozenset!(vm, other.type_pyref()); let mut elements = HashMap::new(); - for element in zelf.elements.borrow().iter() { - let value = vm.call_method(other.as_object(), "__contains__", vec![element.1.clone()])?; + for element in get_elements(&zelf).iter() { + let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; if !objbool::get_value(&value) { elements.insert(element.0.clone(), element.1.clone()); } } - for element in other.elements.borrow().iter() { - let value = vm.call_method(zelf.as_object(), "__contains__", vec![element.1.clone()])?; + for element in get_elements(&other).iter() { + let value = vm.call_method(&zelf, "__contains__", vec![element.1.clone()])?; if !objbool::get_value(&value) { elements.insert(element.0.clone(), element.1.clone()); } } - Ok(PySet { - elements: RefCell::new(elements), - }) + create_set(vm, elements, zelf.type_pyref()) } enum SetCombineOperation { @@ -372,15 +423,17 @@ enum SetCombineOperation { } fn set_combine_inner( - zelf: PySetRef, - other: PySetRef, + zelf: PyObjectRef, + other: PyObjectRef, vm: &VirtualMachine, op: SetCombineOperation, -) -> PyResult { +) -> PyResult { + validate_set_or_frozenset!(vm, zelf.type_pyref()); + validate_set_or_frozenset!(vm, other.type_pyref()); let mut elements = HashMap::new(); - for element in zelf.elements.borrow().iter() { - let value = vm.call_method(other.as_object(), "__contains__", vec![element.1.clone()])?; + for element in get_elements(&zelf).iter() { + let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; let should_add = match op { SetCombineOperation::Intersection => objbool::get_value(&value), SetCombineOperation::Difference => !objbool::get_value(&value), @@ -390,9 +443,7 @@ fn set_combine_inner( } } - Ok(PySet { - elements: RefCell::new(elements), - }) + create_set(vm, elements, zelf.type_pyref()) } fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -614,6 +665,46 @@ pub fn init(context: &PyContext) { frozenset(iterable) -> frozenset object\n\n\ Build an immutable unordered collection of unique elements."; + context.set_attr(frozenset_type, "__new__", context.new_rustfunc(set_new)); + context.set_attr(frozenset_type, "__eq__", context.new_rustfunc(set_eq)); + context.set_attr(frozenset_type, "__ge__", context.new_rustfunc(set_ge)); + context.set_attr(frozenset_type, "__gt__", context.new_rustfunc(set_gt)); + context.set_attr(frozenset_type, "__le__", context.new_rustfunc(set_le)); + context.set_attr(frozenset_type, "__lt__", context.new_rustfunc(set_lt)); + context.set_attr(frozenset_type, "issubset", context.new_rustfunc(set_le)); + context.set_attr(frozenset_type, "issuperset", context.new_rustfunc(set_ge)); + context.set_attr(frozenset_type, "union", context.new_rustfunc(set_union)); + context.set_attr(frozenset_type, "__or__", context.new_rustfunc(set_union)); + context.set_attr( + frozenset_type, + "intersection", + context.new_rustfunc(set_intersection), + ); + context.set_attr( + frozenset_type, + "__and__", + context.new_rustfunc(set_intersection), + ); + context.set_attr( + frozenset_type, + "difference", + context.new_rustfunc(set_difference), + ); + context.set_attr( + frozenset_type, + "__sub__", + context.new_rustfunc(set_difference), + ); + context.set_attr( + frozenset_type, + "symmetric_difference", + context.new_rustfunc(set_symmetric_difference), + ); + context.set_attr( + frozenset_type, + "__xor__", + context.new_rustfunc(set_symmetric_difference), + ); context.set_attr( frozenset_type, "__contains__", @@ -630,4 +721,5 @@ pub fn init(context: &PyContext) { "__repr__", context.new_rustfunc(frozenset_repr), ); + context.set_attr(frozenset_type, "copy", context.new_rustfunc(set_copy)); } From dc05d5f94b43985245e5af0a505d0d0ce4e2a350 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 23 Mar 2019 11:06:18 +0200 Subject: [PATCH 004/658] Remove Refcell around PyFrozenSet elements --- vm/src/obj/objset.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 8bb338d40..1d69d3bde 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -27,7 +27,7 @@ pub type PySetRef = PyRef; #[derive(Default)] pub struct PyFrozenSet { - elements: RefCell>, + elements: HashMap, } pub type PyFrozenSetRef = PyRef; @@ -61,7 +61,7 @@ pub fn get_elements(obj: &PyObjectRef) -> HashMap { if let Some(set) = obj.payload::() { return set.elements.borrow().clone(); } else if let Some(frozenset) = obj.payload::() { - return frozenset.elements.borrow().clone(); + return frozenset.elements.clone(); } panic!("Not frozenset or set"); } @@ -93,9 +93,7 @@ fn create_set( )) } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { Ok(PyObject::new( - PyFrozenSet { - elements: RefCell::new(elements), - }, + PyFrozenSet { elements: elements }, PyFrozenSet::class(vm).into_object(), )) } else { From 0aabe88c5dc1aceea51a579e3bb9c5551bd17949 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 14:04:14 -0500 Subject: [PATCH 005/658] Put `browser` class methods into `impl` blocks --- vm/src/vm.rs | 7 +- wasm/lib/src/browser_module.rs | 239 +++++++++++++++------------------ wasm/lib/src/convert.rs | 14 +- 3 files changed, 123 insertions(+), 137 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 76b06ddfc..9cf4b98e0 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -111,7 +111,10 @@ impl VirtualMachine { } pub fn try_class(&self, module: &str, class: &str) -> PyResult { - let class = self.get_attribute(self.import(module)?, class)?.downcast().expect("not a class"); + let class = self + .get_attribute(self.import(module)?, class)? + .downcast() + .expect("not a class"); Ok(class) } @@ -122,7 +125,7 @@ impl VirtualMachine { let class = self .get_attribute(module.clone(), class) .unwrap_or_else(|_| panic!("module {} has no class {}", module, class)); - class.downcast().expect("not a class"); + class.downcast().expect("not a class") } /// Create a new python string object. diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 0be52c43d..ce2d3d6c1 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,6 +7,7 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ + objfunction::PyFunction, objint, objstr::{self, PyStringRef}, objtype::PyClassRef, @@ -45,8 +46,6 @@ impl FetchResponseFormat { fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); - let promise_type = vm.try_class("browser", "Promise")?; - let response_format = args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; let method = args.get_optional_kwarg_with_type("method", vm.ctx.str_type(), vm)?; @@ -102,7 +101,9 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }) .and_then(JsFuture::from); - Ok(PyPromise::new_obj(promise_type, future_to_promise(future))) + Ok(PyPromise::new(future_to_promise(future)) + .into_ref(vm) + .into_object()) } fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -163,6 +164,7 @@ fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyRe pub struct PyPromise { value: Promise, } +pub type PyPromiseRef = PyRef; impl PyValue for PyPromise { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -171,156 +173,135 @@ impl PyValue for PyPromise { } impl PyPromise { - pub fn new_obj(promise_type: PyClassRef, value: Promise) -> PyObjectRef { - PyObject::new(PyPromise { value }, promise_type.into_object()) + pub fn new(value: Promise) -> PyPromise { + PyPromise { value } } -} - -pub fn get_promise_value(obj: &PyObjectRef) -> Promise { - if let Some(promise) = obj.payload::() { - return promise.value.clone(); + pub fn value(&self) -> Promise { + self.value.clone() } - panic!("Inner error getting promise") -} -fn promise_then(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = vm.try_class("browser", "Promise")?; - arg_check!( - vm, - args, - required = [ - (zelf, Some(promise_type.clone())), - (on_fulfill, Some(vm.ctx.function_type())) - ], - optional = [(on_reject, Some(vm.ctx.function_type()))] - ); + fn then( + zelf: PyPromiseRef, + on_fulfill: PyRef, + on_reject: OptionalArg>, + vm: &VirtualMachine, + ) -> PyResult { + let acc_vm = AccessibleVM::from(vm); - let on_fulfill = on_fulfill.clone(); - let on_reject = on_reject.cloned(); - - let acc_vm = AccessibleVM::from(vm); - - let promise = get_promise_value(zelf); - - let ret_future = JsFuture::from(promise).then(move |res| { - let stored_vm = &acc_vm - .upgrade() - .expect("that the vm is valid when the promise resolves"); - let vm = &stored_vm.vm; - let ret = match res { - Ok(val) => { - let val = convert::js_to_py(vm, val); - vm.invoke(on_fulfill, PyFuncArgs::new(vec![val], vec![])) - } - Err(err) => { - if let Some(on_reject) = on_reject { - let err = convert::js_to_py(vm, err); - vm.invoke(on_reject, PyFuncArgs::new(vec![err], vec![])) - } else { - return Err(err); - } - } - }; - convert::pyresult_to_jsresult(vm, ret) - }); - - let ret_promise = future_to_promise(ret_future); - - Ok(PyPromise::new_obj(promise_type, ret_promise)) -} - -fn promise_catch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = vm.try_class("browser", "Promise")?; - arg_check!( - vm, - args, - required = [ - (zelf, Some(promise_type.clone())), - (on_reject, Some(vm.ctx.function_type())) - ] - ); - - let on_reject = on_reject.clone(); - - let acc_vm = AccessibleVM::from(vm); - - let promise = get_promise_value(zelf); - - let ret_future = JsFuture::from(promise).then(move |res| match res { - Ok(val) => Ok(val), - Err(err) => { - let stored_vm = acc_vm + let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let stored_vm = &acc_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; - let err = convert::js_to_py(vm, err); - let res = vm.invoke(on_reject, PyFuncArgs::new(vec![err], vec![])); - convert::pyresult_to_jsresult(vm, res) - } - }); + let ret = match res { + Ok(val) => { + let val = convert::js_to_py(vm, val); + vm.invoke(on_fulfill.into_object(), PyFuncArgs::new(vec![val], vec![])) + } + Err(err) => { + if let OptionalArg::Present(on_reject) = on_reject { + let err = convert::js_to_py(vm, err); + vm.invoke(on_reject.into_object(), PyFuncArgs::new(vec![err], vec![])) + } else { + return Err(err); + } + } + }; + convert::pyresult_to_jsresult(vm, ret) + }); - let ret_promise = future_to_promise(ret_future); + let ret_promise = future_to_promise(ret_future); - Ok(PyPromise::new_obj(promise_type, ret_promise)) + Ok(PyPromise::new(ret_promise).into_ref(vm)) + } + + fn catch( + zelf: PyPromiseRef, + on_reject: PyRef, + vm: &VirtualMachine, + ) -> PyResult { + let acc_vm = AccessibleVM::from(vm); + + let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| match res { + Ok(val) => Ok(val), + Err(err) => { + let stored_vm = acc_vm + .upgrade() + .expect("that the vm is valid when the promise resolves"); + let vm = &stored_vm.vm; + let err = convert::js_to_py(vm, err); + let res = vm.invoke(on_reject.into_object(), PyFuncArgs::new(vec![err], vec![])); + convert::pyresult_to_jsresult(vm, res) + } + }); + + let ret_promise = future_to_promise(ret_future); + + Ok(PyPromise::new(ret_promise).into_ref(vm)) + } } #[derive(Debug)] -struct PyDocument { +struct Document { doc: web_sys::Document, } -type PyDocumentRef = PyRef; +type DocumentRef = PyRef; -impl PyValue for PyDocument { - fn class(vm: &VirtualMachine) -> PyObjectRef { +impl PyValue for Document { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.class("browser", "Document") } } -fn document_query(zelf: PyDocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { - let elem = zelf - .doc - .query_selector(&query.value) - .map_err(|err| convert::js_py_typeerror(vm, err))?; - let elem = match elem { - Some(elem) => PyElement { elem }.into_ref(vm).into_object(), - None => vm.get_none(), - }; - Ok(elem) +impl Document { + fn query(zelf: DocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = zelf + .doc + .query_selector(&query.value) + .map_err(|err| convert::js_py_typeerror(vm, err))?; + let elem = match elem { + Some(elem) => Element { elem }.into_ref(vm).into_object(), + None => vm.get_none(), + }; + Ok(elem) + } } #[derive(Debug)] -struct PyElement { +struct Element { elem: web_sys::Element, } -type PyElementRef = PyRef; +type ElementRef = PyRef; -impl PyValue for PyElement { - fn class(vm: &VirtualMachine) -> PyObjectRef { +impl PyValue for Element { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.class("browser", "Element") } } -fn elem_get_attr( - zelf: PyElementRef, - attr: PyStringRef, - default: OptionalArg, - vm: &VirtualMachine, -) -> PyObjectRef { - match zelf.elem.get_attribute(&attr.value) { - Some(s) => vm.new_str(s), - None => default.into_option().unwrap_or_else(|| vm.get_none()), +impl Element { + fn get_attr( + zelf: ElementRef, + attr: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyObjectRef { + match zelf.elem.get_attribute(&attr.value) { + Some(s) => vm.new_str(s), + None => default.into_option().unwrap_or_else(|| vm.get_none()), + } } -} -fn elem_set_attr( - zelf: PyElementRef, - attr: PyStringRef, - value: PyStringRef, - vm: &VirtualMachine, -) -> PyResult<()> { - zelf.elem - .set_attribute(&attr.value, &value.value) - .map_err(|err| convert::js_to_py(vm, err)) + fn set_attr( + zelf: ElementRef, + attr: PyStringRef, + value: PyStringRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.elem + .set_attribute(&attr.value, &value.value) + .map_err(|err| convert::js_to_py(vm, err)) + } } fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -370,24 +351,24 @@ fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn make_module(ctx: &PyContext) -> PyObjectRef { let promise = py_class!(ctx, "Promise", ctx.object(), { - "then" => ctx.new_rustfunc(promise_then), - "catch" => ctx.new_rustfunc(promise_catch), + "then" => ctx.new_rustfunc(PyPromise::then), + "catch" => ctx.new_rustfunc(PyPromise::catch), }); let document_class = py_class!(ctx, "Document", ctx.object(), { - "query" => ctx.new_rustfunc(document_query), + "query" => ctx.new_rustfunc(Document::query), }); let document = PyObject::new( - PyDocument { + Document { doc: window().document().expect("Document missing from window"), }, - document_class.clone(), + document_class.clone().into_object(), ); let element = py_class!(ctx, "Element", ctx.object(), { - "get_attr" => ctx.new_rustfunc(elem_get_attr), - "set_attr" => ctx.new_rustfunc(elem_set_attr), + "get_attr" => ctx.new_rustfunc(Element::get_attr), + "set_attr" => ctx.new_rustfunc(Element::set_attr), }); py_module!(ctx, "browser", { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 9c9e9f902..ecdb4b110 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -4,7 +4,7 @@ use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbytes, objint, objsequence, objtype}; -use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult}; +use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; @@ -115,9 +115,9 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { } } // the browser module might not be injected - if let Ok(promise_type) = vm.try_class("browser", "Promise") { - if objtype::isinstance(&py_obj, &promise_type) { - return browser_module::get_promise_value(&py_obj).into(); + if vm.try_class("browser", "Promise").is_ok() { + if let Some(py_prom) = py_obj.payload::() { + return py_prom.value().into(); } } @@ -159,8 +159,10 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { if js_val.is_object() { if let Some(promise) = js_val.dyn_ref::() { // the browser module might not be injected - if let Ok(promise_type) = vm.try_class("browser", "Promise") { - return browser_module::PyPromise::new_obj(promise_type, promise.clone()); + if vm.try_class("browser", "Promise").is_ok() { + return browser_module::PyPromise::new(promise.clone()) + .into_ref(vm) + .into_object(); } } if Array::is_array(&js_val) { From 590b659548a090d08bb51bca46a4457cc560e9a8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 23 Mar 2019 23:24:57 +0200 Subject: [PATCH 006/658] Change validate_set_or_frozenset to a function --- vm/src/obj/objset.rs | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 1d69d3bde..300dc9bca 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -66,17 +66,13 @@ pub fn get_elements(obj: &PyObjectRef) -> HashMap { panic!("Not frozenset or set"); } -macro_rules! validate_set_or_frozenset { - ( $vm:ident, $obj:expr ) => { - if !(objtype::issubclass(&$obj, &$vm.ctx.set_type()) - || objtype::issubclass(&$obj, &$vm.ctx.frozenset_type())) - { - return Err($vm.new_type_error(format!( - "{} is not a subtype of set or frozenset a", - $obj.typ() - ))); - } - }; +fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { + if !(objtype::issubclass(&cls, &vm.ctx.set_type()) + || objtype::issubclass(&cls, &vm.ctx.frozenset_type())) + { + return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))); + } + Ok(()) } fn create_set( @@ -213,7 +209,7 @@ fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { /* Create a new object of sub-type of set */ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset!(vm, cls); + validate_set_or_frozenset(vm, cls.clone())?; let elements: HashMap = match iterable { OptionalArg::Missing => HashMap::new(), @@ -227,20 +223,20 @@ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMach } }; - create_set(vm, elements, cls) + create_set(vm, elements, cls.clone()) } fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { trace!("set.len called with: {:?}", args); arg_check!(vm, args, required = [(s, None)]); - validate_set_or_frozenset!(vm, s.type_pyref()); + validate_set_or_frozenset(vm, s.type_pyref())?; let elements = get_elements(s); Ok(vm.context().new_int(elements.len())) } fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { trace!("set.copy called with: {:?}", obj); - validate_set_or_frozenset!(vm, obj.type_pyref()); + validate_set_or_frozenset(vm, obj.type_pyref())?; let elements = get_elements(&obj).clone(); create_set(vm, elements, obj.type_pyref()) } @@ -267,7 +263,7 @@ fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(set, None), (needle, None)]); - validate_set_or_frozenset!(vm, &set.type_pyref()); + validate_set_or_frozenset(vm, set.type_pyref())?; for element in get_elements(set).iter() { match vm._eq(needle.clone(), element.1.clone()) { Ok(value) => { @@ -335,8 +331,8 @@ fn set_compare_inner( ) -> PyResult { arg_check!(vm, args, required = [(zelf, None), (other, None)]); - validate_set_or_frozenset!(vm, zelf.type_pyref()); - validate_set_or_frozenset!(vm, other.type_pyref()); + validate_set_or_frozenset(vm, zelf.type_pyref())?; + validate_set_or_frozenset(vm, other.type_pyref())?; let get_zelf = |swap: bool| -> &PyObjectRef { if swap { @@ -372,8 +368,8 @@ fn set_compare_inner( } fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset!(vm, zelf.type_pyref()); - validate_set_or_frozenset!(vm, other.type_pyref()); + validate_set_or_frozenset(vm, zelf.type_pyref())?; + validate_set_or_frozenset(vm, other.type_pyref())?; let mut elements = get_elements(&zelf).clone(); elements.extend(get_elements(&other).clone()); @@ -394,8 +390,8 @@ fn set_symmetric_difference( other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - validate_set_or_frozenset!(vm, zelf.type_pyref()); - validate_set_or_frozenset!(vm, other.type_pyref()); + validate_set_or_frozenset(vm, zelf.type_pyref())?; + validate_set_or_frozenset(vm, other.type_pyref())?; let mut elements = HashMap::new(); for element in get_elements(&zelf).iter() { @@ -426,8 +422,8 @@ fn set_combine_inner( vm: &VirtualMachine, op: SetCombineOperation, ) -> PyResult { - validate_set_or_frozenset!(vm, zelf.type_pyref()); - validate_set_or_frozenset!(vm, other.type_pyref()); + validate_set_or_frozenset(vm, zelf.type_pyref())?; + validate_set_or_frozenset(vm, other.type_pyref())?; let mut elements = HashMap::new(); for element in get_elements(&zelf).iter() { From 0f2889a0e5b861c57e97cec3fafc451818e2f1f7 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 19:46:46 -0500 Subject: [PATCH 007/658] Fix weird rustfmt --- vm/src/obj/objsuper.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 1920a73fe..17d015a5f 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -123,10 +123,8 @@ fn super_new( match vm.get_locals().get_item(first_arg) { Some(obj) => obj.clone(), _ => { - return Err(vm.new_type_error(format!( - "super arguement {} was not supplied", - first_arg - ))); + return Err(vm + .new_type_error(format!("super argument {} was not supplied", first_arg))); } } } else { From ec433cdbb91aa9babca4369906237e2db59eb3ff Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 22:17:21 -0500 Subject: [PATCH 008/658] Fix rustfmt again (?) --- vm/src/obj/objsuper.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 53e347b9f..c41cf183e 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -128,8 +128,9 @@ fn super_new( match vm.get_locals().get_item(first_arg) { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super argument {} was not supplied", first_arg))); + return Err( + vm.new_type_error(format!("super arguement {} was not supplied", first_arg)) + ); } } } else { From 12564da369a9b3388b12d01043cda43bb56027c3 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 23 Mar 2019 22:46:21 -0500 Subject: [PATCH 009/658] Rustfmt? --- vm/src/obj/objsuper.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index c41cf183e..f04fb6605 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -128,9 +128,8 @@ fn super_new( match vm.get_locals().get_item(first_arg) { Some(obj) => obj.clone(), _ => { - return Err( - vm.new_type_error(format!("super arguement {} was not supplied", first_arg)) - ); + return Err(vm + .new_type_error(format!("super arguement {} was not supplied", first_arg))); } } } else { From 9a6f7aa8a1f58e346e1da4027a608c67d4dfc7ba Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Fri, 22 Mar 2019 19:12:33 +0100 Subject: [PATCH 010/658] Make PyRef::clone not require T implementing Clone It seems to be a weird consequence of using PhantomData, so I just rolled a custom Clone. --- vm/src/obj/objmodule.rs | 2 +- vm/src/obj/objnone.rs | 2 +- vm/src/obj/objobject.rs | 2 +- vm/src/obj/objtuple.rs | 1 - vm/src/obj/objtype.rs | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index 2d1d8921e..58d8688db 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -3,7 +3,7 @@ use crate::obj::objtype::PyClassRef; use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyModule { pub name: String, pub dict: PyObjectRef, diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index 7b0eefec2..840b00ecc 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -6,7 +6,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyNone; pub type PyNoneRef = PyRef; diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 84ba73fe6..46a90114d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -11,7 +11,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyInstance; impl PyValue for PyInstance { diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index e5bf5c1ee..a199e34b1 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -15,7 +15,6 @@ use super::objsequence::{ }; use super::objtype::{self, PyClassRef}; -#[derive(Default)] pub struct PyTuple { // TODO: shouldn't be public // TODO: tuples are immutable, remove this RefCell diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 9d3c9fe49..a8dd3289a 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -16,7 +16,7 @@ use super::objstr::PyStringRef; use super::objtuple::PyTuple; use crate::obj::objdict::PyDictRef; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyClass { pub name: String, pub mro: Vec, From a895ab35aeadcdc3b6504b531104765221212e18 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Fri, 22 Mar 2019 19:13:26 +0100 Subject: [PATCH 011/658] Make PyWeak::downgrade take a reference --- vm/src/obj/objweakref.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 3a71e3b63..685e32b70 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -11,9 +11,9 @@ pub struct PyWeak { } impl PyWeak { - pub fn downgrade(obj: PyObjectRef) -> PyWeak { + pub fn downgrade(obj: &PyObjectRef) -> PyWeak { PyWeak { - referent: Rc::downgrade(&obj), + referent: Rc::downgrade(obj), } } @@ -33,7 +33,7 @@ pub type PyWeakRef = PyRef; impl PyWeakRef { // TODO callbacks fn create(cls: PyClassRef, referent: PyObjectRef, vm: &VirtualMachine) -> PyResult { - PyWeak::downgrade(referent).into_ref_with_type(vm, cls) + PyWeak::downgrade(&referent).into_ref_with_type(vm, cls) } fn call(self, vm: &VirtualMachine) -> PyObjectRef { From e39df6daefb3ba1dbdb4e3a75770da57c2572d99 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Fri, 22 Mar 2019 19:18:33 +0100 Subject: [PATCH 012/658] Basic implementation of T.__subclasses__(). Ideally, the weak list should be updated whenever a type is removed, and the list shouldn't be a Vec, but this should be good enough for starters. --- tests/snippets/types_snippet.py | 23 +++++++++++++++++++++++ vm/src/obj/objtype.rs | 26 ++++++++++++++++++++++++-- vm/src/pyobject.rs | 13 +++++++++++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/tests/snippets/types_snippet.py b/tests/snippets/types_snippet.py index ac715751e..f441552d1 100644 --- a/tests/snippets/types_snippet.py +++ b/tests/snippets/types_snippet.py @@ -26,3 +26,26 @@ assert None.__class__ is type(None) assert isinstance(type, type) assert issubclass(type, type) + +class A: pass +class B(A): pass +class C(A): pass +class D(B, C): pass + +assert A.__subclasses__() == [B, C] +assert B.__subclasses__() == [D] +assert C.__subclasses__() == [D] +assert D.__subclasses__() == [] + +del D + +try: # gc sweep is needed here for CPython... + import gc; gc.collect() +except: # ...while RustPython doesn't have `gc` yet. + pass + +assert B.__subclasses__() == [] +assert C.__subclasses__() == [] + +assert type in object.__subclasses__() + diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index a8dd3289a..89bd8f34a 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -10,16 +10,18 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use super::objdict; +use super::objdict::PyDictRef; use super::objlist::PyList; use super::objproperty::PropertyBuilder; use super::objstr::PyStringRef; use super::objtuple::PyTuple; -use crate::obj::objdict::PyDictRef; +use super::objweakref::PyWeak; #[derive(Debug)] pub struct PyClass { pub name: String, pub mro: Vec, + pub subclasses: RefCell>, } impl fmt::Display for PyClass { @@ -146,6 +148,17 @@ impl PyClassRef { Err(vm.new_attribute_error(format!("{} has no attribute '{}'", self, name))) } } + + fn subclasses(self, _vm: &VirtualMachine) -> PyList { + let mut subclasses = self.subclasses.borrow_mut(); + subclasses.retain(|x| x.upgrade().is_some()); + PyList::from( + subclasses + .iter() + .map(|x| x.upgrade().unwrap()) + .collect::>(), + ) + } } /* @@ -168,6 +181,8 @@ pub fn init(ctx: &PyContext) { "__repr__" => ctx.new_rustfunc(PyClassRef::repr), "__prepare__" => ctx.new_rustfunc(PyClassRef::prepare), "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), + "__subclasses__" => ctx.new_rustfunc(PyClassRef::subclasses), + "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), "__instancecheck__" => ctx.new_rustfunc(PyClassRef::instance_check), "__subclasscheck__" => ctx.new_rustfunc(PyClassRef::subclass_check), "__doc__" => ctx.new_str(type_doc.to_string()), @@ -352,17 +367,24 @@ pub fn new( bases: Vec, dict: HashMap, ) -> PyResult { - let mros = bases.into_iter().map(|x| _mro(&x)).collect(); + let mros = bases.iter().map(|x| _mro(&x)).collect(); let mro = linearise_mro(mros).unwrap(); let new_type = PyObject { payload: PyClass { name: String::from(name), mro, + subclasses: RefCell::new(vec![]), }, dict: Some(RefCell::new(dict)), typ, } .into_ref(); + for base in bases { + base.subclasses + .borrow_mut() + .push(PyWeak::downgrade(&new_type)); + } + Ok(new_type.downcast().unwrap()) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index da16e1ac8..e4488bdb4 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -183,13 +183,14 @@ fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { // and both `type` and `object are instances of `type`. // to produce this circular dependency, we need an unsafe block. // (and yes, this will never get dropped. TODO?) - unsafe { + let (type_type, object_type) = unsafe { let object_type = PyObject { typ: mem::uninitialized(), // ! dict: Some(RefCell::new(PyAttributes::new())), payload: PyClass { name: String::from("object"), mro: vec![], + subclasses: RefCell::new(vec![]), }, } .into_ref(); @@ -200,6 +201,7 @@ fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { payload: PyClass { name: String::from("type"), mro: vec![object_type.clone().downcast().unwrap()], + subclasses: RefCell::new(vec![]), }, } .into_ref(); @@ -214,7 +216,14 @@ fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { ptr::write(&mut (*type_type_ptr).typ, type_type.clone()); (type_type, object_type) - } + }; + + object_type + .subclasses + .borrow_mut() + .push(objweakref::PyWeak::downgrade(&type_type.as_object())); + + (type_type, object_type) } // Basic objects: From d0c4fcb2fa65260c3eb72120cb303c8d1cbf7668 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 24 Mar 2019 13:14:38 +0100 Subject: [PATCH 013/658] Doing what is needed for all our non-latin friends. --- Cargo.lock | 1 + parser/Cargo.toml | 2 +- parser/src/lexer.rs | 611 +++++++++++++++++++---------------- src/main.rs | 2 +- tests/snippets/unicode_fu.py | 13 + 5 files changed, 341 insertions(+), 288 deletions(-) create mode 100644 tests/snippets/unicode_fu.py diff --git a/Cargo.lock b/Cargo.lock index 818d0af89..57dadb57b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,6 +760,7 @@ dependencies = [ "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e57d42140..7545a27d4 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -14,4 +14,4 @@ log="0.4.1" regex="0.2.2" num-bigint = "0.2" num-traits = "0.2" - +unicode-xid = "0.1.0" diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 113f7afb1..db8fee2c7 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1,12 +1,15 @@ //! This module takes care of lexing python source text. This means source //! code is translated into separate tokens. +extern crate unicode_xid; + pub use super::token::Tok; use num_bigint::BigInt; use num_traits::Num; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; +use unicode_xid::UnicodeXID; #[derive(Clone, Copy, PartialEq, Debug)] struct IndentationLevel { @@ -300,7 +303,7 @@ where } } - while self.is_char() { + while self.is_identifier_continuation() { name.push(self.next_char().unwrap()); } let end_pos = self.get_pos(); @@ -540,10 +543,21 @@ where Ok((start_pos, tok, end_pos)) } - fn is_char(&self) -> bool { - match self.chr0 { - Some('a'..='z') | Some('A'..='Z') | Some('_') | Some('0'..='9') => true, - _ => false, + fn is_identifier_start(&self, c: char) -> bool { + match c { + 'a'..='z' | 'A'..='Z' | '_' => true, + c => UnicodeXID::is_xid_start(c), + } + } + + fn is_identifier_continuation(&self) -> bool { + if let Some(c) = self.chr0 { + match c { + 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => true, + c => UnicodeXID::is_xid_continue(c), + } + } else { + false } } @@ -686,347 +700,372 @@ where } } - match self.chr0 { - Some('0'..='9') => return Some(self.lex_number()), - Some('_') | Some('a'..='z') | Some('A'..='Z') => { + // Check if we have some character: + if let Some(c) = self.chr0 { + // First check identifier: + if self.is_identifier_start(c) { return Some(self.lex_identifier()); - } - Some('#') => { - self.lex_comment(); - continue; - } - Some('"') => { - return Some(self.lex_string(false, false, false, false)); - } - Some('\'') => { - return Some(self.lex_string(false, false, false, false)); - } - Some('=') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::EqEqual, tok_end))); + } else { + match c { + '0'..='9' => return Some(self.lex_number()), + '#' => { + self.lex_comment(); + continue; } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Equal, tok_end))); + '"' => { + return Some(self.lex_string(false, false, false, false)); } - } - } - Some('+') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::PlusEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Plus, tok_end))); - } - } - Some('*') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::StarEqual, tok_end))); + '\'' => { + return Some(self.lex_string(false, false, false, false)); } - Some('*') => { + '=' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleStarEqual, tok_end))); + return Some(Ok((tok_start, Tok::EqEqual, tok_end))); } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleStar, tok_end))); + return Some(Ok((tok_start, Tok::Equal, tok_end))); } } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Star, tok_end))); - } - } - } - Some('/') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { + '+' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::SlashEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::PlusEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Plus, tok_end))); + } } - Some('/') => { + '*' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlashEqual, tok_end))); + return Some(Ok((tok_start, Tok::StarEqual, tok_end))); + } + Some('*') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok(( + tok_start, + Tok::DoubleStarEqual, + tok_end, + ))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::DoubleStar, tok_end))); + } + } } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Some(Ok((tok_start, Tok::Star, tok_end))); } } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Slash, tok_end))); - } - } - } - Some('%') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::PercentEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Percent, tok_end))); - } - } - Some('|') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::VbarEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Vbar, tok_end))); - } - } - Some('^') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::CircumflexEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::CircumFlex, tok_end))); - } - } - Some('&') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::AmperEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Amper, tok_end))); - } - } - Some('-') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::MinusEqual, tok_end))); - } - Some('>') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Rarrow, tok_end))); - } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Minus, tok_end))); - } - } - } - Some('@') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::AtEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::At, tok_end))); - } - } - Some('!') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::NotEqual, tok_end))); - } - _ => panic!("Invalid token '!'"), - } - } - Some('~') => { - return Some(self.eat_single_char(Tok::Tilde)); - } - Some('(') => { - let result = self.eat_single_char(Tok::Lpar); - self.nesting += 1; - return Some(result); - } - Some(')') => { - let result = self.eat_single_char(Tok::Rpar); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some('[') => { - let result = self.eat_single_char(Tok::Lsqb); - self.nesting += 1; - return Some(result); - } - Some(']') => { - let result = self.eat_single_char(Tok::Rsqb); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some('{') => { - let result = self.eat_single_char(Tok::Lbrace); - self.nesting += 1; - return Some(result); - } - Some('}') => { - let result = self.eat_single_char(Tok::Rbrace); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some(':') => { - return Some(self.eat_single_char(Tok::Colon)); - } - Some(';') => { - return Some(self.eat_single_char(Tok::Semi)); - } - Some('<') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('<') => { + '/' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LeftShiftEqual, tok_end))); + return Some(Ok((tok_start, Tok::SlashEqual, tok_end))); + } + Some('/') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok(( + tok_start, + Tok::DoubleSlashEqual, + tok_end, + ))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + tok_end, + ))); + } + } } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LeftShift, tok_end))); + return Some(Ok((tok_start, Tok::Slash, tok_end))); } } } - Some('=') => { + '%' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LessEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::PercentEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Percent, tok_end))); + } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Less, tok_end))); + '|' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::VbarEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Vbar, tok_end))); + } } - } - } - Some('>') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('>') => { + '^' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::CircumflexEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::CircumFlex, tok_end))); + } + } + '&' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::AmperEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Amper, tok_end))); + } + } + '-' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::RightShiftEqual, tok_end))); + return Some(Ok((tok_start, Tok::MinusEqual, tok_end))); + } + Some('>') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Rarrow, tok_end))); } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::RightShift, tok_end))); + return Some(Ok((tok_start, Tok::Minus, tok_end))); } } } - Some('=') => { + '@' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::AtEqual, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::At, tok_end))); + } + } + '!' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::NotEqual, tok_end))); + } else { + return Some(Err(LexicalError::UnrecognizedToken { tok: '!' })); + } + } + '~' => { + return Some(self.eat_single_char(Tok::Tilde)); + } + '(' => { + let result = self.eat_single_char(Tok::Lpar); + self.nesting += 1; + return Some(result); + } + ')' => { + let result = self.eat_single_char(Tok::Rpar); + if self.nesting == 0 { + return Some(Err(LexicalError::NestingError)); + } + self.nesting -= 1; + return Some(result); + } + '[' => { + let result = self.eat_single_char(Tok::Lsqb); + self.nesting += 1; + return Some(result); + } + ']' => { + let result = self.eat_single_char(Tok::Rsqb); + if self.nesting == 0 { + return Some(Err(LexicalError::NestingError)); + } + self.nesting -= 1; + return Some(result); + } + '{' => { + let result = self.eat_single_char(Tok::Lbrace); + self.nesting += 1; + return Some(result); + } + '}' => { + let result = self.eat_single_char(Tok::Rbrace); + if self.nesting == 0 { + return Some(Err(LexicalError::NestingError)); + } + self.nesting -= 1; + return Some(result); + } + ':' => { + return Some(self.eat_single_char(Tok::Colon)); + } + ';' => { + return Some(self.eat_single_char(Tok::Semi)); + } + '<' => { + let tok_start = self.get_pos(); + self.next_char(); + match self.chr0 { + Some('<') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok(( + tok_start, + Tok::LeftShiftEqual, + tok_end, + ))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::LeftShift, tok_end))); + } + } + } + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::LessEqual, tok_end))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Less, tok_end))); + } + } + } + '>' => { + let tok_start = self.get_pos(); + self.next_char(); + match self.chr0 { + Some('>') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok(( + tok_start, + Tok::RightShiftEqual, + tok_end, + ))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::RightShift, tok_end))); + } + } + } + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::GreaterEqual, tok_end))); + } + _ => { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Greater, tok_end))); + } + } + } + ',' => { + let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::GreaterEqual, tok_end))); + return Some(Ok((tok_start, Tok::Comma, tok_end))); } - _ => { + '.' => { + let tok_start = self.get_pos(); + self.next_char(); + if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { + self.next_char(); + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Dot, tok_end))); + } + } + '\n' => { + let tok_start = self.get_pos(); + self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Greater, tok_end))); - } - } - } - Some(',') => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Comma, tok_end))); - } - Some('.') => { - let tok_start = self.get_pos(); - self.next_char(); - if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { - self.next_char(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Dot, tok_end))); - } - } - Some('\n') => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.new_line(); + self.new_line(); - // Depending on the nesting level, we emit newline or not: - if self.nesting == 0 { - self.at_begin_of_line = true; - return Some(Ok((tok_start, Tok::Newline, tok_end))); - } else { - continue; + // Depending on the nesting level, we emit newline or not: + if self.nesting == 0 { + self.at_begin_of_line = true; + return Some(Ok((tok_start, Tok::Newline, tok_end))); + } else { + continue; + } + } + ' ' => { + // Skip whitespaces + self.next_char(); + continue; + } + _ => { + let c = self.next_char(); + return Some(Err(LexicalError::UnrecognizedToken { tok: c.unwrap() })); + } // Ignore all the rest.. } } - Some(' ') => { - // Skip whitespaces - self.next_char(); - continue; - } - None => return None, - _ => { - let c = self.next_char(); - return Some(Err(LexicalError::UnrecognizedToken { tok: c.unwrap() })); - } // Ignore all the rest.. + } else { + return None; } } } diff --git a/src/main.rs b/src/main.rs index 70adf1008..750faf98a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -153,7 +153,7 @@ fn get_prompt(vm: &VirtualMachine, prompt_name: &str) -> String { fn run_shell(vm: &VirtualMachine) -> PyResult { println!( - "Welcome to the magnificent Rust Python {} interpreter", + "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", crate_version!() ); let vars = vm.ctx.new_scope(); diff --git a/tests/snippets/unicode_fu.py b/tests/snippets/unicode_fu.py new file mode 100644 index 000000000..96d5bf977 --- /dev/null +++ b/tests/snippets/unicode_fu.py @@ -0,0 +1,13 @@ + +# Test the unicode support! 👋 + + +áš´=2 + +assert áš´*8 == 16 + +áš´="👋" + +c = áš´*3 + +assert c == '👋👋👋' From 565719dcdacaf83cfcaf7781b5b9437acb983270 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 24 Mar 2019 11:26:41 -0500 Subject: [PATCH 014/658] Update cargo dependencies --- Cargo.lock | 730 +++++++++++++++++++++------------------- wasm/lib/src/convert.rs | 13 +- 2 files changed, 398 insertions(+), 345 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 818d0af89..f3ea4ba4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [[package]] name = "aho-corasick" -version = "0.6.4" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -13,7 +13,7 @@ name = "ansi_term" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -27,15 +27,15 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "arrayvec" -version = "0.4.7" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -52,39 +52,45 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "atty" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "autocfg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "backtrace" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "backtrace-sys" -version = "0.1.24" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -110,7 +116,7 @@ name = "blake2-rfc" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -119,10 +125,15 @@ name = "block-buffer" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bumpalo" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byte-tools" version = "0.2.0" @@ -130,7 +141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.2.6" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -138,30 +149,30 @@ name = "caseless" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "cc" -version = "1.0.25" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cfg-if" -version = "0.1.3" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "clap" -version = "2.31.2" +version = "2.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -186,7 +197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "digest" -version = "0.7.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -194,12 +205,12 @@ dependencies = [ [[package]] name = "dirs" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -207,16 +218,16 @@ name = "docopt" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "either" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -226,46 +237,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "env_logger" -version = "0.5.10" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "env_logger" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "failure" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "failure_derive" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -279,17 +290,8 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" +name = "fuchsia-cprng" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -307,7 +309,7 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -315,7 +317,7 @@ dependencies = [ [[package]] name = "humantime" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -323,23 +325,23 @@ dependencies = [ [[package]] name = "itertools" -version = "0.7.8" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "itoa" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "js-sys" -version = "0.3.6" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -360,17 +362,17 @@ dependencies = [ "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -387,7 +389,7 @@ dependencies = [ "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -404,24 +406,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazy_static" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lexical" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "lexical-core" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -429,7 +431,7 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.42" +version = "0.2.50" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -437,32 +439,26 @@ name = "log" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "log" -version = "0.4.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "memchr" -version = "2.0.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "new_debug_unreachable" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "nix" @@ -470,28 +466,29 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "nodrop" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nom" -version = "4.1.1" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-bigint" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", @@ -519,7 +516,7 @@ name = "num-rational" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -545,19 +542,19 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.7.22" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "phf_shared" -version = "0.7.22" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -565,14 +562,6 @@ name = "precomputed-hash" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "proc-macro2" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "proc-macro2" version = "0.4.27" @@ -586,14 +575,6 @@ name = "quick-error" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "quote" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "quote" version = "0.6.11" @@ -604,42 +585,123 @@ dependencies = [ [[package]] name = "rand" -version = "0.4.2" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rand" -version = "0.5.5" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_jitter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_os" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rand_core" -version = "0.2.2" +name = "rand_pcg" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rand_core" -version = "0.3.0" +name = "rand_xorshift" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "redox_syscall" -version = "0.1.40" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -647,18 +709,18 @@ name = "redox_termios" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_users" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -666,23 +728,23 @@ name = "regex" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex" -version = "1.0.3" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -695,20 +757,20 @@ name = "regex-syntax" version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.6.0" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rustc-demangle" -version = "0.1.9" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -732,9 +794,9 @@ dependencies = [ name = "rustpython" version = "0.0.1" dependencies = [ - "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", "rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -756,8 +818,8 @@ version = "0.0.1" dependencies = [ "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -767,24 +829,24 @@ name = "rustpython_vm" version = "0.1.0" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_derive 0.1.0", "rustpython_parser 0.0.1", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -793,15 +855,15 @@ dependencies = [ name = "rustpython_wasm" version = "0.1.0-pre-alpha.1" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-futures 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -809,15 +871,15 @@ name = "rustyline" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -845,27 +907,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.66" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.66" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.26" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -875,13 +937,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "siphasher" -version = "0.2.2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -907,7 +974,7 @@ name = "statrs" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -915,24 +982,24 @@ name = "string_cache" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "string_cache_codegen" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -951,16 +1018,6 @@ name = "strsim" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "syn" -version = "0.14.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "syn" version = "0.15.29" @@ -973,12 +1030,12 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -991,14 +1048,6 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "termcolor" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "termcolor" version = "1.0.4" @@ -1012,14 +1061,14 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "textwrap" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1030,7 +1079,7 @@ name = "thread_local" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1040,13 +1089,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ucd-util" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unicode-normalization" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "unicode-segmentation" @@ -1073,7 +1125,7 @@ dependencies = [ [[package]] name = "utf8-ranges" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1086,6 +1138,11 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -1093,87 +1150,88 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wasm-bindgen" -version = "0.2.29" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.29" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bumpalo 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-futures" -version = "0.3.6" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.29" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro-support 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.29" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.29" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wasm-bindgen-webidl" -version = "0.2.23" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "web-sys" -version = "0.3.6" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-webidl 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1181,7 +1239,7 @@ name = "weedle" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1191,7 +1249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1210,10 +1268,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1221,21 +1279,13 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "wincolor" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "wincolor" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1244,93 +1294,100 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" -"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" -"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" +"checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" "checksum ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b385d69402821a1c254533a011a312531cbcc0e3e24f19bbb4747a5a2daf37e2" "checksum atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0fd4c0631f06448cc45a6bbb3b710ebb7ff8ccb96a0800c994afe23a70d5df2" -"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" -"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" -"checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +"checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +"checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" +"checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" "checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" "checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" +"checksum bumpalo 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "20bd7761c869bd69b5bd9dbbb1bd84de2f3970b764172efc41e67a06b508c56e" "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" -"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" +"checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" "checksum caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" -"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" -"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" -"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum cc 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "c9ce8bb087aacff865633f0bd5aeaed910fe2fe55b55f4739527f2e023a2e53d" +"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" +"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" -"checksum dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "88972de891f6118092b643d85a0b28e0678e0f948d7f879aa32f2d5aafe97d2a" +"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" +"checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" -"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" +"checksum either 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c67353c641dc847124ea1902d69bd753dee9bb3beff9aa3662ecf86c971d1fac" "checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5" -"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" -"checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e" -"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9" -"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426" +"checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +"checksum env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b61fa891024a945da30a9581546e8cfaf5602c7b3f4c137a2805cf388f92075a" +"checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" +"checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" -"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" -"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" -"checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606" -"checksum js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58cfec35fd4a94f3cf357d5cb7da71c71cd52720c2f2a7320090a8db5f06f655" +"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0d47946d458e94a1b7bcabbf6521ea7c037062c81f534615abcad76e84d4970d" +"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +"checksum js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "d4bda84b98977341bb6ba1086ecbd9eab8bc929de0f2923c118baf76c21dd0c8" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf" "checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f" "checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c" -"checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" -"checksum lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4fac65df7e751b57bb3a334c346239cb4ce2601907d698726ceeb82a54ba4ef" -"checksum lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "025babf624c0c2b4bed1373efd684d5d0b2eecd61138d26ec3eec77bf0f2e33d" -"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" +"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" +"checksum lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c106ed999697325a540c43d66a8d5175668cd96d7eb0bdba03a3bd98256cd698" +"checksum lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e82e023e062f1d25f807ad182008fba1b46538e999f908a08cc0c29e084462e" +"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -"checksum log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6fddaa003a65722a7fb9e26b0ce95921fe4ba590542ced664d8ce2fa26f9f3ac" -"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" -"checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" +"checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" +"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" +"checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30" "checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" -"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" -"checksum nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9c349f68f25f596b9f44cf0e7c69752a5c633b0550c3ff849518bfba0233774a" -"checksum num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "10b8423ea72ec64751198856a853e07b37087cfc9b53a87ecb19bff67b6d1320" +"checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" +"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "57450397855d951f1a41305e54851b1a7b8f5d2e349543a02a2effe25459f718" "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" "checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" "checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" -"checksum phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "05a079dd052e7b674d21cb31cbb6c05efd56a2cd2827db7692e2f1a507ebd998" -"checksum phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2261d544c2bb6aa3b10022b0be371b9c7c64f762ef28c6f5d4f1ef6d97b5930" +"checksum phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +"checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -"checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" -"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" -"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" -"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" -"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" -"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "214a97e49be64fd2c86f568dd0cb2c757d2cc53de95b273b6ad0a1c908482f26" +"checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -"checksum regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3d8c9f33201f46669484bacc312b00e7541bed6aaf296dffe2bb4e0ac6b8ce2a" +"checksum regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53ee8cfdddb2e0291adfb9f13d31d3bbe0a03c9a402c01b1e24188d86c35b24f" "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -"checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b" -"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" +"checksum regex-syntax 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8c2f35eedad5295fdf00a63d7d4b238135723f92b434ec06774dad15c7ab0861" +"checksum rustc-demangle 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "adacaae16d02b6ec37fdc7acfcddf365978de76d1983d3ee22afc260e1ca9619" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6de8ecd7fad7731f306f69b6e10ec5a3178c61e464dcc06979427aa4cc891145" "checksum rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6010155119d53aac4f5b987cb8f6ea913d0d64d9b237da36f8f96a90cb3f5385" @@ -1338,55 +1395,54 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" -"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" -"checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae" +"checksum serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)" = "92514fb95f900c9b5126e32d020f5c6d40564c27a5ea6d1d7d9f157a96623560" +"checksum serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6eabf4b5914e88e24eea240bb7c9f9a2cbc1bbbe8d961d381975ec3c6b806c" +"checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" -"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" +"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" +"checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" "checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8" "checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423" -"checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191" +"checksum string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eea1eee654ef80933142157fdad9dd8bc43cf7c74e999e369263496f04ff4da" "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" "checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" -"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" +"checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" -"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" -"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" +"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" +"checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "91f95b8f30407b9ca0c2de157281d3828bbed1fc1f55bea6eb54f40c52ec75ec" -"checksum wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "ab7c242ebcb45bae45340986c48d1853eb2c1c52ff551f7724951b62a2c51429" -"checksum wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d1784e7401a90119b2a4e8ec9c8d37c3594c3e3bb9ba24533ee1969eebaf0485" -"checksum wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "6e353f83716dec9a3597b5719ef88cb6c9e461ec16528f38aa023d3224b4e569" -"checksum wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "3cc90b65fe69c3dd5a09684517dc79f42b847baa2d479c234d125e0a629d9b0a" -"checksum wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "a71a37df4f5845025f96f279d20bbe5b19cbcb77f5410a3a90c6c544d889a162" -"checksum wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b064b8b2336b5a6bf5f31bc95fc1310842395df29877d910cb6f8f791070f319" -"checksum web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d7c588c2e5a091bc4532c5a87032955f9133b644e868b54d08ead0185dcc5b9" +"checksum wasm-bindgen 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "9742fc4860f47bede1090a5e4b0cfc33afcd70cfdf45dd28f2cfb02d4662b0dd" +"checksum wasm-bindgen-backend 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c6d7f35ecbb4180513cdb9b298543321bcb278670730415cbb3205ff2c66a477" +"checksum wasm-bindgen-futures 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "67bfbec85e48e8915b6acfa1c4d90b770ce033c96dc2067688f3bccd00d3b9e1" +"checksum wasm-bindgen-macro 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "e3c86b06bcd28e92e87d2c2ad208889b2f69ea33f79810b91ef660cc3de65a4c" +"checksum wasm-bindgen-macro-support 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "81d7338dd8c67e193d8ef18e5802dc03d8710456baa792c1c2e66847e57fd389" +"checksum wasm-bindgen-shared 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0d57c3b66f2f3e4d96b50f49b7b7e2f4cfcddc88b15744433c98c5c105b26672" +"checksum wasm-bindgen-webidl 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "fc7052a577eefce17a439edfce5c2eedc8432fe7457bbe949624cdf946233fa7" +"checksum web-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "72d02003924c77af871ad74216f828b6e32af776cbb3f5dab3dcde2f2713c012" "checksum weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26a4c67f132386d965390b8a734d5d10adbcd30eb5cc74bd9229af8b83f10044" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" +"checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" +"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index ecdb4b110..909955b6d 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -125,12 +125,10 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { || objtype::isinstance(&py_obj, &vm.ctx.bytearray_type()) { let bytes = objbytes::get_value(&py_obj); - let arr = Uint8Array::new_with_length(bytes.len() as u32); - for (i, byte) in bytes.iter().enumerate() { - Reflect::set(&arr, &(i as u32).into(), &(*byte).into()) - .expect("setting Uint8Array value failed"); + unsafe { + let view = Uint8Array::view(&bytes); + view.slice(0, bytes.len() as u32).into() } - arr.into() } else { match vm.serialize(&py_obj) { Ok(json) => js_sys::JSON::parse(&json).unwrap_or(JsValue::UNDEFINED), @@ -182,9 +180,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { .cloned() .unwrap_or_else(|| js_val.unchecked_ref::().buffer()), ); - let mut vec = Vec::with_capacity(u8_array.length() as usize); - // TODO: use Uint8Array::copy_to once updating js_sys doesn't break everything - u8_array.for_each(&mut |byte, _, _| vec.push(byte)); + let mut vec = vec![0; u8_array.length() as usize]; + u8_array.copy_to(&mut vec); vm.ctx.new_bytes(vec) } else { let dict = vm.new_dict(); From c2e1e9b0dc0a4180a1ccb8c8bb5a4c7e5dee27c2 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 24 Mar 2019 08:51:26 -0700 Subject: [PATCH 015/658] Convert range to new args style --- vm/src/obj/objint.rs | 7 + vm/src/obj/objrange.rs | 322 ++++++++++++++++------------------------- 2 files changed, 129 insertions(+), 200 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 7c19968da..345fe90c3 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,3 +1,4 @@ +use std::fmt; use std::hash::{Hash, Hasher}; use num_bigint::{BigInt, ToBigInt}; @@ -22,6 +23,12 @@ pub struct PyInt { pub value: BigInt, } +impl fmt::Display for PyInt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + BigInt::fmt(&self.value, f) + } +} + pub type PyIntRef = PyRef; impl PyInt { diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 468490eea..9ae2fcd54 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -3,17 +3,15 @@ use std::ops::Mul; use num_bigint::{BigInt, Sign}; use num_integer::Integer; -use num_traits::{One, Signed, ToPrimitive, Zero}; +use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - Either, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{Either, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objint::{self, PyInt, PyIntRef}; +use super::objint::{PyInt, PyIntRef}; use super::objslice::PySliceRef; -use super::objtype::{self, PyClassRef}; +use super::objtype::PyClassRef; #[derive(Debug, Clone)] pub struct PyRange { @@ -31,26 +29,6 @@ impl PyValue for PyRange { } impl PyRange { - #[inline] - pub fn try_len(&self) -> Option { - match self.step.sign() { - Sign::Plus if self.start < self.stop => ((&self.stop - &self.start - 1usize) - / &self.step) - .to_usize() - .map(|sz| sz + 1), - Sign::Minus if self.start > self.stop => ((&self.start - &self.stop - 1usize) - / (-&self.step)) - .to_usize() - .map(|sz| sz + 1), - _ => Some(0), - } - } - - #[inline] - pub fn len(&self) -> usize { - self.try_len().unwrap() - } - #[inline] fn offset(&self, value: &BigInt) -> Option { match self.step.sign() { @@ -60,14 +38,6 @@ impl PyRange { } } - #[inline] - pub fn contains(&self, value: &BigInt) -> bool { - match self.offset(value) { - Some(ref offset) => offset.is_multiple_of(&self.step), - None => false, - } - } - #[inline] pub fn index_of(&self, value: &BigInt) -> Option { match self.offset(value) { @@ -78,15 +48,6 @@ impl PyRange { } } - #[inline] - pub fn count(&self, value: &BigInt) -> usize { - if self.index_of(value).is_some() { - 1 - } else { - 0 - } - } - #[inline] pub fn is_empty(&self) -> bool { (self.start <= self.stop && self.step.is_negative()) @@ -113,40 +74,6 @@ impl PyRange { None } } - - #[inline] - pub fn reversed(&self) -> Self { - // compute the last element that is actually contained within the range - // this is the new start - let remainder = ((&self.stop - &self.start) % &self.step).abs(); - let start = if remainder.is_zero() { - &self.stop - &self.step - } else { - &self.stop - &remainder - }; - - match self.step.sign() { - Sign::Plus => PyRange { - start, - stop: &self.start - 1, - step: -&self.step, - }, - Sign::Minus => PyRange { - start, - stop: &self.start + 1, - step: -&self.step, - }, - Sign::NoSign => unreachable!(), - } - } - - pub fn repr(&self) -> String { - if self.step == BigInt::one() { - format!("range({}, {})", self.start, self.stop) - } else { - format!("range({}, {}, {})", self.start, self.stop, self.step) - } - } } pub fn get_value(obj: &PyObjectRef) -> PyRange { @@ -165,20 +92,20 @@ pub fn init(context: &PyContext) { When step is given, it specifies the increment (or decrement)."; extend_class!(context, range_type, { - "__bool__" => context.new_rustfunc(range_bool), - "__contains__" => context.new_rustfunc(range_contains), + "__bool__" => context.new_rustfunc(PyRangeRef::bool), + "__contains__" => context.new_rustfunc(PyRangeRef::contains), "__doc__" => context.new_str(range_doc.to_string()), "__getitem__" => context.new_rustfunc(PyRangeRef::getitem), - "__iter__" => context.new_rustfunc(range_iter), - "__len__" => context.new_rustfunc(range_len), + "__iter__" => context.new_rustfunc(PyRangeRef::iter), + "__len__" => context.new_rustfunc(PyRangeRef::len), "__new__" => context.new_rustfunc(range_new), - "__repr__" => context.new_rustfunc(range_repr), - "__reversed__" => context.new_rustfunc(range_reversed), - "count" => context.new_rustfunc(range_count), - "index" => context.new_rustfunc(range_index), - "start" => context.new_property(range_start), - "step" => context.new_property(range_step), - "stop" => context.new_property(range_stop) + "__repr__" => context.new_rustfunc(PyRangeRef::repr), + "__reversed__" => context.new_rustfunc(PyRangeRef::reversed), + "count" => context.new_rustfunc(PyRangeRef::count), + "index" => context.new_rustfunc(PyRangeRef::index), + "start" => context.new_property(PyRangeRef::start), + "stop" => context.new_property(PyRangeRef::stop), + "step" => context.new_property(PyRangeRef::step), }); } @@ -212,6 +139,113 @@ impl PyRangeRef { .into_ref_with_type(vm, cls) } + fn start(self, _vm: &VirtualMachine) -> BigInt { + self.start.clone() + } + + fn stop(self, _vm: &VirtualMachine) -> BigInt { + self.stop.clone() + } + + fn step(self, _vm: &VirtualMachine) -> BigInt { + self.step.clone() + } + + fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyIteratorValue { + PyIteratorValue { + position: Cell::new(0), + iterated_obj: self.into_object(), + } + } + + fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { + // compute the last element that is actually contained within the range + // this is the new start + let remainder = ((&self.stop - &self.start) % &self.step).abs(); + let start = if remainder.is_zero() { + &self.stop - &self.step + } else { + &self.stop - &remainder + }; + + let reversed = match self.step.sign() { + Sign::Plus => PyRange { + start, + stop: &self.start - 1, + step: -&self.step, + }, + Sign::Minus => PyRange { + start, + stop: &self.start + 1, + step: -&self.step, + }, + Sign::NoSign => unreachable!(), + }; + PyIteratorValue { + position: Cell::new(0), + iterated_obj: reversed.into_ref(vm).into_object(), + } + } + + fn len(self, _vm: &VirtualMachine) -> PyInt { + match self.step.sign() { + Sign::Plus if self.start < self.stop => { + PyInt::new((&self.stop - &self.start - 1usize) / &self.step + 1) + } + Sign::Minus if self.start > self.stop => { + PyInt::new((&self.start - &self.stop - 1usize) / (-&self.step) + 1) + } + Sign::Plus | Sign::Minus => PyInt::new(0), + Sign::NoSign => unreachable!(), + } + } + + fn repr(self, _vm: &VirtualMachine) -> String { + if self.step.is_one() { + format!("range({}, {})", self.start, self.stop) + } else { + format!("range({}, {}, {})", self.start, self.stop, self.step) + } + } + + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.is_empty() + } + + fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { + if let Ok(int) = needle.downcast::() { + match self.offset(&int.value) { + Some(ref offset) => offset.is_multiple_of(&self.step), + None => false, + } + } else { + false + } + } + + fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(int) = needle.downcast::() { + match self.index_of(&int.value) { + Some(idx) => Ok(PyInt::new(idx)), + None => Err(vm.new_value_error(format!("{} is not in range", int))), + } + } else { + Err(vm.new_value_error("sequence.index(x): x not in sequence".to_string())) + } + } + + fn count(self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { + if let Ok(int) = item.downcast::() { + if self.index_of(&int.value).is_some() { + PyInt::new(1) + } else { + PyInt::new(0) + } + } else { + PyInt::new(0) + } + } + fn getitem(self, subscript: Either, vm: &VirtualMachine) -> PyResult { match subscript { Either::A(index) => { @@ -271,115 +305,3 @@ fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(range.into_object()) } - -fn range_iter(range: PyRangeRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { - position: Cell::new(0), - iterated_obj: range.into_object(), - } -} - -fn range_reversed(zelf: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { - let range = zelf.reversed(); - - PyIteratorValue { - position: Cell::new(0), - iterated_obj: range.into_ref(vm).into_object(), - } -} - -fn range_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - - if let Some(len) = get_value(zelf).try_len() { - Ok(vm.ctx.new_int(len)) - } else { - Err(vm.new_overflow_error("Python int too large to convert to Rust usize".to_string())) - } -} - -fn range_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - - let repr = get_value(zelf).repr(); - - Ok(vm.ctx.new_str(repr)) -} - -fn range_bool(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - - let len = get_value(zelf).len(); - - Ok(vm.ctx.new_bool(len > 0)) -} - -fn range_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (needle, None)] - ); - - let range = get_value(zelf); - - let result = if objtype::isinstance(needle, &vm.ctx.int_type()) { - range.contains(&objint::get_value(needle)) - } else { - false - }; - - Ok(vm.ctx.new_bool(result)) -} - -fn range_index(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (needle, None)] - ); - - let range = get_value(zelf); - - if objtype::isinstance(needle, &vm.ctx.int_type()) { - let needle = objint::get_value(needle); - - match range.index_of(&needle) { - Some(idx) => Ok(vm.ctx.new_int(idx)), - None => Err(vm.new_value_error(format!("{} is not in range", needle))), - } - } else { - Err(vm.new_value_error("sequence.index(x): x not in sequence".to_string())) - } -} - -fn range_count(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (item, None)] - ); - - let range = get_value(zelf); - - if objtype::isinstance(item, &vm.ctx.int_type()) { - Ok(vm.ctx.new_int(range.count(&objint::get_value(item)))) - } else { - Ok(vm.ctx.new_int(0)) - } -} - -fn range_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).start)) -} - -fn range_stop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).stop)) -} - -fn range_step(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).step)) -} From 9a0113deedde20c7791f6c07936dc3e0c4b21e40 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 24 Mar 2019 09:57:26 -0700 Subject: [PATCH 016/658] range: represent w/ int refs --- tests/snippets/builtin_range.py | 4 ++ vm/src/obj/objint.rs | 4 ++ vm/src/obj/objrange.rs | 122 +++++++++++++++++--------------- 3 files changed, 73 insertions(+), 57 deletions(-) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index 89014880c..2ea71f7a7 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -50,3 +50,7 @@ assert 'foo' not in range(10) assert list(reversed(range(5))) == [4, 3, 2, 1, 0] assert list(reversed(range(5, 0, -1))) == [1, 2, 3, 4, 5] assert list(reversed(range(1,10,5))) == [6, 1] + +# range retains the original int refs +i = 2**64 +assert range(i).stop is i diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 345fe90c3..fddc759bc 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -35,6 +35,10 @@ impl PyInt { pub fn new>(i: T) -> Self { PyInt { value: i.into() } } + + pub fn as_bigint(&self) -> &BigInt { + &self.value + } } impl IntoPyObject for BigInt { diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 9ae2fcd54..380754b5f 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -15,11 +15,9 @@ use super::objtype::PyClassRef; #[derive(Debug, Clone)] pub struct PyRange { - // Unfortunately Rust's built in range type doesn't support things like indexing - // or ranges where start > end so we need to roll our own. - pub start: BigInt, - pub stop: BigInt, - pub step: BigInt, + pub start: PyIntRef, + pub stop: PyIntRef, + pub step: PyIntRef, } impl PyValue for PyRange { @@ -31,32 +29,36 @@ impl PyValue for PyRange { impl PyRange { #[inline] fn offset(&self, value: &BigInt) -> Option { - match self.step.sign() { - Sign::Plus if *value >= self.start && *value < self.stop => Some(value - &self.start), - Sign::Minus if *value <= self.start && *value > self.stop => Some(&self.start - value), + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + match step.sign() { + Sign::Plus if value >= start && value < stop => Some(value - start), + Sign::Minus if value <= self.start.as_bigint() && value > stop => Some(start - value), _ => None, } } #[inline] pub fn index_of(&self, value: &BigInt) -> Option { + let step = self.step.as_bigint(); match self.offset(value) { - Some(ref offset) if offset.is_multiple_of(&self.step) => { - Some((offset / &self.step).abs()) - } + Some(ref offset) if offset.is_multiple_of(step) => Some((offset / step).abs()), Some(_) | None => None, } } #[inline] pub fn is_empty(&self) -> bool { - (self.start <= self.stop && self.step.is_negative()) - || (self.start >= self.stop && self.step.is_positive()) + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + (start <= stop && step.is_negative()) || (start >= stop && step.is_positive()) } #[inline] pub fn forward(&self) -> bool { - self.start < self.stop + self.start.as_bigint() < self.stop.as_bigint() } #[inline] @@ -64,10 +66,14 @@ impl PyRange { where &'a BigInt: Mul, { - let result = &self.start + &self.step * index; + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); - if (self.forward() && !self.is_empty() && result < self.stop) - || (!self.forward() && !self.is_empty() && result > self.stop) + let result = start + step * index; + + if (self.forward() && !self.is_empty() && &result < stop) + || (!self.forward() && !self.is_empty() && &result > stop) { Some(result) } else { @@ -114,9 +120,9 @@ type PyRangeRef = PyRef; impl PyRangeRef { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { - start: Zero::zero(), - stop: stop.value.clone(), - step: One::one(), + start: PyInt::new(BigInt::zero()).into_ref(vm), + stop: stop.clone(), + step: PyInt::new(BigInt::one()).into_ref(vm), } .into_ref_with_type(vm, cls) } @@ -129,25 +135,24 @@ impl PyRangeRef { vm: &VirtualMachine, ) -> PyResult { PyRange { - start: start.value.clone(), - stop: stop.value.clone(), + start: start, + stop: stop, step: step .into_option() - .map(|i| i.value.clone()) - .unwrap_or_else(One::one), + .unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)), } .into_ref_with_type(vm, cls) } - fn start(self, _vm: &VirtualMachine) -> BigInt { + fn start(self, _vm: &VirtualMachine) -> PyIntRef { self.start.clone() } - fn stop(self, _vm: &VirtualMachine) -> BigInt { + fn stop(self, _vm: &VirtualMachine) -> PyIntRef { self.stop.clone() } - fn step(self, _vm: &VirtualMachine) -> BigInt { + fn step(self, _vm: &VirtualMachine) -> PyIntRef { self.step.clone() } @@ -159,28 +164,31 @@ impl PyRangeRef { } fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + // compute the last element that is actually contained within the range // this is the new start - let remainder = ((&self.stop - &self.start) % &self.step).abs(); - let start = if remainder.is_zero() { - &self.stop - &self.step + let remainder = ((stop - start) % step).abs(); + let new_start = if remainder.is_zero() { + stop - step } else { - &self.stop - &remainder + stop - &remainder }; - let reversed = match self.step.sign() { - Sign::Plus => PyRange { - start, - stop: &self.start - 1, - step: -&self.step, - }, - Sign::Minus => PyRange { - start, - stop: &self.start + 1, - step: -&self.step, - }, + let new_stop: BigInt = match step.sign() { + Sign::Plus => start - 1, + Sign::Minus => start + 1, Sign::NoSign => unreachable!(), }; + + let reversed = PyRange { + start: PyInt::new(new_start).into_ref(vm), + stop: PyInt::new(new_stop).into_ref(vm), + step: PyInt::new(-step).into_ref(vm), + }; + PyIteratorValue { position: Cell::new(0), iterated_obj: reversed.into_ref(vm).into_object(), @@ -188,20 +196,20 @@ impl PyRangeRef { } fn len(self, _vm: &VirtualMachine) -> PyInt { - match self.step.sign() { - Sign::Plus if self.start < self.stop => { - PyInt::new((&self.stop - &self.start - 1usize) / &self.step + 1) - } - Sign::Minus if self.start > self.stop => { - PyInt::new((&self.start - &self.stop - 1usize) / (-&self.step) + 1) - } + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + + match step.sign() { + Sign::Plus if start < stop => PyInt::new((stop - start - 1usize) / step + 1), + Sign::Minus if start > stop => PyInt::new((start - stop - 1usize) / (-step) + 1), Sign::Plus | Sign::Minus => PyInt::new(0), Sign::NoSign => unreachable!(), } } fn repr(self, _vm: &VirtualMachine) -> String { - if self.step.is_one() { + if self.step.as_bigint().is_one() { format!("range({}, {})", self.start, self.stop) } else { format!("range({}, {}, {})", self.start, self.stop, self.step) @@ -215,7 +223,7 @@ impl PyRangeRef { fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { match self.offset(&int.value) { - Some(ref offset) => offset.is_multiple_of(&self.step), + Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), None => false, } } else { @@ -256,9 +264,9 @@ impl PyRangeRef { } } Either::B(slice) => { - let new_start = if let Some(int) = slice.start.clone() { + let new_start = if let Some(int) = slice.start.as_ref() { if let Some(i) = self.get(int) { - i + PyInt::new(i).into_ref(vm) } else { self.start.clone() } @@ -266,9 +274,9 @@ impl PyRangeRef { self.start.clone() }; - let new_end = if let Some(int) = slice.stop.clone() { + let new_end = if let Some(int) = slice.stop.as_ref() { if let Some(i) = self.get(int) { - i + PyInt::new(i).into_ref(vm) } else { self.stop.clone() } @@ -276,8 +284,8 @@ impl PyRangeRef { self.stop.clone() }; - let new_step = if let Some(int) = slice.step.clone() { - int * self.step.clone() + let new_step = if let Some(int) = slice.step.as_ref() { + PyInt::new(int * self.step.as_bigint()).into_ref(vm) } else { self.step.clone() }; From f2d562a4ccf4b3ae4d6bf6bf3f9e6d157d95ce29 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 24 Mar 2019 10:05:25 -0700 Subject: [PATCH 017/658] pyint: use as_bigint() everywhere --- vm/src/frame.rs | 2 +- vm/src/obj/objbool.rs | 4 ++-- vm/src/obj/objenumerate.rs | 2 +- vm/src/obj/objint.rs | 3 +-- vm/src/obj/objrange.rs | 8 ++++---- vm/src/obj/objsequence.rs | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2a0951ab3..b9e6cf965 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -413,7 +413,7 @@ impl Frame { if x.is(&vm.ctx.none()) { None } else if let Some(i) = x.payload::() { - Some(i.value.clone()) + Some(i.as_bigint().clone()) } else { panic!("Expect Int or None as BUILD_SLICE arguments") } diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index e21a3a329..d4ef4b0fb 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -25,7 +25,7 @@ pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { Ok(if let Ok(f) = vm.get_method(obj.clone(), "__bool__") { let bool_res = vm.invoke(f, PyFuncArgs::default())?; match bool_res.payload::() { - Some(i) => !i.value.is_zero(), + Some(i) => !i.as_bigint().is_zero(), None => return Err(vm.new_type_error(String::from("TypeError"))), } } else { @@ -59,7 +59,7 @@ pub fn not(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { // Retrieve inner int value: pub fn get_value(obj: &PyObjectRef) -> bool { - !obj.payload::().unwrap().value.is_zero() + !obj.payload::().unwrap().as_bigint().is_zero() } fn bool_repr(vm: &VirtualMachine, args: PyFuncArgs) -> Result { diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 0b0e5d448..68d1e49f6 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -32,7 +32,7 @@ fn enumerate_new( vm: &VirtualMachine, ) -> PyResult { let counter = match start { - OptionalArg::Present(start) => start.value.clone(), + OptionalArg::Present(start) => start.as_bigint().clone(), OptionalArg::Missing => BigInt::zero(), }; diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index fddc759bc..dd40f06a4 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -19,8 +19,7 @@ use crate::obj::objtype::PyClassRef; #[derive(Debug)] pub struct PyInt { - // TODO: shouldn't be public - pub value: BigInt, + value: BigInt, } impl fmt::Display for PyInt { diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 380754b5f..04c9c273b 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -222,7 +222,7 @@ impl PyRangeRef { fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { - match self.offset(&int.value) { + match self.offset(int.as_bigint()) { Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), None => false, } @@ -233,7 +233,7 @@ impl PyRangeRef { fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { - match self.index_of(&int.value) { + match self.index_of(int.as_bigint()) { Some(idx) => Ok(PyInt::new(idx)), None => Err(vm.new_value_error(format!("{} is not in range", int))), } @@ -244,7 +244,7 @@ impl PyRangeRef { fn count(self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { if let Ok(int) = item.downcast::() { - if self.index_of(&int.value).is_some() { + if self.index_of(int.as_bigint()).is_some() { PyInt::new(1) } else { PyInt::new(0) @@ -257,7 +257,7 @@ impl PyRangeRef { fn getitem(self, subscript: Either, vm: &VirtualMachine) -> PyResult { match subscript { Either::A(index) => { - if let Some(value) = self.get(index.value.clone()) { + if let Some(value) = self.get(index.as_bigint()) { Ok(PyInt::new(value).into_ref(vm).into_object()) } else { Err(vm.new_index_error("range object index out of range".to_string())) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 83d0f9f9f..b8d91e590 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -144,7 +144,7 @@ pub fn get_item( subscript: PyObjectRef, ) -> PyResult { if let Some(i) = subscript.payload::() { - return match i.value.to_i32() { + return match i.as_bigint().to_i32() { Some(value) => { if let Some(pos_index) = elements.to_vec().get_pos(value) { let obj = elements[pos_index].clone(); From ca11e7dea79ed413138ee85acdd06b1070801ae6 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 24 Mar 2019 20:16:38 +0100 Subject: [PATCH 018/658] Add support for emoji as name. --- Cargo.lock | 42 ++++++++++++++++++++++++++++++++++++++++++ parser/Cargo.toml | 1 + parser/src/lexer.rs | 24 +++++++++++++++++------- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57dadb57b..6193772c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,6 +760,7 @@ dependencies = [ "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1044,6 +1045,42 @@ name = "ucd-util" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-emoji-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicode-normalization" version = "0.1.7" @@ -1364,6 +1401,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +"checksum unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" +"checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" +"checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" +"checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 7545a27d4..f2f8e5a6b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -15,3 +15,4 @@ regex="0.2.2" num-bigint = "0.2" num-traits = "0.2" unicode-xid = "0.1.0" +unic-emoji-char = "0.9.0" diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index db8fee2c7..defa5e232 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1,6 +1,7 @@ //! This module takes care of lexing python source text. This means source //! code is translated into separate tokens. +extern crate unic_emoji_char; extern crate unicode_xid; pub use super::token::Tok; @@ -9,6 +10,7 @@ use num_traits::Num; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; +use unic_emoji_char::is_emoji_presentation; use unicode_xid::UnicodeXID; #[derive(Clone, Copy, PartialEq, Debug)] @@ -545,7 +547,7 @@ where fn is_identifier_start(&self, c: char) -> bool { match c { - 'a'..='z' | 'A'..='Z' | '_' => true, + '_' => true, c => UnicodeXID::is_xid_start(c), } } @@ -553,7 +555,7 @@ where fn is_identifier_continuation(&self) -> bool { if let Some(c) = self.chr0 { match c { - 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => true, + '_' | '0'..='9' => true, c => UnicodeXID::is_xid_continue(c), } } else { @@ -705,6 +707,18 @@ where // First check identifier: if self.is_identifier_start(c) { return Some(self.lex_identifier()); + } else if is_emoji_presentation(c) { + let tok_start = self.get_pos(); + self.next_char(); + let tok_end = self.get_pos(); + println!("Emoji: {}", c); + return Some(Ok(( + tok_start, + Tok::Name { + name: c.to_string(), + }, + tok_end, + ))); } else { match c { '0'..='9' => return Some(self.lex_number()), @@ -801,11 +815,7 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok(( - tok_start, - Tok::DoubleSlash, - tok_end, - ))); + return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); } } } From 3562b8f59c2eccb034ac5c0f3de7638fcd586140 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 25 Mar 2019 11:21:01 +0000 Subject: [PATCH 019/658] Store class attributes inside PyClass struct. --- vm/src/obj/objtype.rs | 65 ++++++++++++++++++++++--------------------- vm/src/pyobject.rs | 12 ++++++-- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 89bd8f34a..6399d68c9 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -22,6 +22,7 @@ pub struct PyClass { pub name: String, pub mro: Vec, pub subclasses: RefCell>, + pub attributes: RefCell, } impl fmt::Display for PyClass { @@ -149,6 +150,25 @@ impl PyClassRef { } } + fn set_attr( + self, + attr_name: PyStringRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(attr) = class_get_attr(&self.type_pyref(), &attr_name.value) { + if let Some(descriptor) = class_get_attr(&attr.type_pyref(), "__set__") { + vm.invoke(descriptor, vec![attr, self.into_object(), value])?; + return Ok(()); + } + } + + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value); + Ok(()) + } + fn subclasses(self, _vm: &VirtualMachine) -> PyList { let mut subclasses = self.subclasses.borrow_mut(); subclasses.retain(|x| x.upgrade().is_some()); @@ -181,6 +201,7 @@ pub fn init(ctx: &PyContext) { "__repr__" => ctx.new_rustfunc(PyClassRef::repr), "__prepare__" => ctx.new_rustfunc(PyClassRef::prepare), "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), + "__setattr__" => ctx.new_rustfunc(PyClassRef::set_attr), "__subclasses__" => ctx.new_rustfunc(PyClassRef::subclasses), "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), "__instancecheck__" => ctx.new_rustfunc(PyClassRef::instance_check), @@ -260,32 +281,13 @@ pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMach Ok(obj) } -// Very private helper function for class_get_attr -fn class_get_attr_in_dict(class: &PyClassRef, attr_name: &str) -> Option { - if let Some(ref dict) = class.as_object().dict { - dict.borrow().get(attr_name).cloned() - } else { - panic!("Only classes should be in MRO!"); - } -} - -// Very private helper function for class_has_attr -fn class_has_attr_in_dict(class: &PyClassRef, attr_name: &str) -> bool { - if let Some(ref dict) = class.as_object().dict { - dict.borrow().contains_key(attr_name) - } else { - panic!("All classes are expected to have dicts!"); - } -} - // This is the internal get_attr implementation for fast lookup on a class. -pub fn class_get_attr(zelf: &PyClassRef, attr_name: &str) -> Option { - let mro = &zelf.mro; - if let Some(item) = class_get_attr_in_dict(zelf, attr_name) { +pub fn class_get_attr(class: &PyClassRef, attr_name: &str) -> Option { + if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { return Some(item); } - for class in mro { - if let Some(item) = class_get_attr_in_dict(class, attr_name) { + for class in &class.mro { + if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { return Some(item); } } @@ -293,12 +295,12 @@ pub fn class_get_attr(zelf: &PyClassRef, attr_name: &str) -> Option } // This is the internal has_attr implementation for fast lookup on a class. -pub fn class_has_attr(zelf: &PyClassRef, attr_name: &str) -> bool { - class_has_attr_in_dict(zelf, attr_name) - || zelf +pub fn class_has_attr(class: &PyClassRef, attr_name: &str) -> bool { + class.attributes.borrow().contains_key(attr_name) + || class .mro .iter() - .any(|d| class_has_attr_in_dict(d, attr_name)) + .any(|c| c.attributes.borrow().contains_key(attr_name)) } pub fn get_attributes(cls: PyClassRef) -> PyAttributes { @@ -309,10 +311,8 @@ pub fn get_attributes(cls: PyClassRef) -> PyAttributes { base_classes.reverse(); for bc in base_classes { - if let Some(ref dict) = &bc.as_object().dict { - for (name, value) in dict.borrow().iter() { - attributes.insert(name.to_string(), value.clone()); - } + for (name, value) in bc.attributes.borrow().iter() { + attributes.insert(name.to_string(), value.clone()); } } @@ -374,8 +374,9 @@ pub fn new( name: String::from(name), mro, subclasses: RefCell::new(vec![]), + attributes: RefCell::new(dict), }, - dict: Some(RefCell::new(dict)), + dict: None, typ, } .into_ref(); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index e4488bdb4..81fec3406 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -186,22 +186,24 @@ fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { let (type_type, object_type) = unsafe { let object_type = PyObject { typ: mem::uninitialized(), // ! - dict: Some(RefCell::new(PyAttributes::new())), + dict: None, payload: PyClass { name: String::from("object"), mro: vec![], subclasses: RefCell::new(vec![]), + attributes: RefCell::new(PyAttributes::new()), }, } .into_ref(); let type_type = PyObject { typ: mem::uninitialized(), // ! - dict: Some(RefCell::new(PyAttributes::new())), + dict: None, payload: PyClass { name: String::from("type"), mro: vec![object_type.clone().downcast().unwrap()], subclasses: RefCell::new(vec![]), + attributes: RefCell::new(PyAttributes::new()), }, } .into_ref(); @@ -655,7 +657,11 @@ impl PyContext { value: V, ) { let obj = obj.into(); - if let Some(PyModule { ref dict, .. }) = obj.payload::() { + if let Some(PyClass { ref attributes, .. }) = obj.payload::() { + attributes + .borrow_mut() + .insert(attr_name.to_string(), value.into()); + } else if let Some(PyModule { ref dict, .. }) = obj.payload::() { dict.set_item(self, attr_name, value.into()) } else if let Some(ref dict) = obj.dict { dict.borrow_mut() From 17c3f9f024f79f972b20f019c4464b9b2a28593b Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 25 Mar 2019 16:34:07 +0000 Subject: [PATCH 020/658] builtin_locals can just return a reference to the locals dict. --- vm/src/builtins.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 9dfe7afd6..acd0719d8 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -11,7 +11,6 @@ use num_traits::{Signed, ToPrimitive}; use crate::compile; use crate::import::import_module; use crate::obj::objbool; -use crate::obj::objdict; use crate::obj::objint; use crate::obj::objiter; use crate::obj::objstr::{self, PyStringRef}; @@ -29,14 +28,7 @@ use crate::obj::objcode::PyCodeRef; use crate::stdlib::io::io_open; fn get_locals(vm: &VirtualMachine) -> PyObjectRef { - let d = vm.new_dict(); - // TODO: implement dict_iter_items? - let locals = vm.get_locals(); - let key_value_pairs = objdict::get_key_value_pairs(&locals); - for (key, value) in key_value_pairs { - objdict::set_item(&d, vm, &key, &value); - } - d + vm.get_locals() } fn dir_locals(vm: &VirtualMachine) -> PyObjectRef { From 602f2a83da41a4f9d857a8abd55fd581117ab016 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 25 Mar 2019 17:35:21 +0100 Subject: [PATCH 021/658] Update grcov --- tests/.travis-runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/.travis-runner.sh b/tests/.travis-runner.sh index 8a004d3b1..36b9f43ac 100755 --- a/tests/.travis-runner.sh +++ b/tests/.travis-runner.sh @@ -31,7 +31,7 @@ then zip -0 ccov.zip `find . \( -name "rustpython*.gc*" \) -print` # Install grcov - curl -L https://github.com/mozilla/grcov/releases/download/v0.4.1/grcov-linux-x86_64.tar.bz2 | tar jxf - + curl -L https://github.com/mozilla/grcov/releases/download/v0.4.2/grcov-linux-x86_64.tar.bz2 | tar jxf - ./grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" -p "x" > lcov.info From bbb716247266e6c0eca7993e2aacb456c3b79208 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 25 Mar 2019 16:37:20 +0000 Subject: [PATCH 022/658] Various dictionary changes. * vm.ctx.new_dict returns a PyDictRef * Special case for module goes away. * Instances get a real dictionary. --- vm/src/builtins.rs | 4 +- vm/src/frame.rs | 14 +++-- vm/src/function.rs | 9 +++ vm/src/import.rs | 2 +- vm/src/obj/objdict.rs | 110 +++++++++++++++++++-------------- vm/src/obj/objmodule.rs | 26 ++++---- vm/src/obj/objobject.rs | 38 +++++------- vm/src/obj/objproperty.rs | 4 +- vm/src/obj/objsequence.rs | 2 + vm/src/obj/objtype.rs | 4 +- vm/src/pyobject.rs | 94 +++++++++++++--------------- vm/src/stdlib/json.rs | 2 +- vm/src/vm.rs | 8 +-- wasm/lib/src/browser_module.rs | 1 + wasm/lib/src/convert.rs | 4 +- 15 files changed, 164 insertions(+), 158 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index acd0719d8..4d9fb7989 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -815,9 +815,9 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; - let cells = vm.new_dict(); + let cells = vm.ctx.new_dict(); - vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; + vm.invoke_with_locals(function, cells.clone().into_object(), namespace.clone())?; let class = vm.call_method( metaclass.as_object(), "__call__", diff --git a/vm/src/frame.rs b/vm/src/frame.rs index b9e6cf965..3584a57a8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -119,7 +119,7 @@ impl Scope { } pub fn child_scope(&self, ctx: &PyContext) -> Scope { - self.child_scope_with_locals(ctx.new_dict()) + self.child_scope_with_locals(ctx.new_dict().into_object()) } } @@ -142,7 +142,7 @@ impl NameProtocol for Scope { return Some(value); } - vm.builtins.get_item(name) + vm.get_attribute(vm.builtins.clone(), name).ok() } fn load_cell(&self, _vm: &VirtualMachine, name: &str) -> Option { @@ -386,7 +386,7 @@ impl Frame { Ok(None) } bytecode::Instruction::BuildMap { size, unpack } => { - let map_obj = vm.ctx.new_dict(); + let map_obj = vm.ctx.new_dict().into_object(); for _x in 0..*size { let obj = self.pop_value(); if *unpack { @@ -572,7 +572,7 @@ impl Frame { let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { - vm.new_dict() + vm.ctx.new_dict().into_object() }; let defaults = if flags.contains(bytecode::FunctionOpArg::HAS_DEFAULTS) { @@ -839,8 +839,10 @@ impl Frame { let module = vm.import(module)?; // Grab all the names from the module and put them in the context - for (k, v) in module.get_key_value_pairs().iter() { - self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + if let Some(dict) = &module.dict { + for (k, v) in dict.get_key_value_pairs().iter() { + self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + } } Ok(None) } diff --git a/vm/src/function.rs b/vm/src/function.rs index 1e2a38f71..ed959d85c 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -247,6 +247,15 @@ where } } +impl IntoIterator for KwArgs { + type Item = (String, T); + type IntoIter = std::collections::hash_map::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + /// A list of positional argument values. /// /// A built-in function with a `Args` parameter is analagous to a Python diff --git a/vm/src/import.rs b/vm/src/import.rs index 936b89680..987555915 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -40,7 +40,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let attrs = vm.ctx.new_dict(); attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); - vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; + vm.run_code_obj(code_obj, Scope::new(None, attrs.clone().into_object()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 0c8d5fe5e..061c3477e 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,9 +3,9 @@ use std::collections::HashMap; use std::fmt; use std::ops::{Deref, DerefMut}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + DictProtocol, PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -124,57 +124,48 @@ pub fn py_dict_to_attributes(dict: &PyObjectRef) -> PyAttributes { attrs } -pub fn attributes_to_py_dict(vm: &VirtualMachine, attributes: PyAttributes) -> PyResult { - let dict = vm.ctx.new_dict(); - for (key, value) in attributes { - let key = vm.ctx.new_str(key); - set_item(&dict, vm, &key, &value); - } - Ok(dict) -} - // Python dict methods: -fn dict_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_ty, Some(vm.ctx.type_type()))], - optional = [(dict_obj, None)] - ); - let dict = vm.ctx.new_dict(); - if let Some(dict_obj) = dict_obj { - if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { - for (needle, value) in get_key_value_pairs(&dict_obj) { - set_item(&dict, vm, &needle, &value); - } - } else { - let iter = objiter::get_iter(vm, dict_obj)?; - loop { - fn err(vm: &VirtualMachine) -> PyObjectRef { - vm.new_type_error("Iterator must have exactly two elements".to_string()) +impl PyDictRef { + fn new( + _class: PyClassRef, // TODO Support subclasses of int. + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let dict = vm.ctx.new_dict(); + if let OptionalArg::Present(dict_obj) = dict_obj { + if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { + for (needle, value) in get_key_value_pairs(&dict_obj) { + set_item(dict.as_object(), vm, &needle, &value); } - let element = match objiter::get_next_object(vm, &iter)? { - Some(obj) => obj, - None => break, - }; - let elem_iter = objiter::get_iter(vm, &element)?; - let needle = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; - let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; - if objiter::get_next_object(vm, &elem_iter)?.is_some() { - return Err(err(vm)); + } else { + let iter = objiter::get_iter(vm, &dict_obj)?; + loop { + fn err(vm: &VirtualMachine) -> PyObjectRef { + vm.new_type_error("Iterator must have exactly two elements".to_string()) + } + let element = match objiter::get_next_object(vm, &iter)? { + Some(obj) => obj, + None => break, + }; + let elem_iter = objiter::get_iter(vm, &element)?; + let needle = + objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + if objiter::get_next_object(vm, &elem_iter)?.is_some() { + return Err(err(vm)); + } + set_item(dict.as_object(), vm, &needle, &value); } - set_item(&dict, vm, &needle, &value); } } + for (needle, value) in kwargs.into_iter() { + let py_needle = vm.new_str(needle); + set_item(&dict.as_object(), vm, &py_needle, &value); + } + Ok(dict) } - for (needle, value) in args.kwargs { - let py_needle = vm.new_str(needle); - set_item(&dict, vm, &py_needle, &value); - } - Ok(dict) -} -impl PyDictRef { fn bool(self, _vm: &VirtualMachine) -> bool { !self.entries.borrow().is_empty() } @@ -302,6 +293,31 @@ impl PyDictRef { } } +impl DictProtocol for PyDictRef { + fn contains_key(&self, k: &str) -> bool { + content_contains_key_str(&self.entries.borrow(), k) + } + + fn get_item(&self, k: &str) -> Option { + content_get_key_str(&self.entries.borrow(), k) + } + + fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + get_key_value_pairs(self.as_object()) + } + + // Item set/get: + fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { + let key = ctx.new_str(key.to_string()); + set_item_in_content(&mut self.entries.borrow_mut(), &key, &v); + } + + fn del_item(&self, key: &str) { + let mut elements = get_mut_elements(self.as_object()); + elements.remove(key).unwrap(); + } +} + pub fn init(context: &PyContext) { extend_class!(context, &context.dict_type, { "__bool__" => context.new_rustfunc(PyDictRef::bool), @@ -310,7 +326,7 @@ pub fn init(context: &PyContext) { "__delitem__" => context.new_rustfunc(PyDictRef::delitem), "__getitem__" => context.new_rustfunc(PyDictRef::getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), - "__new__" => context.new_rustfunc(dict_new), + "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), "__setitem__" => context.new_rustfunc(PyDictRef::setitem), "clear" => context.new_rustfunc(PyDictRef::clear), diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index 58d8688db..b641f0426 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -1,12 +1,10 @@ -use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{DictProtocol, PyContext, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] pub struct PyModule { pub name: String, - pub dict: PyObjectRef, } pub type PyModuleRef = PyRef; @@ -18,23 +16,21 @@ impl PyValue for PyModule { impl PyModuleRef { fn dir(self: PyModuleRef, vm: &VirtualMachine) -> PyResult { - let keys = self - .dict - .get_key_value_pairs() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); - Ok(vm.ctx.new_list(keys)) - } - - fn set_attr(self, attr: PyStringRef, value: PyObjectRef, vm: &VirtualMachine) { - self.dict.set_item(&vm.ctx, &attr.value, value) + if let Some(dict) = &self.into_object().dict { + let keys = dict + .get_key_value_pairs() + .iter() + .map(|(k, _v)| k.clone()) + .collect(); + Ok(vm.ctx.new_list(keys)) + } else { + panic!("Modules should definitely have a dict."); + } } } pub fn init(context: &PyContext) { extend_class!(&context, &context.module_type, { "__dir__" => context.new_rustfunc(PyModuleRef::dir), - "__setattr__" => context.new_rustfunc(PyModuleRef::set_attr) }); } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 46a90114d..60473fad0 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,5 +1,5 @@ +use super::objdict::{self, PyDictRef}; use super::objlist::PyList; -use super::objmodule::PyModule; use super::objstr::{self, PyStringRef}; use super::objtype; use crate::function::PyFuncArgs; @@ -23,11 +23,12 @@ impl PyValue for PyInstance { pub fn new_instance(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { // more or less __new__ operator let cls = PyClassRef::try_from_object(vm, args.shift())?; - Ok(if cls.is(&vm.ctx.object) { - PyObject::new_without_dict(PyInstance, cls) + let dict = if cls.is(&vm.ctx.object) { + None } else { - PyObject::new(PyInstance, cls) - }) + Some(vm.ctx.new_dict()) + }; + Ok(PyObject::new(PyInstance, cls, dict)) } fn object_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -114,7 +115,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.borrow_mut().insert(attr_name.value.clone(), value); + dict.set_item(&vm.ctx, &attr_name.value, value); Ok(()) } else { let type_name = objtype::get_type_name(obj.type_ref()); @@ -135,7 +136,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.borrow_mut().remove(&attr_name.value); + dict.del_item(&attr_name.value); Ok(()) } else { let type_name = objtype::get_type_name(obj.type_ref()); @@ -227,13 +228,9 @@ fn object_class_setter( Err(vm.new_type_error(format!("can't change class of type '{}'", type_repr))) } -fn object_dict(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if let Some(ref dict) = args.args[0].dict { - let new_dict = vm.new_dict(); - for (attr, value) in dict.borrow().iter() { - new_dict.set_item(&vm.ctx, &attr, value.clone()); - } - Ok(new_dict) +fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(ref dict) = object.dict { + Ok(dict.clone()) } else { Err(vm.new_type_error("TypeError: no dictionary.".to_string())) } @@ -273,15 +270,8 @@ fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn object_getattr(obj: &PyObjectRef, attr_name: &str) -> Option { - // TODO: - // This is an all kinds of wrong work-around for the temporary difference in - // shape between modules and object. It will disappear once that is fixed. - if let Some(PyModule { ref dict, .. }) = obj.payload::() { - return dict.get_item(attr_name); - } - if let Some(ref dict) = obj.dict { - dict.borrow().get(attr_name).cloned() + dict.get_item(attr_name) } else { None } @@ -293,8 +283,8 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (name, value) in dict.borrow().iter() { - attributes.insert(name.to_string(), value.clone()); + for (key, value) in objdict::get_key_value_pairs(dict.as_object()) { + attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index c5459a9fb..e6b11904e 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -200,7 +200,7 @@ impl<'a> PropertyBuilder<'a> { deleter: None, }; - PyObject::new(payload, self.ctx.property_type()) + PyObject::new(payload, self.ctx.property_type(), None) } else { let payload = PyReadOnlyProperty { getter: self.getter.expect( @@ -208,7 +208,7 @@ impl<'a> PropertyBuilder<'a> { ), }; - PyObject::new(payload, self.ctx.readonly_property_type()) + PyObject::new(payload, self.ctx.readonly_property_type(), None) } } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index b8d91e590..80f4bf40f 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -164,11 +164,13 @@ pub fn get_item( Ok(PyObject::new( PyList::from(elements.to_vec().get_slice_items(vm, &subscript)?), sequence.type_pyref(), + None, )) } else if sequence.payload::().is_some() { Ok(PyObject::new( PyTuple::from(elements.to_vec().get_slice_items(vm, &subscript)?), sequence.type_pyref(), + None, )) } else { panic!("sequence get_item called for non-sequence") diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 6399d68c9..d87ca3843 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -110,8 +110,8 @@ impl PyClassRef { format!("", self.name) } - fn prepare(_name: PyStringRef, _bases: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.new_dict() + fn prepare(_name: PyStringRef, _bases: PyObjectRef, vm: &VirtualMachine) -> PyDictRef { + vm.ctx.new_dict() } fn getattribute(self, name_ref: PyStringRef, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 81fec3406..78e82c51b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -24,7 +24,7 @@ use crate::obj::objclassmethod; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::obj::objcomplex::{self, PyComplex}; -use crate::obj::objdict::{self, PyDict}; +use crate::obj::objdict::{self, PyDict, PyDictRef}; use crate::obj::objellipsis; use crate::obj::objenumerate; use crate::obj::objfilter; @@ -271,7 +271,7 @@ impl PyContext { fn create_object(payload: T, cls: &PyClassRef) -> PyRef { PyRef { - obj: PyObject::new(payload, cls.clone()), + obj: PyObject::new(payload, cls.clone(), None), _payload: PhantomData, } } @@ -521,32 +521,32 @@ impl PyContext { self.object.clone() } - pub fn new_object(&self) -> PyObjectRef { - self.new_instance(self.object.clone(), None) - } - pub fn new_int>(&self, i: T) -> PyObjectRef { - PyObject::new(PyInt::new(i), self.int_type()) + PyObject::new(PyInt::new(i), self.int_type(), None) } pub fn new_float(&self, value: f64) -> PyObjectRef { - PyObject::new(PyFloat::from(value), self.float_type()) + PyObject::new(PyFloat::from(value), self.float_type(), None) } pub fn new_complex(&self, value: Complex64) -> PyObjectRef { - PyObject::new(PyComplex::from(value), self.complex_type()) + PyObject::new(PyComplex::from(value), self.complex_type(), None) } pub fn new_str(&self, s: String) -> PyObjectRef { - PyObject::new(objstr::PyString { value: s }, self.str_type()) + PyObject::new(objstr::PyString { value: s }, self.str_type(), None) } pub fn new_bytes(&self, data: Vec) -> PyObjectRef { - PyObject::new(objbytes::PyBytes::new(data), self.bytes_type()) + PyObject::new(objbytes::PyBytes::new(data), self.bytes_type(), None) } pub fn new_bytearray(&self, data: Vec) -> PyObjectRef { - PyObject::new(objbytearray::PyByteArray::new(data), self.bytearray_type()) + PyObject::new( + objbytearray::PyByteArray::new(data), + self.bytearray_type(), + None, + ) } pub fn new_bool(&self, b: bool) -> PyObjectRef { @@ -558,21 +558,23 @@ impl PyContext { } pub fn new_tuple(&self, elements: Vec) -> PyObjectRef { - PyObject::new(PyTuple::from(elements), self.tuple_type()) + PyObject::new(PyTuple::from(elements), self.tuple_type(), None) } pub fn new_list(&self, elements: Vec) -> PyObjectRef { - PyObject::new(PyList::from(elements), self.list_type()) + PyObject::new(PyList::from(elements), self.list_type(), None) } pub fn new_set(&self) -> PyObjectRef { // Initialized empty, as calling __hash__ is required for adding each object to the set // which requires a VM context - this is done in the objset code itself. - PyObject::new(PySet::default(), self.set_type()) + PyObject::new(PySet::default(), self.set_type(), None) } - pub fn new_dict(&self) -> PyObjectRef { - PyObject::new(PyDict::default(), self.dict_type()) + pub fn new_dict(&self) -> PyDictRef { + PyObject::new(PyDict::default(), self.dict_type(), None) + .downcast() + .unwrap() } pub fn new_class(&self, name: &str, base: PyClassRef) -> PyClassRef { @@ -580,16 +582,16 @@ impl PyContext { } pub fn new_scope(&self) -> Scope { - Scope::new(None, self.new_dict()) + Scope::new(None, self.new_dict().into_object()) } - pub fn new_module(&self, name: &str, dict: PyObjectRef) -> PyObjectRef { + pub fn new_module(&self, name: &str, dict: PyDictRef) -> PyObjectRef { PyObject::new( PyModule { name: name.to_string(), - dict, }, self.module_type.clone(), + Some(dict), ) } @@ -600,6 +602,7 @@ impl PyContext { PyObject::new( PyBuiltinFunction::new(f.into_func()), self.builtin_function_or_method_type(), + None, ) } @@ -611,7 +614,7 @@ impl PyContext { } pub fn new_code_object(&self, code: bytecode::CodeObject) -> PyObjectRef { - PyObject::new(objcode::PyCode::new(code), self.code_type()) + PyObject::new(objcode::PyCode::new(code), self.code_type(), None) } pub fn new_function( @@ -623,18 +626,22 @@ impl PyContext { PyObject::new( PyFunction::new(code_obj, scope, defaults), self.function_type(), + Some(self.new_dict()), ) } pub fn new_bound_method(&self, function: PyObjectRef, object: PyObjectRef) -> PyObjectRef { - PyObject::new(PyMethod::new(object, function), self.bound_method_type()) + PyObject::new( + PyMethod::new(object, function), + self.bound_method_type(), + None, + ) } - pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { - let dict = dict.unwrap_or_default(); + pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ: class, - dict: Some(RefCell::new(dict)), + dict: dict, payload: objobject::PyInstance, } .into_ref() @@ -661,11 +668,8 @@ impl PyContext { attributes .borrow_mut() .insert(attr_name.to_string(), value.into()); - } else if let Some(PyModule { ref dict, .. }) = obj.payload::() { - dict.set_item(self, attr_name, value.into()) } else if let Some(ref dict) = obj.dict { - dict.borrow_mut() - .insert(attr_name.to_string(), value.into()); + dict.set_item(self, attr_name, value.into()); } else { unimplemented!("set_attr unimplemented for: {:?}", obj); }; @@ -707,7 +711,7 @@ where T: ?Sized + PyObjectPayload, { pub typ: PyClassRef, - pub dict: Option>, // __dict__ member + pub dict: Option, // __dict__ member pub payload: T, } @@ -917,8 +921,6 @@ impl DictProtocol for PyObjectRef { fn get_item(&self, k: &str) -> Option { if let Some(dict) = self.payload::() { objdict::content_get_key_str(&dict.entries.borrow(), k) - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.get_item(k) } else { panic!("TODO {:?}", k) } @@ -927,8 +929,6 @@ impl DictProtocol for PyObjectRef { fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { if self.payload_is::() { objdict::get_key_value_pairs(self) - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.get_key_value_pairs() } else { panic!("TODO") } @@ -939,8 +939,6 @@ impl DictProtocol for PyObjectRef { if let Some(dict) = self.payload::() { let key = ctx.new_str(key.to_string()); objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.set_item(ctx, key, v); } else { panic!("TODO {:?}", self); } @@ -1121,7 +1119,7 @@ where T: PyValue + Sized, { fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new(self, T::class(vm))) + Ok(PyObject::new(self, T::class(vm), None)) } } @@ -1143,19 +1141,10 @@ impl PyObject where T: Sized + PyObjectPayload, { - pub fn new(payload: T, typ: PyClassRef) -> PyObjectRef { + pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ, - dict: Some(RefCell::new(PyAttributes::new())), - payload, - } - .into_ref() - } - - pub fn new_without_dict(payload: T, typ: PyClassRef) -> PyObjectRef { - PyObject { - typ, - dict: None, + dict: dict, payload, } .into_ref() @@ -1184,7 +1173,7 @@ pub trait PyValue: fmt::Debug + Sized + 'static { fn into_ref(self, vm: &VirtualMachine) -> PyRef { PyRef { - obj: PyObject::new(self, Self::class(vm)), + obj: PyObject::new(self, Self::class(vm), None), _payload: PhantomData, } } @@ -1192,8 +1181,13 @@ pub trait PyValue: fmt::Debug + Sized + 'static { fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyClassRef) -> PyResult> { let class = Self::class(vm); if objtype::issubclass(&cls, &class) { + let dict = if cls.is(&class) { + None + } else { + Some(vm.ctx.new_dict()) + }; Ok(PyRef { - obj: PyObject::new(self, cls), + obj: PyObject::new(self, cls, dict), _payload: PhantomData, }) } else { diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 2c5c95d34..eac3619c2 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -177,7 +177,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { }; dict.set_item(&self.vm.ctx, &key, value); } - Ok(dict) + Ok(dict.into_object()) } fn visit_unit(self) -> Result diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7713c561d..0ee9798ab 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -141,10 +141,6 @@ impl VirtualMachine { self.ctx.new_bool(b) } - pub fn new_dict(&self) -> PyObjectRef { - self.ctx.new_dict() - } - pub fn new_empty_exception(&self, exc_type: PyClassRef) -> PyResult { info!("New exception created: no msg"); let args = PyFuncArgs::default(); @@ -454,11 +450,11 @@ impl VirtualMachine { // Do we support `**kwargs` ? let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { - let d = self.new_dict(); + let d = self.ctx.new_dict().into_object(); locals.set_item(&self.ctx, kwargs_name, d.clone()); Some(d) } - bytecode::Varargs::Unnamed => Some(self.new_dict()), + bytecode::Varargs::Unnamed => Some(self.ctx.new_dict().into_object()), bytecode::Varargs::None => None, }; diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index a309284cf..0e13aeb3a 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -363,6 +363,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { doc: window().document().expect("Document missing from window"), }, document_class.clone(), + None, ); let element = py_class!(ctx, "Element", ctx.object(), { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index ecdb4b110..588c7571c 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -187,13 +187,13 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { u8_array.for_each(&mut |byte, _, _| vec.push(byte)); vm.ctx.new_bytes(vec) } else { - let dict = vm.new_dict(); + let dict = vm.ctx.new_dict(); for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); dict.set_item(&vm.ctx, &String::from(js_sys::JsString::from(key)), py_val); } - dict + dict.into_object() } } else if js_val.is_function() { let func = js_sys::Function::from(js_val); From 55e0fb1f19f84d377f9aa5368d606e40f2a49371 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 25 Mar 2019 16:39:25 +0000 Subject: [PATCH 023/658] AST class for nodes instead of just using object everywhere. --- vm/src/stdlib/ast.rs | 459 ++++++++++++++++++++++--------------------- 1 file changed, 238 insertions(+), 221 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index a3010a519..0011407ab 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -11,9 +11,20 @@ use rustpython_parser::{ast, parser}; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; +#[derive(Debug)] +struct AstNode; +// type AstNodeRef = PyRef; + +impl PyValue for AstNode { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("ast", "AST") + } +} + /* * Idea: maybe we can create a sort of struct with some helper functions? struct AstToPyAst { @@ -30,35 +41,35 @@ impl AstToPyAst { } */ -fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { let mut body = vec![]; for statement in &program.statements { - body.push(statement_to_ast(ctx, statement)); + body.push(statement_to_ast(&vm, statement)); } // TODO: create Module node: // let ast_node = ctx.new_instance(this.Module); - let ast_node = ctx.new_object(); - let py_body = ctx.new_list(body); - ctx.set_attr(&ast_node, "body", py_body); + let ast_node = create_node(vm, "program"); + let py_body = vm.ctx.new_list(body); + vm.ctx.set_attr(&ast_node, "body", py_body); ast_node } // Create a node class instance -fn create_node(ctx: &PyContext, _name: &str) -> PyObjectRef { +fn create_node(vm: &VirtualMachine, _name: &str) -> PyObjectRef { // TODO: instantiate a class of type given by name // TODO: lookup in the current module? - ctx.new_object() + PyObject::new(AstNode, AstNode::class(vm), Some(vm.ctx.new_dict())) } -fn statements_to_ast(ctx: &PyContext, statements: &[ast::LocatedStatement]) -> PyObjectRef { +fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyObjectRef { let mut py_statements = vec![]; for statement in statements { - py_statements.push(statement_to_ast(ctx, statement)); + py_statements.push(statement_to_ast(vm, statement)); } - ctx.new_list(py_statements) + vm.ctx.new_list(py_statements) } -fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { +fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { let node = match &statement.node { ast::Statement::ClassDef { name, @@ -66,17 +77,18 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj decorator_list, .. } => { - let node = create_node(ctx, "ClassDef"); + let node = create_node(vm, "ClassDef"); // Set name: - ctx.set_attr(&node, "name", ctx.new_str(name.to_string())); + vm.ctx + .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); // Set body: - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = statements_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); - let py_decorator_list = expressions_to_ast(ctx, decorator_list); - ctx.set_attr(&node, "decorator_list", py_decorator_list); + let py_decorator_list = expressions_to_ast(vm, decorator_list); + vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); node } ast::Statement::FunctionDef { @@ -86,80 +98,83 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj decorator_list, returns, } => { - let node = create_node(ctx, "FunctionDef"); + let node = create_node(vm, "FunctionDef"); // Set name: - ctx.set_attr(&node, "name", ctx.new_str(name.to_string())); + vm.ctx + .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); - ctx.set_attr(&node, "args", parameters_to_ast(ctx, args)); + vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); // Set body: - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = statements_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); - let py_decorator_list = expressions_to_ast(ctx, decorator_list); - ctx.set_attr(&node, "decorator_list", py_decorator_list); + let py_decorator_list = expressions_to_ast(vm, decorator_list); + vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); let py_returns = if let Some(hint) = returns { - expression_to_ast(ctx, hint) + expression_to_ast(vm, hint) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "returns", py_returns); + vm.ctx.set_attr(&node, "returns", py_returns); node } - ast::Statement::Continue => create_node(ctx, "Continue"), - ast::Statement::Break => create_node(ctx, "Break"), - ast::Statement::Pass => create_node(ctx, "Pass"), + ast::Statement::Continue => create_node(vm, "Continue"), + ast::Statement::Break => create_node(vm, "Break"), + ast::Statement::Pass => create_node(vm, "Pass"), ast::Statement::Assert { test, msg } => { - let node = create_node(ctx, "Pass"); + let node = create_node(vm, "Pass"); - ctx.set_attr(&node, "test", expression_to_ast(ctx, test)); + vm.ctx.set_attr(&node, "test", expression_to_ast(vm, test)); let py_msg = match msg { - Some(msg) => expression_to_ast(ctx, msg), - None => ctx.none(), + Some(msg) => expression_to_ast(vm, msg), + None => vm.ctx.none(), }; - ctx.set_attr(&node, "msg", py_msg); + vm.ctx.set_attr(&node, "msg", py_msg); node } ast::Statement::Delete { targets } => { - let node = create_node(ctx, "Delete"); + let node = create_node(vm, "Delete"); - let py_targets = - ctx.new_tuple(targets.iter().map(|v| expression_to_ast(ctx, v)).collect()); - ctx.set_attr(&node, "targets", py_targets); + let py_targets = vm + .ctx + .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); + vm.ctx.set_attr(&node, "targets", py_targets); node } ast::Statement::Return { value } => { - let node = create_node(ctx, "Return"); + let node = create_node(vm, "Return"); let py_value = if let Some(value) = value { - ctx.new_tuple(value.iter().map(|v| expression_to_ast(ctx, v)).collect()) + vm.ctx + .new_tuple(value.iter().map(|v| expression_to_ast(vm, v)).collect()) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "value", py_value); + vm.ctx.set_attr(&node, "value", py_value); node } ast::Statement::If { test, body, orelse } => { - let node = create_node(ctx, "If"); + let node = create_node(vm, "If"); - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); + let py_test = expression_to_ast(vm, test); + vm.ctx.set_attr(&node, "test", py_test); - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = statements_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + statements_to_ast(vm, orelse) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "orelse", py_orelse); + vm.ctx.set_attr(&node, "orelse", py_orelse); node } @@ -169,49 +184,49 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj body, orelse, } => { - let node = create_node(ctx, "For"); + let node = create_node(vm, "For"); - let py_target = expression_to_ast(ctx, target); - ctx.set_attr(&node, "target", py_target); + let py_target = expression_to_ast(vm, target); + vm.ctx.set_attr(&node, "target", py_target); - let py_iter = expressions_to_ast(ctx, iter); - ctx.set_attr(&node, "iter", py_iter); + let py_iter = expressions_to_ast(vm, iter); + vm.ctx.set_attr(&node, "iter", py_iter); - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = statements_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + statements_to_ast(vm, orelse) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "orelse", py_orelse); + vm.ctx.set_attr(&node, "orelse", py_orelse); node } ast::Statement::While { test, body, orelse } => { - let node = create_node(ctx, "While"); + let node = create_node(vm, "While"); - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); + let py_test = expression_to_ast(vm, test); + vm.ctx.set_attr(&node, "test", py_test); - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = statements_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + statements_to_ast(vm, orelse) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "orelse", py_orelse); + vm.ctx.set_attr(&node, "orelse", py_orelse); node } ast::Statement::Expression { expression } => { - let node = create_node(ctx, "Expr"); + let node = create_node(vm, "Expr"); - let value = expression_to_ast(ctx, expression); - ctx.set_attr(&node, "value", value); + let value = expression_to_ast(vm, expression); + vm.ctx.set_attr(&node, "value", value); node } @@ -221,38 +236,38 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj }; // set lineno on node: - let lineno = ctx.new_int(statement.location.get_row()); - ctx.set_attr(&node, "lineno", lineno); + let lineno = vm.ctx.new_int(statement.location.get_row()); + vm.ctx.set_attr(&node, "lineno", lineno); node } -fn expressions_to_ast(ctx: &PyContext, expressions: &[ast::Expression]) -> PyObjectRef { +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyObjectRef { let mut py_expression_nodes = vec![]; for expression in expressions { - py_expression_nodes.push(expression_to_ast(ctx, expression)); + py_expression_nodes.push(expression_to_ast(vm, expression)); } - ctx.new_list(py_expression_nodes) + vm.ctx.new_list(py_expression_nodes) } -fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { let node = match &expression { ast::Expression::Call { function, args, .. } => { - let node = create_node(ctx, "Call"); + let node = create_node(vm, "Call"); - let py_func_ast = expression_to_ast(ctx, function); - ctx.set_attr(&node, "func", py_func_ast); + let py_func_ast = expression_to_ast(vm, function); + vm.ctx.set_attr(&node, "func", py_func_ast); - let py_args = expressions_to_ast(ctx, args); - ctx.set_attr(&node, "args", py_args); + let py_args = expressions_to_ast(vm, args); + vm.ctx.set_attr(&node, "args", py_args); node } ast::Expression::Binop { a, op, b } => { - let node = create_node(ctx, "BinOp"); + let node = create_node(vm, "BinOp"); - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "left", py_a); + let py_a = expression_to_ast(vm, a); + vm.ctx.set_attr(&node, "left", py_a); // Operator: let str_op = match op { @@ -270,15 +285,15 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR ast::Operator::BitAnd => "BitAnd", ast::Operator::FloorDiv => "FloorDiv", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); + let py_op = vm.ctx.new_str(str_op.to_string()); + vm.ctx.set_attr(&node, "op", py_op); - let py_b = expression_to_ast(ctx, b); - ctx.set_attr(&node, "right", py_b); + let py_b = expression_to_ast(vm, b); + vm.ctx.set_attr(&node, "right", py_b); node } ast::Expression::Unop { op, a } => { - let node = create_node(ctx, "UnaryOp"); + let node = create_node(vm, "UnaryOp"); let str_op = match op { ast::UnaryOperator::Not => "Not", @@ -286,37 +301,37 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR ast::UnaryOperator::Neg => "USub", ast::UnaryOperator::Pos => "UAdd", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); + let py_op = vm.ctx.new_str(str_op.to_string()); + vm.ctx.set_attr(&node, "op", py_op); - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "operand", py_a); + let py_a = expression_to_ast(vm, a); + vm.ctx.set_attr(&node, "operand", py_a); node } ast::Expression::BoolOp { a, op, b } => { - let node = create_node(ctx, "BoolOp"); + let node = create_node(vm, "BoolOp"); // Attach values: - let py_a = expression_to_ast(ctx, a); - let py_b = expression_to_ast(ctx, b); - let py_values = ctx.new_tuple(vec![py_a, py_b]); - ctx.set_attr(&node, "values", py_values); + let py_a = expression_to_ast(vm, a); + let py_b = expression_to_ast(vm, b); + let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); + vm.ctx.set_attr(&node, "values", py_values); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); + let py_op = vm.ctx.new_str(str_op.to_string()); + vm.ctx.set_attr(&node, "op", py_op); node } ast::Expression::Compare { a, op, b } => { - let node = create_node(ctx, "Compare"); + let node = create_node(vm, "Compare"); - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "left", py_a); + let py_a = expression_to_ast(vm, a); + vm.ctx.set_attr(&node, "left", py_a); // Operator: let str_op = match op { @@ -331,282 +346,283 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = ctx.new_list(vec![ctx.new_str(str_op.to_string())]); - ctx.set_attr(&node, "ops", py_ops); + let py_ops = vm.ctx.new_list(vec![vm.ctx.new_str(str_op.to_string())]); + vm.ctx.set_attr(&node, "ops", py_ops); - let py_b = ctx.new_list(vec![expression_to_ast(ctx, b)]); - ctx.set_attr(&node, "comparators", py_b); + let py_b = vm.ctx.new_list(vec![expression_to_ast(vm, b)]); + vm.ctx.set_attr(&node, "comparators", py_b); node } ast::Expression::Identifier { name } => { - let node = create_node(ctx, "Identifier"); + let node = create_node(vm, "Identifier"); // Id: - let py_name = ctx.new_str(name.clone()); - ctx.set_attr(&node, "id", py_name); + let py_name = vm.ctx.new_str(name.clone()); + vm.ctx.set_attr(&node, "id", py_name); node } ast::Expression::Lambda { args, body } => { - let node = create_node(ctx, "Lambda"); + let node = create_node(vm, "Lambda"); - ctx.set_attr(&node, "args", parameters_to_ast(ctx, args)); + vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); - let py_body = expression_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = expression_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); node } ast::Expression::IfExpression { test, body, orelse } => { - let node = create_node(ctx, "IfExp"); + let node = create_node(vm, "IfExp"); - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); + let py_test = expression_to_ast(vm, test); + vm.ctx.set_attr(&node, "test", py_test); - let py_body = expression_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); + let py_body = expression_to_ast(vm, body); + vm.ctx.set_attr(&node, "body", py_body); - let py_orelse = expression_to_ast(ctx, orelse); - ctx.set_attr(&node, "orelse", py_orelse); + let py_orelse = expression_to_ast(vm, orelse); + vm.ctx.set_attr(&node, "orelse", py_orelse); node } ast::Expression::Number { value } => { - let node = create_node(ctx, "Num"); + let node = create_node(vm, "Num"); let py_n = match value { - ast::Number::Integer { value } => ctx.new_int(value.clone()), - ast::Number::Float { value } => ctx.new_float(*value), + ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), + ast::Number::Float { value } => vm.ctx.new_float(*value), ast::Number::Complex { real, imag } => { - ctx.new_complex(Complex64::new(*real, *imag)) + vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - ctx.set_attr(&node, "n", py_n); + vm.ctx.set_attr(&node, "n", py_n); node } ast::Expression::True => { - let node = create_node(ctx, "NameConstant"); + let node = create_node(vm, "NameConstant"); - ctx.set_attr(&node, "value", ctx.new_bool(true)); + vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(true)); node } ast::Expression::False => { - let node = create_node(ctx, "NameConstant"); + let node = create_node(vm, "NameConstant"); - ctx.set_attr(&node, "value", ctx.new_bool(false)); + vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(false)); node } ast::Expression::None => { - let node = create_node(ctx, "NameConstant"); + let node = create_node(vm, "NameConstant"); - ctx.set_attr(&node, "value", ctx.none()); + vm.ctx.set_attr(&node, "value", vm.ctx.none()); node } - ast::Expression::Ellipsis => create_node(ctx, "Ellipsis"), + ast::Expression::Ellipsis => create_node(vm, "Ellipsis"), ast::Expression::List { elements } => { - let node = create_node(ctx, "List"); + let node = create_node(vm, "List"); - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); + let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); + let py_elts = vm.ctx.new_list(elts); + vm.ctx.set_attr(&node, "elts", py_elts); node } ast::Expression::Tuple { elements } => { - let node = create_node(ctx, "Tuple"); + let node = create_node(vm, "Tuple"); - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); + let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); + let py_elts = vm.ctx.new_list(elts); + vm.ctx.set_attr(&node, "elts", py_elts); node } ast::Expression::Set { elements } => { - let node = create_node(ctx, "Set"); + let node = create_node(vm, "Set"); - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); + let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); + let py_elts = vm.ctx.new_list(elts); + vm.ctx.set_attr(&node, "elts", py_elts); node } ast::Expression::Dict { elements } => { - let node = create_node(ctx, "Dict"); + let node = create_node(vm, "Dict"); let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(ctx, k)); - values.push(expression_to_ast(ctx, v)); + keys.push(expression_to_ast(vm, k)); + values.push(expression_to_ast(vm, v)); } - let py_keys = ctx.new_list(keys); - ctx.set_attr(&node, "keys", py_keys); + let py_keys = vm.ctx.new_list(keys); + vm.ctx.set_attr(&node, "keys", py_keys); - let py_values = ctx.new_list(values); - ctx.set_attr(&node, "values", py_values); + let py_values = vm.ctx.new_list(values); + vm.ctx.set_attr(&node, "values", py_values); node } ast::Expression::Comprehension { kind, generators } => { let node = match kind.deref() { ast::ComprehensionKind::GeneratorExpression { .. } => { - create_node(ctx, "GeneratorExp") + create_node(vm, "GeneratorExp") } - ast::ComprehensionKind::List { .. } => create_node(ctx, "ListComp"), - ast::ComprehensionKind::Set { .. } => create_node(ctx, "SetComp"), - ast::ComprehensionKind::Dict { .. } => create_node(ctx, "DictComp"), + ast::ComprehensionKind::List { .. } => create_node(vm, "ListComp"), + ast::ComprehensionKind::Set { .. } => create_node(vm, "SetComp"), + ast::ComprehensionKind::Dict { .. } => create_node(vm, "DictComp"), }; let g = generators .iter() - .map(|g| comprehension_to_ast(ctx, g)) + .map(|g| comprehension_to_ast(vm, g)) .collect(); - let py_generators = ctx.new_list(g); - ctx.set_attr(&node, "generators", py_generators); + let py_generators = vm.ctx.new_list(g); + vm.ctx.set_attr(&node, "generators", py_generators); node } ast::Expression::Yield { value } => { - let node = create_node(ctx, "Yield"); + let node = create_node(vm, "Yield"); let py_value = match value { - Some(value) => expression_to_ast(ctx, value), - None => ctx.none(), + Some(value) => expression_to_ast(vm, value), + None => vm.ctx.none(), }; - ctx.set_attr(&node, "value", py_value); + vm.ctx.set_attr(&node, "value", py_value); node } ast::Expression::YieldFrom { value } => { - let node = create_node(ctx, "YieldFrom"); + let node = create_node(vm, "YieldFrom"); - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); + let py_value = expression_to_ast(vm, value); + vm.ctx.set_attr(&node, "value", py_value); node } ast::Expression::Subscript { a, b } => { - let node = create_node(ctx, "Subscript"); + let node = create_node(vm, "Subscript"); - let py_value = expression_to_ast(ctx, a); - ctx.set_attr(&node, "value", py_value); + let py_value = expression_to_ast(vm, a); + vm.ctx.set_attr(&node, "value", py_value); - let py_slice = expression_to_ast(ctx, b); - ctx.set_attr(&node, "slice", py_slice); + let py_slice = expression_to_ast(vm, b); + vm.ctx.set_attr(&node, "slice", py_slice); node } ast::Expression::Attribute { value, name } => { - let node = create_node(ctx, "Attribute"); + let node = create_node(vm, "Attribute"); - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); + let py_value = expression_to_ast(vm, value); + vm.ctx.set_attr(&node, "value", py_value); - let py_attr = ctx.new_str(name.to_string()); - ctx.set_attr(&node, "attr", py_attr); + let py_attr = vm.ctx.new_str(name.to_string()); + vm.ctx.set_attr(&node, "attr", py_attr); node } ast::Expression::Starred { value } => { - let node = create_node(ctx, "Starred"); + let node = create_node(vm, "Starred"); - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); + let py_value = expression_to_ast(vm, value); + vm.ctx.set_attr(&node, "value", py_value); node } ast::Expression::Slice { elements } => { - let node = create_node(ctx, "Slice"); + let node = create_node(vm, "Slice"); - let py_value = expressions_to_ast(ctx, elements); - ctx.set_attr(&node, "bounds", py_value); + let py_value = expressions_to_ast(vm, elements); + vm.ctx.set_attr(&node, "bounds", py_value); node } - ast::Expression::String { value } => string_to_ast(ctx, value), + ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { - let node = create_node(ctx, "Bytes"); - ctx.set_attr(&node, "s", ctx.new_bytes(value.clone())); + let node = create_node(vm, "Bytes"); + vm.ctx.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())); node } }; // TODO: retrieve correct lineno: - let lineno = ctx.new_int(1); - ctx.set_attr(&node, "lineno", lineno); + let lineno = vm.ctx.new_int(1); + vm.ctx.set_attr(&node, "lineno", lineno); node } -fn parameters_to_ast(ctx: &PyContext, args: &ast::Parameters) -> PyObjectRef { - let node = create_node(ctx, "arguments"); +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { + let node = create_node(vm, "arguments"); - ctx.set_attr( + vm.ctx.set_attr( &node, "args", - ctx.new_list(args.args.iter().map(|a| parameter_to_ast(ctx, a)).collect()), + vm.ctx + .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), ); node } -fn parameter_to_ast(ctx: &PyContext, parameter: &ast::Parameter) -> PyObjectRef { - let node = create_node(ctx, "arg"); +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { + let node = create_node(vm, "arg"); - let py_arg = ctx.new_str(parameter.arg.to_string()); - ctx.set_attr(&node, "arg", py_arg); + let py_arg = vm.ctx.new_str(parameter.arg.to_string()); + vm.ctx.set_attr(&node, "arg", py_arg); let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(ctx, annotation) + expression_to_ast(vm, annotation) } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "annotation", py_annotation); + vm.ctx.set_attr(&node, "annotation", py_annotation); node } -fn comprehension_to_ast(ctx: &PyContext, comprehension: &ast::Comprehension) -> PyObjectRef { - let node = create_node(ctx, "comprehension"); +fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { + let node = create_node(vm, "comprehension"); - let py_target = expression_to_ast(ctx, &comprehension.target); - ctx.set_attr(&node, "target", py_target); + let py_target = expression_to_ast(vm, &comprehension.target); + vm.ctx.set_attr(&node, "target", py_target); - let py_iter = expression_to_ast(ctx, &comprehension.iter); - ctx.set_attr(&node, "iter", py_iter); + let py_iter = expression_to_ast(vm, &comprehension.iter); + vm.ctx.set_attr(&node, "iter", py_iter); - let py_ifs = expressions_to_ast(ctx, &comprehension.ifs); - ctx.set_attr(&node, "ifs", py_ifs); + let py_ifs = expressions_to_ast(vm, &comprehension.ifs); + vm.ctx.set_attr(&node, "ifs", py_ifs); node } -fn string_to_ast(ctx: &PyContext, string: &ast::StringGroup) -> PyObjectRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { match string { ast::StringGroup::Constant { value } => { - let node = create_node(ctx, "Str"); - ctx.set_attr(&node, "s", ctx.new_str(value.clone())); + let node = create_node(vm, "Str"); + vm.ctx.set_attr(&node, "s", vm.ctx.new_str(value.clone())); node } ast::StringGroup::FormattedValue { value, .. } => { - let node = create_node(ctx, "FormattedValue"); - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); + let node = create_node(vm, "FormattedValue"); + let py_value = expression_to_ast(vm, value); + vm.ctx.set_attr(&node, "value", py_value); node } ast::StringGroup::Joined { values } => { - let node = create_node(ctx, "JoinedStr"); - let py_values = ctx.new_list( + let node = create_node(vm, "JoinedStr"); + let py_values = vm.ctx.new_list( values .iter() - .map(|value| string_to_ast(ctx, value)) + .map(|value| string_to_ast(vm, value)) .collect(), ); - ctx.set_attr(&node, "values", py_values); + vm.ctx.set_attr(&node, "values", py_values); node } } @@ -619,7 +635,7 @@ fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let internal_ast = parser::parse_program(&source_string) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - let ast_node = program_to_ast(&vm.ctx, &internal_ast); + let ast_node = program_to_ast(&vm, &internal_ast); Ok(ast_node) } @@ -628,6 +644,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "parse" => ctx.new_rustfunc(ast_parse), "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), - "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}) + "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}), + "AST" => py_class!(ctx, "_ast.AST", ctx.object(), {}), }) } From e44b02b88f726408478377fc62f094661d59c8ce Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 25 Mar 2019 17:52:37 +0100 Subject: [PATCH 024/658] Avoid trace log in tests --- tests/test_snippets.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 4be73b7bd..bc4c43a77 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -65,8 +65,7 @@ def run_via_cpython_bytecode(filename, test_type): # Step2: run cpython bytecode: env = os.environ.copy() - log_level = 'info' if test_type == _TestType.benchmark else 'debug' - env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) + env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' with pushd(CPYTHON_RUNNER_DIR): subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) @@ -74,8 +73,7 @@ def run_via_cpython_bytecode(filename, test_type): def run_via_rustpython(filename, test_type): env = os.environ.copy() - log_level = 'info' if test_type == _TestType.benchmark else 'trace' - env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) + env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' if env.get('CODE_COVERAGE', 'false') == 'true': subprocess.check_call( From 4ab59a8e4c369104ab8518f651038877ada301f1 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 25 Mar 2019 17:53:13 +0100 Subject: [PATCH 025/658] Do not run benchmarks in the main test suite --- tests/test_snippets.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_snippets.py b/tests/test_snippets.py index bc4c43a77..7aa9c2456 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -17,7 +17,6 @@ import compile_code class _TestType(enum.Enum): functional = 1 - benchmark = 2 logger = logging.getLogger('tests') @@ -25,7 +24,6 @@ ROOT_DIR = '..' TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) TEST_DIRS = { _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), - _TestType.benchmark: os.path.join(TEST_ROOT, 'benchmarks'), } CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) From 16e04cf298a67f1c2686fe846177789425a26df4 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 25 Mar 2019 17:54:04 +0100 Subject: [PATCH 026/658] Run rustpython tests without cargo middleman --- tests/test_snippets.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7aa9c2456..7edbd7b7e 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -73,12 +73,13 @@ def run_via_rustpython(filename, test_type): env = os.environ.copy() env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' + + target = 'release' if env.get('CODE_COVERAGE', 'false') == 'true': - subprocess.check_call( - ['cargo', 'run', filename], env=env) - else: - subprocess.check_call( - ['cargo', 'run', '--release', filename], env=env) + target = 'debug' + binary = os.path.abspath(os.path.join(ROOT_DIR, 'target', target, 'rustpython')) + + subprocess.check_call([binary, filename], env=env) def create_test_function(cls, filename, method, test_type): @@ -120,4 +121,7 @@ def get_test_files(): # @populate('cpython_bytecode') @populate('rustpython') class SampleTestCase(unittest.TestCase): - pass + @classmethod + def setUpClass(cls): + subprocess.check_call(['cargo', 'build']) + subprocess.check_call(['cargo', 'build', '--release']) From 2131a8130cfcf344a81437d7c7413211976dc25e Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 25 Mar 2019 17:55:17 +0100 Subject: [PATCH 027/658] Clean up some tests --- tests/snippets/append.py | 2 -- tests/snippets/strings.py | 8 ++++++++ tests/snippets/xfail_3.1.2.17.py | 7 ------- 3 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 tests/snippets/append.py delete mode 100644 tests/snippets/xfail_3.1.2.17.py diff --git a/tests/snippets/append.py b/tests/snippets/append.py deleted file mode 100644 index a0490cb6d..000000000 --- a/tests/snippets/append.py +++ /dev/null @@ -1,2 +0,0 @@ -x = [] -x.append(1) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 4483b37f6..31511db11 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + assert "a" == 'a' assert """a""" == "a" assert len(""" " "" " "" """) == 11 @@ -124,3 +126,9 @@ assert 'a' <= 'a' assert 'z' > 'b' assert 'z' >= 'b' assert 'a' >= 'a' + +def try_mutate_str(): + word = "word" + word[0] = 'x' + +assert_raises(TypeError, try_mutate_str) diff --git a/tests/snippets/xfail_3.1.2.17.py b/tests/snippets/xfail_3.1.2.17.py deleted file mode 100644 index fc956bef4..000000000 --- a/tests/snippets/xfail_3.1.2.17.py +++ /dev/null @@ -1,7 +0,0 @@ -word = "Python" - -word[0] = "J" # Should raise a error, immutable -word[2:] = "Jy" # Should raise a error, immutable - - - From 6ae10ed938a12dfdcede666a147fb4bba99666b7 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Mon, 25 Mar 2019 20:23:13 +0100 Subject: [PATCH 028/658] Fix build and add extend_class macro usage for set and frozenset --- vm/src/obj/objset.rs | 180 +++++++++++++++---------------------------- 1 file changed, 62 insertions(+), 118 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 300dc9bca..72ee8c897 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -85,12 +85,12 @@ fn create_set( PySet { elements: RefCell::new(elements), }, - PySet::class(vm).into_object(), + PySet::class(vm), )) } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { Ok(PyObject::new( PyFrozenSet { elements: elements }, - PyFrozenSet::class(vm).into_object(), + PyFrozenSet::class(vm), )) } else { Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) @@ -593,65 +593,43 @@ pub fn init(context: &PyContext) { set(iterable) -> new set object\n\n\ Build an unordered collection of unique elements."; - context.set_attr(set_type, "__contains__", context.new_rustfunc(set_contains)); - context.set_attr(set_type, "__len__", context.new_rustfunc(set_len)); - context.set_attr(set_type, "__new__", context.new_rustfunc(set_new)); - context.set_attr(set_type, "__repr__", context.new_rustfunc(set_repr)); - context.set_attr(set_type, "__eq__", context.new_rustfunc(set_eq)); - context.set_attr(set_type, "__ge__", context.new_rustfunc(set_ge)); - context.set_attr(set_type, "__gt__", context.new_rustfunc(set_gt)); - context.set_attr(set_type, "__le__", context.new_rustfunc(set_le)); - context.set_attr(set_type, "__lt__", context.new_rustfunc(set_lt)); - context.set_attr(set_type, "issubset", context.new_rustfunc(set_le)); - context.set_attr(set_type, "issuperset", context.new_rustfunc(set_ge)); - context.set_attr(set_type, "union", context.new_rustfunc(set_union)); - context.set_attr(set_type, "__or__", context.new_rustfunc(set_union)); - context.set_attr( - set_type, - "intersection", - context.new_rustfunc(set_intersection), - ); - context.set_attr(set_type, "__and__", context.new_rustfunc(set_intersection)); - context.set_attr(set_type, "difference", context.new_rustfunc(set_difference)); - context.set_attr(set_type, "__sub__", context.new_rustfunc(set_difference)); - context.set_attr( - set_type, - "symmetric_difference", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr( - set_type, - "__xor__", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr(set_type, "__doc__", context.new_str(set_doc.to_string())); - context.set_attr(set_type, "add", context.new_rustfunc(set_add)); - context.set_attr(set_type, "remove", context.new_rustfunc(set_remove)); - context.set_attr(set_type, "discard", context.new_rustfunc(set_discard)); - context.set_attr(set_type, "clear", context.new_rustfunc(set_clear)); - context.set_attr(set_type, "copy", context.new_rustfunc(set_copy)); - context.set_attr(set_type, "pop", context.new_rustfunc(set_pop)); - context.set_attr(set_type, "update", context.new_rustfunc(set_update)); - context.set_attr(set_type, "__ior__", context.new_rustfunc(set_ior)); - context.set_attr( - set_type, - "intersection_update", - context.new_rustfunc(set_intersection_update), - ); - context.set_attr(set_type, "__iand__", context.new_rustfunc(set_iand)); - context.set_attr( - set_type, - "difference_update", - context.new_rustfunc(set_difference_update), - ); - context.set_attr(set_type, "__isub__", context.new_rustfunc(set_isub)); - context.set_attr( - set_type, - "symmetric_difference_update", - context.new_rustfunc(set_symmetric_difference_update), - ); - context.set_attr(set_type, "__ixor__", context.new_rustfunc(set_ixor)); - context.set_attr(set_type, "__iter__", context.new_rustfunc(set_iter)); + extend_class!(context, set_type, { + "__contains__" => context.new_rustfunc(set_contains), + "__len__" => context.new_rustfunc(set_len), + "__new__" => context.new_rustfunc(set_new), + "__repr__" => context.new_rustfunc(set_repr), + "__eq__" => context.new_rustfunc(set_eq), + "__ge__" => context.new_rustfunc(set_ge), + "__gt__" => context.new_rustfunc(set_gt), + "__le__" => context.new_rustfunc(set_le), + "__lt__" => context.new_rustfunc(set_lt), + "issubset" => context.new_rustfunc(set_le), + "issuperset" => context.new_rustfunc(set_ge), + "union" => context.new_rustfunc(set_union), + "__or__" => context.new_rustfunc(set_union), + "intersection" => context.new_rustfunc(set_intersection), + "__and__" => context.new_rustfunc(set_intersection), + "difference" => context.new_rustfunc(set_difference), + "__sub__" => context.new_rustfunc(set_difference), + "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), + "__xor__" => context.new_rustfunc(set_symmetric_difference), + "__doc__" => context.new_str(set_doc.to_string()), + "add" => context.new_rustfunc(set_add), + "remove" => context.new_rustfunc(set_remove), + "discard" => context.new_rustfunc(set_discard), + "clear" => context.new_rustfunc(set_clear), + "copy" => context.new_rustfunc(set_copy), + "pop" => context.new_rustfunc(set_pop), + "update" => context.new_rustfunc(set_update), + "__ior__" => context.new_rustfunc(set_ior), + "intersection_update" => context.new_rustfunc(set_intersection_update), + "__iand__" => context.new_rustfunc(set_iand), + "difference_update" => context.new_rustfunc(set_difference_update), + "__isub__" => context.new_rustfunc(set_isub), + "symmetric_difference_update" => context.new_rustfunc(set_symmetric_difference_update), + "__ixor__" => context.new_rustfunc(set_ixor), + "__iter__" => context.new_rustfunc(set_iter) + }); let frozenset_type = &context.frozenset_type; @@ -659,61 +637,27 @@ pub fn init(context: &PyContext) { frozenset(iterable) -> frozenset object\n\n\ Build an immutable unordered collection of unique elements."; - context.set_attr(frozenset_type, "__new__", context.new_rustfunc(set_new)); - context.set_attr(frozenset_type, "__eq__", context.new_rustfunc(set_eq)); - context.set_attr(frozenset_type, "__ge__", context.new_rustfunc(set_ge)); - context.set_attr(frozenset_type, "__gt__", context.new_rustfunc(set_gt)); - context.set_attr(frozenset_type, "__le__", context.new_rustfunc(set_le)); - context.set_attr(frozenset_type, "__lt__", context.new_rustfunc(set_lt)); - context.set_attr(frozenset_type, "issubset", context.new_rustfunc(set_le)); - context.set_attr(frozenset_type, "issuperset", context.new_rustfunc(set_ge)); - context.set_attr(frozenset_type, "union", context.new_rustfunc(set_union)); - context.set_attr(frozenset_type, "__or__", context.new_rustfunc(set_union)); - context.set_attr( - frozenset_type, - "intersection", - context.new_rustfunc(set_intersection), - ); - context.set_attr( - frozenset_type, - "__and__", - context.new_rustfunc(set_intersection), - ); - context.set_attr( - frozenset_type, - "difference", - context.new_rustfunc(set_difference), - ); - context.set_attr( - frozenset_type, - "__sub__", - context.new_rustfunc(set_difference), - ); - context.set_attr( - frozenset_type, - "symmetric_difference", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr( - frozenset_type, - "__xor__", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr( - frozenset_type, - "__contains__", - context.new_rustfunc(set_contains), - ); - context.set_attr(frozenset_type, "__len__", context.new_rustfunc(set_len)); - context.set_attr( - frozenset_type, - "__doc__", - context.new_str(frozenset_doc.to_string()), - ); - context.set_attr( - frozenset_type, - "__repr__", - context.new_rustfunc(frozenset_repr), - ); - context.set_attr(frozenset_type, "copy", context.new_rustfunc(set_copy)); + extend_class!(context, frozenset_type, { + "__new__" => context.new_rustfunc(set_new), + "__eq__" => context.new_rustfunc(set_eq), + "__ge__" => context.new_rustfunc(set_ge), + "__gt__" => context.new_rustfunc(set_gt), + "__le__" => context.new_rustfunc(set_le), + "__lt__" => context.new_rustfunc(set_lt), + "issubset" => context.new_rustfunc(set_le), + "issuperset" => context.new_rustfunc(set_ge), + "union" => context.new_rustfunc(set_union), + "__or__" => context.new_rustfunc(set_union), + "intersection" => context.new_rustfunc(set_intersection), + "__and__" => context.new_rustfunc(set_intersection), + "difference" => context.new_rustfunc(set_difference), + "__sub__" => context.new_rustfunc(set_difference), + "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), + "__xor__" => context.new_rustfunc(set_symmetric_difference), + "__contains__" => context.new_rustfunc(set_contains), + "__len__" => context.new_rustfunc(set_len), + "__doc__" => context.new_str(frozenset_doc.to_string()), + "__repr__" => context.new_rustfunc(frozenset_repr), + "copy" => context.new_rustfunc(set_copy) + }); } From 17b816fbf83069332441c98daee231a9772cf809 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 25 Mar 2019 20:37:47 +0000 Subject: [PATCH 029/658] Fix objset compilation error caused by merge race. --- vm/src/obj/objset.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 72ee8c897..7071cc6e4 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -86,11 +86,13 @@ fn create_set( elements: RefCell::new(elements), }, PySet::class(vm), + None, )) } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { Ok(PyObject::new( PyFrozenSet { elements: elements }, PyFrozenSet::class(vm), + None, )) } else { Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) From ac2a5e281c55cc9fce29491e1795152e498f2395 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 24 Mar 2019 13:36:13 -0400 Subject: [PATCH 030/658] Add comment explaining bytes -> Uint8Array conversion --- wasm/lib/src/convert.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 909955b6d..641eb3538 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -126,6 +126,10 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { { let bytes = objbytes::get_value(&py_obj); unsafe { + // `Uint8Array::view` is an `unsafe fn` because it provides + // a direct view into the WASM linear memory; if you were to allocate + // something with Rust that view would probably become invalid. It's safe + // because we then copy the array using `Uint8Array::slice`. let view = Uint8Array::view(&bytes); view.slice(0, bytes.len() as u32).into() } From cc4f3fdb4097a6701158da7c30a8bf1f70e2f954 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 25 Mar 2019 19:18:01 -0700 Subject: [PATCH 031/658] Clean up TypeProtocol --- vm/src/builtins.rs | 14 +++++++------- vm/src/exceptions.rs | 2 +- vm/src/frame.rs | 6 +++--- vm/src/function.rs | 2 +- vm/src/macros.rs | 2 +- vm/src/obj/objfloat.rs | 2 +- vm/src/obj/objfunction.rs | 2 +- vm/src/obj/objint.rs | 10 +++++----- vm/src/obj/objiter.rs | 2 +- vm/src/obj/objnone.rs | 10 +++++----- vm/src/obj/objobject.rs | 24 ++++++++++++------------ vm/src/obj/objproperty.rs | 10 ++++++---- vm/src/obj/objsequence.rs | 4 ++-- vm/src/obj/objset.rs | 30 +++++++++++++++--------------- vm/src/obj/objstr.rs | 6 +++--- vm/src/obj/objsuper.rs | 2 +- vm/src/obj/objtype.rs | 28 +++++++++------------------- vm/src/pyobject.rs | 36 ++++++++++++++++++------------------ vm/src/stdlib/json.rs | 2 +- vm/src/vm.rs | 16 ++++++++-------- 20 files changed, 101 insertions(+), 109 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 4d9fb7989..fbd2d5356 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -88,7 +88,7 @@ fn builtin_bin(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn builtin_callable(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - let is_callable = objtype::class_has_attr(&obj.type_pyref(), "__call__"); + let is_callable = objtype::class_has_attr(&obj.class(), "__call__"); Ok(vm.new_bool(is_callable)) } @@ -240,7 +240,7 @@ fn make_scope( } else if vm.isinstance(arg, &dict_type)? { Some(arg) } else { - let arg_typ = arg.typ(); + let arg_typ = arg.class(); let actual_type = vm.to_pystr(&arg_typ)?; let expected_type_name = vm.to_pystr(&dict_type)?; return Err(vm.new_type_error(format!( @@ -368,7 +368,7 @@ fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) => vm.invoke(value, PyFuncArgs::default()), Err(..) => Err(vm.new_type_error(format!( "object of type '{}' has no method {:?}", - objtype::get_type_name(&obj.typ()), + objtype::get_type_name(&obj.class()), len_method_name ))), } @@ -607,7 +607,7 @@ fn builtin_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: fallback to using __len__ and __getitem__, if object supports sequence protocol Err(..) => Err(vm.new_type_error(format!( "'{}' object is not reversible", - objtype::get_type_name(&obj.typ()), + objtype::get_type_name(&obj.class()), ))), } } @@ -802,9 +802,9 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu }; for base in bases.clone() { - if objtype::issubclass(&base.type_pyref(), &metaclass) { - metaclass = base.type_pyref(); - } else if !objtype::issubclass(&metaclass, &base.type_pyref()) { + if objtype::issubclass(&base.class(), &metaclass) { + metaclass = base.class(); + } else if !objtype::issubclass(&metaclass, &base.class()) { return Err(vm.new_type_error("metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases".to_string())); } } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 3eb962edf..7177527da 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -68,7 +68,7 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { args, required = [(exc, Some(vm.ctx.exceptions.exception_type.clone()))] ); - let type_name = objtype::get_type_name(&exc.typ()); + let type_name = objtype::get_type_name(&exc.class()); let msg = if let Ok(m) = vm.get_attribute(exc.clone(), "msg") { match vm.to_pystr(&m) { Ok(msg) => msg, diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 3584a57a8..53479f8fc 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -949,7 +949,7 @@ impl Frame { // let args = PyFuncArgs::default(); // TODO: what happens when we got an error during handling exception? let args = if let Some(exc) = exc { - let exc_type = exc.typ(); + let exc_type = exc.class().into_object(); let exc_val = exc.clone(); let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? vec![exc_type, exc_val, exc_tb] @@ -1093,7 +1093,7 @@ impl Frame { Ok(found) => Ok(found), Err(_) => Err(vm.new_type_error(format!( "{} has no __contains__ method", - objtype::get_type_name(&haystack.typ()) + objtype::get_type_name(&haystack.class()) ))), } } @@ -1103,7 +1103,7 @@ impl Frame { Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), Err(_) => Err(vm.new_type_error(format!( "{} has no __contains__ method", - objtype::get_type_name(&haystack.typ()) + objtype::get_type_name(&haystack.class()) ))), } } diff --git a/vm/src/function.rs b/vm/src/function.rs index ed959d85c..1c61ffc87 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -99,7 +99,7 @@ impl PyFuncArgs { Ok(Some(kwarg)) } else { let expected_ty_name = vm.to_pystr(&ty)?; - let actual_ty_name = vm.to_pystr(&kwarg.typ())?; + let actual_ty_name = vm.to_pystr(&kwarg.class())?; Err(vm.new_type_error(format!( "argument of type {} is required for named parameter `{}` (got: {})", expected_ty_name, key, actual_ty_name diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 0c205ffc8..a63186797 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -20,7 +20,7 @@ macro_rules! type_check { let arg = &$args.args[$arg_count]; if !$crate::obj::objtype::isinstance(arg, &expected_type) { - let arg_typ = arg.typ(); + let arg_typ = arg.class(); let expected_type_name = $vm.to_pystr(&expected_type)?; let actual_type = $vm.to_pystr(&arg_typ)?; return Err($vm.new_type_error(format!( diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 958d368a8..5e68f5960 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -190,7 +190,7 @@ impl PyFloatRef { } } } else { - let type_name = objtype::get_type_name(&arg.typ()); + let type_name = objtype::get_type_name(&arg.class()); return Err(vm.new_type_error(format!("can't convert {} to float", type_name))); }; PyFloat { value }.into_ref_with_type(vm, cls) diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 1488ee79c..507d7697a 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -76,7 +76,7 @@ fn bind_method(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(function, None), (obj, None), (cls, None)] ); - if obj.is(&vm.get_none()) && !cls.is(&obj.typ()) { + if obj.is(&vm.get_none()) && !cls.is(&obj.class()) { Ok(function.clone()) } else { Ok(vm.ctx.new_bound_method(function.clone(), obj.clone())) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index dd40f06a4..f31997ead 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -213,8 +213,8 @@ impl PyIntRef { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Err(vm.new_type_error(format!( "unsupported operand type(s) for << '{}' and '{}'", - objtype::get_type_name(&self.as_object().typ()), - objtype::get_type_name(&other.typ()) + objtype::get_type_name(&self.as_object().class()), + objtype::get_type_name(&other.class()) ))); } @@ -236,8 +236,8 @@ impl PyIntRef { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Err(vm.new_type_error(format!( "unsupported operand type(s) for >> '{}' and '{}'", - objtype::get_type_name(&self.as_object().typ()), - objtype::get_type_name(&other.typ()) + objtype::get_type_name(&self.as_object().class()), + objtype::get_type_name(&other.class()) ))); } @@ -420,7 +420,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult PyResult { vm.call_method(iter_target, "__iter__", vec![]) - // let type_str = objstr::get_value(&vm.to_str(iter_target.typ()).unwrap()); + // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); // return Err(type_error); } diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index 840b00ecc..aff20f465 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -12,7 +12,7 @@ pub type PyNoneRef = PyRef; impl PyValue for PyNone { fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.none().type_pyref() + vm.ctx.none().class() } } @@ -44,7 +44,7 @@ impl PyNoneRef { fn get_attribute(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { trace!("None.__getattribute__({:?}, {:?})", self, name); - let cls = self.typ(); + let cls = self.class(); // Properties use a comparision with None to determine if they are either invoked by am // instance binding or a class binding. But if the object itself is None then this detection @@ -69,7 +69,7 @@ impl PyNoneRef { } if let Some(attr) = class_get_attr(&cls, &name.value) { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if class_has_attr(&attr_class, "__set__") { if let Some(get_func) = class_get_attr(&attr_class, "__get__") { return call_descriptor( @@ -88,7 +88,7 @@ impl PyNoneRef { // Ok(obj_attr) // } else if let Some(attr) = class_get_attr(&cls, &name.value) { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if let Some(get_func) = class_get_attr(&attr_class, "__get__") { call_descriptor(attr, get_func, self.into_object(), cls.into_object(), vm) } else { @@ -107,7 +107,7 @@ fn none_new(_: PyClassRef, vm: &VirtualMachine) -> PyNoneRef { } pub fn init(context: &PyContext) { - extend_class!(context, &context.none.typ(), { + extend_class!(context, &context.none.class(), { "__new__" => context.new_rustfunc(none_new), "__repr__" => context.new_rustfunc(PyNoneRef::repr), "__bool__" => context.new_rustfunc(PyNoneRef::bool), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 60473fad0..14dbb2e26 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -104,10 +104,10 @@ fn object_setattr( vm: &VirtualMachine, ) -> PyResult<()> { trace!("object.__setattr__({:?}, {}, {:?})", obj, attr_name, value); - let cls = obj.type_pyref(); + let cls = obj.class(); if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) { - if let Some(descriptor) = objtype::class_get_attr(&attr.type_pyref(), "__set__") { + if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__set__") { return vm .invoke(descriptor, vec![attr, obj.clone(), value]) .map(|_| ()); @@ -118,7 +118,7 @@ fn object_setattr( dict.set_item(&vm.ctx, &attr_name.value, value); Ok(()) } else { - let type_name = objtype::get_type_name(obj.type_ref()); + let type_name = objtype::get_type_name(&obj.class()); Err(vm.new_attribute_error(format!( "'{}' object has no attribute '{}'", type_name, &attr_name.value @@ -127,10 +127,10 @@ fn object_setattr( } fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let cls = obj.type_pyref(); + let cls = obj.class(); if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) { - if let Some(descriptor) = objtype::class_get_attr(&attr.type_pyref(), "__delete__") { + if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__delete__") { return vm.invoke(descriptor, vec![attr, obj.clone()]).map(|_| ()); } } @@ -139,7 +139,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) dict.del_item(&attr_name.value); Ok(()) } else { - let type_name = objtype::get_type_name(obj.type_ref()); + let type_name = objtype::get_type_name(&obj.class()); Err(vm.new_attribute_error(format!( "'{}' object has no attribute '{}'", type_name, &attr_name.value @@ -154,7 +154,7 @@ fn object_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn object_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, Some(vm.ctx.object()))]); - let type_name = objtype::get_type_name(&obj.typ()); + let type_name = objtype::get_type_name(&obj.class()); let address = obj.get_id(); Ok(vm.new_str(format!("<{} object at 0x{:x}>", type_name, address))) } @@ -216,7 +216,7 @@ fn object_init(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { } fn object_class(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { - obj.typ() + obj.class().into_object() } fn object_class_setter( @@ -224,7 +224,7 @@ fn object_class_setter( _value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - let type_repr = vm.to_pystr(&instance.typ())?; + let type_repr = vm.to_pystr(&instance.class())?; Err(vm.new_type_error(format!("can't change class of type '{}'", type_repr))) } @@ -247,10 +247,10 @@ fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { ); let name = objstr::get_value(&name_str); trace!("object.__getattribute__({:?}, {:?})", obj, name); - let cls = obj.type_pyref(); + let cls = obj.class(); if let Some(attr) = objtype::class_get_attr(&cls, &name) { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if objtype::class_has_attr(&attr_class, "__set__") { if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") { return vm.invoke(descriptor, vec![attr, obj.clone(), cls.into_object()]); @@ -279,7 +279,7 @@ fn object_getattr(obj: &PyObjectRef, attr_name: &str) -> Option { pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get class attributes: - let mut attributes = objtype::get_attributes(obj.type_pyref()); + let mut attributes = objtype::get_attributes(obj.class()); // Get instance attributes: if let Some(dict) = &obj.dict { diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index e6b11904e..9d7217110 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -6,7 +6,9 @@ use crate::function::IntoPyNativeFunc; use crate::function::OptionalArg; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{IdProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IdProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, +}; use crate::vm::VirtualMachine; /// Read-only property, doesn't have __set__ or __delete__ @@ -137,7 +139,7 @@ impl PyPropertyRef { setter: self.setter.clone(), deleter: self.deleter.clone(), } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&self)) } fn setter(self, setter: Option, vm: &VirtualMachine) -> PyResult { @@ -146,7 +148,7 @@ impl PyPropertyRef { setter: setter.or_else(|| self.setter.clone()), deleter: self.deleter.clone(), } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&self)) } fn deleter(self, deleter: Option, vm: &VirtualMachine) -> PyResult { @@ -155,7 +157,7 @@ impl PyPropertyRef { setter: self.setter.clone(), deleter: deleter.or_else(|| self.deleter.clone()), } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&self)) } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 80f4bf40f..a07b2454d 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -163,13 +163,13 @@ pub fn get_item( if sequence.payload::().is_some() { Ok(PyObject::new( PyList::from(elements.to_vec().get_slice_items(vm, &subscript)?), - sequence.type_pyref(), + sequence.class(), None, )) } else if sequence.payload::().is_some() { Ok(PyObject::new( PyTuple::from(elements.to_vec().get_slice_items(vm, &subscript)?), - sequence.type_pyref(), + sequence.class(), None, )) } else { diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 7071cc6e4..ad82c34de 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -231,16 +231,16 @@ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMach fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { trace!("set.len called with: {:?}", args); arg_check!(vm, args, required = [(s, None)]); - validate_set_or_frozenset(vm, s.type_pyref())?; + validate_set_or_frozenset(vm, s.class())?; let elements = get_elements(s); Ok(vm.context().new_int(elements.len())) } fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { trace!("set.copy called with: {:?}", obj); - validate_set_or_frozenset(vm, obj.type_pyref())?; + validate_set_or_frozenset(vm, obj.class())?; let elements = get_elements(&obj).clone(); - create_set(vm, elements, obj.type_pyref()) + create_set(vm, elements, obj.class()) } fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -265,7 +265,7 @@ fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(set, None), (needle, None)]); - validate_set_or_frozenset(vm, set.type_pyref())?; + validate_set_or_frozenset(vm, set.class())?; for element in get_elements(set).iter() { match vm._eq(needle.clone(), element.1.clone()) { Ok(value) => { @@ -333,8 +333,8 @@ fn set_compare_inner( ) -> PyResult { arg_check!(vm, args, required = [(zelf, None), (other, None)]); - validate_set_or_frozenset(vm, zelf.type_pyref())?; - validate_set_or_frozenset(vm, other.type_pyref())?; + validate_set_or_frozenset(vm, zelf.class())?; + validate_set_or_frozenset(vm, other.class())?; let get_zelf = |swap: bool| -> &PyObjectRef { if swap { @@ -370,13 +370,13 @@ fn set_compare_inner( } fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, zelf.type_pyref())?; - validate_set_or_frozenset(vm, other.type_pyref())?; + validate_set_or_frozenset(vm, zelf.class())?; + validate_set_or_frozenset(vm, other.class())?; let mut elements = get_elements(&zelf).clone(); elements.extend(get_elements(&other).clone()); - create_set(vm, elements, zelf.type_pyref()) + create_set(vm, elements, zelf.class()) } fn set_intersection(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -392,8 +392,8 @@ fn set_symmetric_difference( other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - validate_set_or_frozenset(vm, zelf.type_pyref())?; - validate_set_or_frozenset(vm, other.type_pyref())?; + validate_set_or_frozenset(vm, zelf.class())?; + validate_set_or_frozenset(vm, other.class())?; let mut elements = HashMap::new(); for element in get_elements(&zelf).iter() { @@ -410,7 +410,7 @@ fn set_symmetric_difference( } } - create_set(vm, elements, zelf.type_pyref()) + create_set(vm, elements, zelf.class()) } enum SetCombineOperation { @@ -424,8 +424,8 @@ fn set_combine_inner( vm: &VirtualMachine, op: SetCombineOperation, ) -> PyResult { - validate_set_or_frozenset(vm, zelf.type_pyref())?; - validate_set_or_frozenset(vm, other.type_pyref())?; + validate_set_or_frozenset(vm, zelf.class())?; + validate_set_or_frozenset(vm, other.class())?; let mut elements = HashMap::new(); for element in get_elements(&zelf).iter() { @@ -439,7 +439,7 @@ fn set_combine_inner( } } - create_set(vm, elements, zelf.type_pyref()) + create_set(vm, elements, zelf.class()) } fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 67ee8e5e7..aa62812a2 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -715,7 +715,7 @@ fn str_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let zelf = &args.args[0]; if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { - let zelf_typ = zelf.typ(); + let zelf_typ = zelf.class(); let actual_type = vm.to_pystr(&zelf_typ)?; return Err(vm.new_type_error(format!( "descriptor 'format' requires a 'str' object but received a '{}'", @@ -738,7 +738,7 @@ fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: & let returned_type = vm.ctx.new_str(format_spec.to_string()); let result = vm.call_method(&argument, "__format__", vec![returned_type])?; if !objtype::isinstance(&result, &vm.ctx.str_type()) { - let result_type = result.typ(); + let result_type = result.class(); let actual_type = vm.to_pystr(&result_type)?; return Err(vm.new_type_error(format!("__format__ must return a str, not {}", actual_type))); } @@ -809,7 +809,7 @@ fn str_new( OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), OptionalArg::Missing => vm.new_str("".to_string()), }; - if string.typ().is(&cls) { + if string.class().is(&cls) { TryFromObject::try_from_object(vm, string) } else { let payload = string.payload::().unwrap(); diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index f04fb6605..fad365e24 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -112,7 +112,7 @@ fn super_new( // Check type argument: if !objtype::isinstance(py_type.as_object(), &vm.get_type()) { - let type_name = objtype::get_type_name(py_type.as_object().type_ref()); + let type_name = objtype::get_type_name(&py_type.as_object().class()); return Err(vm.new_type_error(format!( "super() argument 1 must be type, not {}", type_name diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index d87ca3843..857ec402f 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -39,12 +39,6 @@ impl PyValue for PyClass { } } -impl TypeProtocol for PyClassRef { - fn type_ref(&self) -> &PyObjectRef { - &self.as_object().type_ref() - } -} - struct IterMro<'a> { cls: &'a PyClassRef, offset: Option, @@ -117,10 +111,10 @@ impl PyClassRef { fn getattribute(self, name_ref: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_ref.value; trace!("type.__getattribute__({:?}, {:?})", self, name); - let mcl = self.type_pyref(); + let mcl = self.class(); if let Some(attr) = class_get_attr(&mcl, &name) { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if class_has_attr(&attr_class, "__set__") { if let Some(descriptor) = class_get_attr(&attr_class, "__get__") { return vm.invoke( @@ -132,7 +126,7 @@ impl PyClassRef { } if let Some(attr) = class_get_attr(&self, &name) { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if let Some(descriptor) = class_get_attr(&attr_class, "__get__") { let none = vm.get_none(); return vm.invoke(descriptor, vec![attr, none, self.into_object()]); @@ -156,8 +150,8 @@ impl PyClassRef { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - if let Some(attr) = class_get_attr(&self.type_pyref(), &attr_name.value) { - if let Some(descriptor) = class_get_attr(&attr.type_pyref(), "__set__") { + if let Some(attr) = class_get_attr(&self.class(), &attr_name.value) { + if let Some(descriptor) = class_get_attr(&attr.class(), "__set__") { vm.invoke(descriptor, vec![attr, self.into_object(), value])?; return Ok(()); } @@ -218,7 +212,7 @@ fn _mro(cls: &PyClassRef) -> Vec { /// Determines if `obj` actually an instance of `cls`, this doesn't call __instancecheck__, so only /// use this if `cls` is known to have not overridden the base __instancecheck__ magic method. pub fn isinstance(obj: &PyObjectRef, cls: &PyClassRef) -> bool { - issubclass(&obj.type_pyref(), &cls) + issubclass(&obj.class(), &cls) } /// Determines if `subclass` is actually a subclass of `cls`, this doesn't call __subclasscheck__, @@ -229,18 +223,14 @@ pub fn issubclass(subclass: &PyClassRef, cls: &PyClassRef) -> bool { subclass.is(cls) || mro.iter().any(|c| c.is(cls.as_object())) } -pub fn get_type_name(typ: &PyObjectRef) -> String { - if let Some(PyClass { name, .. }) = &typ.payload::() { - name.clone() - } else { - panic!("Cannot get type_name of non-type type {:?}", typ); - } +pub fn get_type_name(typ: &PyClassRef) -> String { + typ.name.clone() } pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { debug!("type.__new__ {:?}", args); if args.args.len() == 2 { - Ok(args.args[1].typ()) + Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { let (typ, name, bases, dict) = args.bind(vm)?; type_new_class(vm, typ, name, bases, dict).map(|x| x.into_object()) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 78e82c51b..739012c5e 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -88,7 +88,7 @@ impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { - let type_name = objtype::get_type_name(&self.typ()); + let type_name = objtype::get_type_name(&self.class()); // We don't have access to a vm, so just assume that if its parent's name // is type, it's a type if type_name == "type" { @@ -101,7 +101,7 @@ impl fmt::Display for PyObject { if let Some(PyModule { ref name, .. }) = self.payload::() { return write!(f, "module '{}'", name); } - write!(f, "'{}' object", objtype::get_type_name(&self.typ())) + write!(f, "'{}' object", objtype::get_type_name(&self.class())) } } @@ -163,7 +163,7 @@ pub struct PyNotImplemented; impl PyValue for PyNotImplemented { fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.not_implemented().type_pyref() + vm.ctx.not_implemented().class() } } @@ -772,7 +772,7 @@ impl PyRef { pub fn typ(&self) -> PyClassRef { PyRef { - obj: self.obj.typ(), + obj: self.obj.class().into_object(), _payload: PhantomData, } } @@ -802,7 +802,7 @@ where } else { let class = T::class(vm); let expected_type = vm.to_pystr(&class)?; - let actual_type = vm.to_pystr(&obj.typ())?; + let actual_type = vm.to_pystr(&obj.class())?; Err(vm.new_type_error(format!( "Expected type {}, not {}", expected_type, actual_type, @@ -877,18 +877,12 @@ impl IdProtocol for PyRef { } pub trait TypeProtocol { - fn typ(&self) -> PyObjectRef { - self.type_ref().clone() - } - fn type_pyref(&self) -> PyClassRef { - self.typ().downcast().unwrap() - } - fn type_ref(&self) -> &PyObjectRef; + fn class(&self) -> PyClassRef; } impl TypeProtocol for PyObjectRef { - fn type_ref(&self) -> &PyObjectRef { - (**self).type_ref() + fn class(&self) -> PyClassRef { + (**self).class() } } @@ -896,8 +890,14 @@ impl TypeProtocol for PyObject where T: ?Sized + PyObjectPayload, { - fn type_ref(&self) -> &PyObjectRef { - self.typ.as_object() + fn class(&self) -> PyClassRef { + self.typ.clone() + } +} + +impl TypeProtocol for PyRef { + fn class(&self) -> PyClassRef { + self.obj.typ.clone() } } @@ -956,7 +956,7 @@ pub trait BufferProtocol { impl BufferProtocol for PyObjectRef { fn readonly(&self) -> bool { - match objtype::get_type_name(&self.typ()).as_ref() { + match objtype::get_type_name(&self.class()).as_ref() { "bytes" => false, "bytearray" | "memoryview" => true, _ => panic!("Bytes-Like type expected not {:?}", self), @@ -1249,7 +1249,7 @@ where "must be {} or {}, not {}", A::class(vm), B::class(vm), - obj.type_pyref() + obj.class() )) }) } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index eac3619c2..b881b7a72 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -74,7 +74,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { } else { Err(serde::ser::Error::custom(format!( "Object of type '{:?}' is not serializable", - self.pyobject.typ() + self.pyobject.class() ))) } } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 0ee9798ab..92c264bcd 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -174,8 +174,8 @@ impl VirtualMachine { b: PyObjectRef, op: &str, ) -> PyObjectRef { - let a_type_name = objtype::get_type_name(&a.typ()); - let b_type_name = objtype::get_type_name(&b.typ()); + let a_type_name = objtype::get_type_name(&a.class()); + let b_type_name = objtype::get_type_name(&b.class()); self.new_type_error(format!( "Unsupported operand types for '{}': '{}' and '{}'", op, a_type_name, b_type_name @@ -270,7 +270,7 @@ impl VirtualMachine { pub fn isinstance(&self, obj: &PyObjectRef, cls: &PyClassRef) -> PyResult { // cpython first does an exact check on the type, although documentation doesn't state that // https://github.com/python/cpython/blob/a24107b04c1277e3c1105f98aff5bfa3a98b33a0/Objects/abstract.c#L2408 - if Rc::ptr_eq(&obj.typ(), cls.as_object()) { + if Rc::ptr_eq(&obj.class().into_object(), cls.as_object()) { Ok(true) } else { let ret = self.call_method(cls.as_object(), "__instancecheck__", vec![obj.clone()])?; @@ -286,10 +286,10 @@ impl VirtualMachine { } pub fn call_get_descriptor(&self, attr: PyObjectRef, obj: PyObjectRef) -> PyResult { - let attr_class = attr.type_pyref(); + let attr_class = attr.class(); if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") { - let cls = obj.typ(); - self.invoke(descriptor, vec![attr, obj.clone(), cls]) + let cls = obj.class(); + self.invoke(descriptor, vec![attr, obj.clone(), cls.into_object()]) } else { Ok(attr) } @@ -300,7 +300,7 @@ impl VirtualMachine { T: Into, { // This is only used in the vm for magic methods, which use a greatly simplified attribute lookup. - let cls = obj.type_pyref(); + let cls = obj.class(); match objtype::class_get_attr(&cls, method_name) { Some(func) => { trace!( @@ -583,7 +583,7 @@ impl VirtualMachine { // get_method should be used for internal access to magic methods (by-passing // the full getattribute look-up. pub fn get_method(&self, obj: PyObjectRef, method_name: &str) -> PyResult { - let cls = obj.type_pyref(); + let cls = obj.class(); match objtype::class_get_attr(&cls, method_name) { Some(method) => self.call_get_descriptor(method, obj.clone()), None => Err(self.new_type_error(format!("{} has no method {:?}", obj, method_name))), From 6474a4a6ef895c8b57e4482ec6d26b0474ffbd2c Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 25 Mar 2019 19:18:07 -0700 Subject: [PATCH 032/658] Remove objtype::get_type_name() --- vm/src/builtins.rs | 7 ++----- vm/src/exceptions.rs | 3 +-- vm/src/frame.rs | 14 ++++++-------- vm/src/obj/objfloat.rs | 3 +-- vm/src/obj/objint.rs | 15 +++------------ vm/src/obj/objobject.rs | 11 +++++------ vm/src/obj/objsuper.rs | 3 +-- vm/src/obj/objtype.rs | 4 ---- vm/src/pyobject.rs | 6 +++--- vm/src/vm.rs | 6 +++--- 10 files changed, 25 insertions(+), 47 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index fbd2d5356..27ce8aadd 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -368,7 +368,7 @@ fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) => vm.invoke(value, PyFuncArgs::default()), Err(..) => Err(vm.new_type_error(format!( "object of type '{}' has no method {:?}", - objtype::get_type_name(&obj.class()), + obj.class(), len_method_name ))), } @@ -605,10 +605,7 @@ fn builtin_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { match vm.get_method(obj.clone(), "__reversed__") { Ok(value) => vm.invoke(value, PyFuncArgs::default()), // TODO: fallback to using __len__ and __getitem__, if object supports sequence protocol - Err(..) => Err(vm.new_type_error(format!( - "'{}' object is not reversible", - objtype::get_type_name(&obj.class()), - ))), + Err(..) => Err(vm.new_type_error(format!("'{}' object is not reversible", obj.class()))), } } // builtin_reversed diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 7177527da..f850d7697 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -68,7 +68,6 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { args, required = [(exc, Some(vm.ctx.exceptions.exception_type.clone()))] ); - let type_name = objtype::get_type_name(&exc.class()); let msg = if let Ok(m) = vm.get_attribute(exc.clone(), "msg") { match vm.to_pystr(&m) { Ok(msg) => msg, @@ -77,7 +76,7 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { panic!("Error message must be set"); }; - let s = format!("{}: {}", type_name, msg); + let s = format!("{}: {}", exc.class(), msg); Ok(vm.new_str(s)) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 53479f8fc..6f8815cbe 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1091,20 +1091,18 @@ impl Frame { fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { match self._membership(vm, needle, &haystack) { Ok(found) => Ok(found), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - objtype::get_type_name(&haystack.class()) - ))), + Err(_) => { + Err(vm.new_type_error(format!("{} has no __contains__ method", haystack.class()))) + } } } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { match self._membership(vm, needle, &haystack) { Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - objtype::get_type_name(&haystack.class()) - ))), + Err(_) => { + Err(vm.new_type_error(format!("{} has no __contains__ method", haystack.class()))) + } } } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 5e68f5960..e261a689e 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -190,8 +190,7 @@ impl PyFloatRef { } } } else { - let type_name = objtype::get_type_name(&arg.class()); - return Err(vm.new_type_error(format!("can't convert {} to float", type_name))); + return Err(vm.new_type_error(format!("can't convert {} to float", arg.class()))); }; PyFloat { value }.into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index f31997ead..e2203469a 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -211,11 +211,7 @@ impl PyIntRef { fn lshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { - return Err(vm.new_type_error(format!( - "unsupported operand type(s) for << '{}' and '{}'", - objtype::get_type_name(&self.as_object().class()), - objtype::get_type_name(&other.class()) - ))); + return Ok(vm.ctx.not_implemented()); } if let Some(n_bits) = get_value(&other).to_usize() { @@ -234,11 +230,7 @@ impl PyIntRef { fn rshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { - return Err(vm.new_type_error(format!( - "unsupported operand type(s) for >> '{}' and '{}'", - objtype::get_type_name(&self.as_object().class()), - objtype::get_type_name(&other.class()) - ))); + return Ok(vm.ctx.not_implemented()); } if let Some(n_bits) = get_value(&other).to_usize() { @@ -420,10 +412,9 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult PyResult { fn object_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, Some(vm.ctx.object()))]); - let type_name = objtype::get_type_name(&obj.class()); let address = obj.get_id(); - Ok(vm.new_str(format!("<{} object at 0x{:x}>", type_name, address))) + Ok(vm.new_str(format!("<{} object at 0x{:x}>", obj.class(), address))) } pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index fad365e24..0061b41e9 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -112,10 +112,9 @@ fn super_new( // Check type argument: if !objtype::isinstance(py_type.as_object(), &vm.get_type()) { - let type_name = objtype::get_type_name(&py_type.as_object().class()); return Err(vm.new_type_error(format!( "super() argument 1 must be type, not {}", - type_name + py_type.class() ))); } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 857ec402f..14958300b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -223,10 +223,6 @@ pub fn issubclass(subclass: &PyClassRef, cls: &PyClassRef) -> bool { subclass.is(cls) || mro.iter().any(|c| c.is(cls.as_object())) } -pub fn get_type_name(typ: &PyClassRef) -> String { - typ.name.clone() -} - pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { debug!("type.__new__ {:?}", args); if args.args.len() == 2 { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 739012c5e..f15c5e09b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -88,7 +88,7 @@ impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { - let type_name = objtype::get_type_name(&self.class()); + let type_name = self.class().name.clone(); // We don't have access to a vm, so just assume that if its parent's name // is type, it's a type if type_name == "type" { @@ -101,7 +101,7 @@ impl fmt::Display for PyObject { if let Some(PyModule { ref name, .. }) = self.payload::() { return write!(f, "module '{}'", name); } - write!(f, "'{}' object", objtype::get_type_name(&self.class())) + write!(f, "'{}' object", self.class()) } } @@ -956,7 +956,7 @@ pub trait BufferProtocol { impl BufferProtocol for PyObjectRef { fn readonly(&self) -> bool { - match objtype::get_type_name(&self.class()).as_ref() { + match self.class().name.as_str() { "bytes" => false, "bytearray" | "memoryview" => true, _ => panic!("Bytes-Like type expected not {:?}", self), diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 92c264bcd..df092bdc8 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -174,11 +174,11 @@ impl VirtualMachine { b: PyObjectRef, op: &str, ) -> PyObjectRef { - let a_type_name = objtype::get_type_name(&a.class()); - let b_type_name = objtype::get_type_name(&b.class()); self.new_type_error(format!( "Unsupported operand types for '{}': '{}' and '{}'", - op, a_type_name, b_type_name + op, + a.class(), + b.class() )) } From 8bdc766bedf04063a5a0027f73db6d469202570a Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 25 Mar 2019 19:32:58 -0700 Subject: [PATCH 033/658] Use name field directly --- vm/src/builtins.rs | 6 ++++-- vm/src/exceptions.rs | 2 +- vm/src/frame.rs | 14 ++++++++------ vm/src/obj/objfloat.rs | 2 +- vm/src/obj/objint.rs | 2 +- vm/src/obj/objobject.rs | 6 +++--- vm/src/obj/objsuper.rs | 2 +- vm/src/pyobject.rs | 2 +- vm/src/vm.rs | 4 ++-- 9 files changed, 22 insertions(+), 18 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 27ce8aadd..8a4a6adc3 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -368,7 +368,7 @@ fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) => vm.invoke(value, PyFuncArgs::default()), Err(..) => Err(vm.new_type_error(format!( "object of type '{}' has no method {:?}", - obj.class(), + obj.class().name, len_method_name ))), } @@ -605,7 +605,9 @@ fn builtin_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { match vm.get_method(obj.clone(), "__reversed__") { Ok(value) => vm.invoke(value, PyFuncArgs::default()), // TODO: fallback to using __len__ and __getitem__, if object supports sequence protocol - Err(..) => Err(vm.new_type_error(format!("'{}' object is not reversible", obj.class()))), + Err(..) => { + Err(vm.new_type_error(format!("'{}' object is not reversible", obj.class().name))) + } } } // builtin_reversed diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index f850d7697..fb757cd0f 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -76,7 +76,7 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { panic!("Error message must be set"); }; - let s = format!("{}: {}", exc.class(), msg); + let s = format!("{}: {}", exc.class().name, msg); Ok(vm.new_str(s)) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 6f8815cbe..6fd5cfea8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1091,18 +1091,20 @@ impl Frame { fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { match self._membership(vm, needle, &haystack) { Ok(found) => Ok(found), - Err(_) => { - Err(vm.new_type_error(format!("{} has no __contains__ method", haystack.class()))) - } + Err(_) => Err(vm.new_type_error(format!( + "{} has no __contains__ method", + haystack.class().name + ))), } } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { match self._membership(vm, needle, &haystack) { Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => { - Err(vm.new_type_error(format!("{} has no __contains__ method", haystack.class()))) - } + Err(_) => Err(vm.new_type_error(format!( + "{} has no __contains__ method", + haystack.class().name + ))), } } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index e261a689e..ca6bad9d2 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -190,7 +190,7 @@ impl PyFloatRef { } } } else { - return Err(vm.new_type_error(format!("can't convert {} to float", arg.class()))); + return Err(vm.new_type_error(format!("can't convert {} to float", arg.class().name))); }; PyFloat { value }.into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index e2203469a..c7f0ff5af 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -414,7 +414,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult PyResult { fn object_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, Some(vm.ctx.object()))]); let address = obj.get_id(); - Ok(vm.new_str(format!("<{} object at 0x{:x}>", obj.class(), address))) + Ok(vm.new_str(format!("<{} object at 0x{:x}>", obj.class().name, address))) } pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 0061b41e9..24424d265 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -114,7 +114,7 @@ fn super_new( if !objtype::isinstance(py_type.as_object(), &vm.get_type()) { return Err(vm.new_type_error(format!( "super() argument 1 must be type, not {}", - py_type.class() + py_type.class().name ))); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index f15c5e09b..afee7060e 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -101,7 +101,7 @@ impl fmt::Display for PyObject { if let Some(PyModule { ref name, .. }) = self.payload::() { return write!(f, "module '{}'", name); } - write!(f, "'{}' object", self.class()) + write!(f, "'{}' object", self.class().name) } } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index df092bdc8..41321c266 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -177,8 +177,8 @@ impl VirtualMachine { self.new_type_error(format!( "Unsupported operand types for '{}': '{}' and '{}'", op, - a.class(), - b.class() + a.class().name, + b.class().name )) } From 625d2357bd493175a7379eaf876fc33711fdb1c1 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 25 Mar 2019 19:34:24 -0700 Subject: [PATCH 034/658] Fix wasm --- wasm/lib/src/wasm_builtins.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wasm/lib/src/wasm_builtins.rs b/wasm/lib/src/wasm_builtins.rs index d5bc830df..ada0c83f5 100644 --- a/wasm/lib/src/wasm_builtins.rs +++ b/wasm/lib/src/wasm_builtins.rs @@ -25,7 +25,7 @@ pub fn format_print_args(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result Date: Tue, 26 Mar 2019 15:16:30 +0000 Subject: [PATCH 035/658] Change types inside Scope to PyDictRef. --- vm/src/builtins.rs | 28 +++++++++-------------- vm/src/frame.rs | 29 +++++++++++------------ vm/src/import.rs | 2 +- vm/src/obj/objframe.rs | 52 ++++++++++++++++++------------------------ vm/src/pyobject.rs | 10 +++++--- vm/src/vm.rs | 9 ++++---- 6 files changed, 59 insertions(+), 71 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 8a4a6adc3..1de24b86a 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -11,6 +11,7 @@ use num_traits::{Signed, ToPrimitive}; use crate::compile; use crate::import::import_module; use crate::obj::objbool; +use crate::obj::objdict::PyDictRef; use crate::obj::objint; use crate::obj::objiter; use crate::obj::objstr::{self, PyStringRef}; @@ -27,14 +28,6 @@ use crate::obj::objcode::PyCodeRef; #[cfg(not(target_arch = "wasm32"))] use crate::stdlib::io::io_open; -fn get_locals(vm: &VirtualMachine) -> PyObjectRef { - vm.get_locals() -} - -fn dir_locals(vm: &VirtualMachine) -> PyObjectRef { - get_locals(vm) -} - fn builtin_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(x, None)]); match vm.get_method(x.clone(), "__abs__") { @@ -146,7 +139,7 @@ fn builtin_delattr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if args.args.is_empty() { - Ok(dir_locals(vm)) + Ok(vm.get_locals().into_object()) } else { let obj = args.args.into_iter().next().unwrap(); let seq = vm.call_method(&obj, "__dir__", vec![])?; @@ -254,11 +247,11 @@ fn make_scope( let current_scope = vm.current_scope(); let globals = match globals { - Some(dict) => dict.clone(), + Some(dict) => dict.clone().downcast().unwrap(), None => current_scope.globals.clone(), }; let locals = match locals { - Some(dict) => Some(dict.clone()), + Some(dict) => dict.clone().downcast().ok(), None => current_scope.get_only_locals(), }; @@ -300,7 +293,7 @@ fn builtin_getattr( } } -fn builtin_globals(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { +fn builtin_globals(vm: &VirtualMachine) -> PyResult { Ok(vm.current_scope().globals.clone()) } @@ -374,9 +367,8 @@ fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -fn builtin_locals(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args); - Ok(vm.get_locals()) +fn builtin_locals(vm: &VirtualMachine) -> PyDictRef { + vm.get_locals() } fn builtin_max(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -814,13 +806,15 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; + let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?; + let cells = vm.ctx.new_dict(); - vm.invoke_with_locals(function, cells.clone().into_object(), namespace.clone())?; + vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; let class = vm.call_method( metaclass.as_object(), "__call__", - vec![name_arg, bases, namespace], + vec![name_arg, bases, namespace.into_object()], )?; cells.set_item(&vm.ctx, "__class__", class.clone()); Ok(class) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 6fd5cfea8..88a619eb3 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,8 +12,7 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict; -use crate::obj::objdict::PyDict; +use crate::obj::objdict::{self, PyDictRef}; use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; @@ -80,8 +79,8 @@ impl<'a, T> Iterator for Iter<'a, T> { #[derive(Clone)] pub struct Scope { - locals: RcList, - pub globals: PyObjectRef, + locals: RcList, + pub globals: PyDictRef, } impl fmt::Debug for Scope { @@ -92,7 +91,7 @@ impl fmt::Debug for Scope { } impl Scope { - pub fn new(locals: Option, globals: PyObjectRef) -> Scope { + pub fn new(locals: Option, globals: PyDictRef) -> Scope { let locals = match locals { Some(dict) => RcList::new().insert(dict), None => RcList::new(), @@ -100,18 +99,18 @@ impl Scope { Scope { locals, globals } } - pub fn get_locals(&self) -> PyObjectRef { + pub fn get_locals(&self) -> PyDictRef { match self.locals.iter().next() { Some(dict) => dict.clone(), None => self.globals.clone(), } } - pub fn get_only_locals(&self) -> Option { + pub fn get_only_locals(&self) -> Option { self.locals.iter().next().cloned() } - pub fn child_scope_with_locals(&self, locals: PyObjectRef) -> Scope { + pub fn child_scope_with_locals(&self, locals: PyDictRef) -> Scope { Scope { locals: self.locals.clone().insert(locals), globals: self.globals.clone(), @@ -119,7 +118,7 @@ impl Scope { } pub fn child_scope(&self, ctx: &PyContext) -> Scope { - self.child_scope_with_locals(ctx.new_dict().into_object()) + self.child_scope_with_locals(ctx.new_dict()) } } @@ -1232,13 +1231,11 @@ impl fmt::Debug for Frame { .iter() .map(|elem| format!("\n > {:?}", elem)) .collect::(); - let local_str = match self.scope.get_locals().payload::() { - Some(dict) => objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) - .iter() - .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) - .collect::(), - None => panic!("locals unexpectedly not wrapping a dict!",), - }; + let dict = self.scope.get_locals(); + let local_str = objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) + .iter() + .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) + .collect::(); write!( f, "Frame Object {{ \n Stack:{}\n Blocks:{}\n Locals:{}\n}}", diff --git a/vm/src/import.rs b/vm/src/import.rs index 987555915..936b89680 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -40,7 +40,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let attrs = vm.ctx.new_dict(); attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); - vm.run_code_obj(code_obj, Scope::new(None, attrs.clone().into_object()))?; + vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/obj/objframe.rs b/vm/src/obj/objframe.rs index 8657dc280..b4ed15f74 100644 --- a/vm/src/obj/objframe.rs +++ b/vm/src/obj/objframe.rs @@ -2,43 +2,35 @@ */ -use crate::frame::Frame; -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use super::objcode::PyCodeRef; +use super::objdict::PyDictRef; +use crate::frame::FrameRef; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let frame_type = &context.frame_type; - extend_class!(context, frame_type, { - "__new__" => context.new_rustfunc(frame_new), - "__repr__" => context.new_rustfunc(frame_repr), - "f_locals" => context.new_property(frame_flocals), - "f_code" => context.new_property(frame_fcode) + extend_class!(context, &context.frame_type, { + "__new__" => context.new_rustfunc(FrameRef::new), + "__repr__" => context.new_rustfunc(FrameRef::repr), + "f_locals" => context.new_property(FrameRef::flocals), + "f_code" => context.new_property(FrameRef::fcode), }); } -fn frame_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_cls, None)]); - Err(vm.new_type_error("Cannot directly create frame object".to_string())) -} +impl FrameRef { + fn new(_class: FrameRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot directly create frame object".to_string())) + } -fn frame_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_frame, Some(vm.ctx.frame_type()))]); - let repr = "".to_string(); - Ok(vm.new_str(repr)) -} + fn repr(self, _vm: &VirtualMachine) -> String { + "".to_string() + } -fn frame_flocals(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(frame, Some(vm.ctx.frame_type()))]); - let frame = get_value(frame); - Ok(frame.scope.get_locals().clone()) -} + fn flocals(self, _vm: &VirtualMachine) -> PyDictRef { + self.scope.get_locals() + } -fn frame_fcode(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(frame, Some(vm.ctx.frame_type()))]); - Ok(vm.ctx.new_code_object(get_value(frame).code.clone())) -} - -pub fn get_value(obj: &PyObjectRef) -> &Frame { - &obj.payload::().unwrap() + fn fcode(self, vm: &VirtualMachine) -> PyCodeRef { + vm.ctx.new_code_object(self.code.clone()) + } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index afee7060e..aa3348be5 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -582,7 +582,7 @@ impl PyContext { } pub fn new_scope(&self) -> Scope { - Scope::new(None, self.new_dict().into_object()) + Scope::new(None, self.new_dict()) } pub fn new_module(&self, name: &str, dict: PyDictRef) -> PyObjectRef { @@ -613,8 +613,10 @@ impl PyContext { PropertyBuilder::new(self).add_getter(f).create() } - pub fn new_code_object(&self, code: bytecode::CodeObject) -> PyObjectRef { + pub fn new_code_object(&self, code: bytecode::CodeObject) -> PyCodeRef { PyObject::new(objcode::PyCode::new(code), self.code_type(), None) + .downcast() + .unwrap() } pub fn new_function( @@ -683,7 +685,9 @@ impl PyContext { bytecode::Constant::String { ref value } => self.new_str(value.clone()), bytecode::Constant::Bytes { ref value } => self.new_bytes(value.clone()), bytecode::Constant::Boolean { ref value } => self.new_bool(value.clone()), - bytecode::Constant::Code { ref code } => self.new_code_object(*code.clone()), + bytecode::Constant::Code { ref code } => { + self.new_code_object(*code.clone()).into_object() + } bytecode::Constant::Tuple { ref elements } => { let elements = elements .iter() diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 41321c266..321d659b8 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -19,6 +19,7 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGeneratorRef; use crate::obj::objiter; @@ -231,7 +232,7 @@ impl VirtualMachine { self.ctx.object() } - pub fn get_locals(&self) -> PyObjectRef { + pub fn get_locals(&self) -> PyDictRef { self.current_scope().get_locals().clone() } @@ -371,8 +372,8 @@ impl VirtualMachine { pub fn invoke_with_locals( &self, function: PyObjectRef, - cells: PyObjectRef, - locals: PyObjectRef, + cells: PyDictRef, + locals: PyDictRef, ) -> PyResult { if let Some(PyFunction { code, @@ -395,7 +396,7 @@ impl VirtualMachine { fn fill_locals_from_args( &self, code_object: &bytecode::CodeObject, - locals: &PyObjectRef, + locals: &PyDictRef, args: PyFuncArgs, defaults: &PyObjectRef, ) -> PyResult<()> { From c5203a4934621292fb87d7e23d91b4068fd9ae51 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Tue, 26 Mar 2019 19:00:06 +0100 Subject: [PATCH 036/658] Remove .gcno files before a coverage run --- tests/.travis-runner.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/.travis-runner.sh b/tests/.travis-runner.sh index 36b9f43ac..f55526589 100755 --- a/tests/.travis-runner.sh +++ b/tests/.travis-runner.sh @@ -13,6 +13,7 @@ pip install pipenv if [ $CODE_COVERAGE = "true" ] then find . -name '*.gcda' -delete + find . -name '*.gcno' -delete export CARGO_INCREMENTAL=0 export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads" From 4cb7f026dee431e611863d7378815a98401c5b48 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 26 Mar 2019 15:38:19 +0000 Subject: [PATCH 037/658] Remove unnecessary cast to pyobject in kwarg construction. --- vm/src/vm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 321d659b8..4148184f4 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -451,11 +451,11 @@ impl VirtualMachine { // Do we support `**kwargs` ? let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { - let d = self.ctx.new_dict().into_object(); - locals.set_item(&self.ctx, kwargs_name, d.clone()); + let d = self.ctx.new_dict(); + locals.set_item(&self.ctx, kwargs_name, d.as_object().clone()); Some(d) } - bytecode::Varargs::Unnamed => Some(self.ctx.new_dict().into_object()), + bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), bytecode::Varargs::None => None, }; From c64417654700084d508458ecb4cb7d9fe4d1dbfe Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 26 Mar 2019 15:38:40 +0000 Subject: [PATCH 038/658] Modules have attributes, not items. --- vm/src/stdlib/json.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index b881b7a72..fbe7ea6b1 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -199,8 +199,7 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { de.deserialize(&mut serde_json::Deserializer::from_str(s)) .map_err(|err| { let json_decode_error = vm - .sys_module - .get_item("modules") + .get_attribute(vm.sys_module.clone(), "modules") .unwrap() .get_item("json") .unwrap() From 3ca387b509dca33d09dba944d3573b8bd0cc5f21 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 28 Mar 2019 05:55:15 +1300 Subject: [PATCH 039/658] Enhance FromArgs derive proc macro to allow positional, and positional or keyword arguments. --- derive/src/lib.rs | 55 ++++++++++++++++++++++------ tests/snippets/ints.py | 18 +++++++++- vm/src/builtins.rs | 23 ++++++++---- vm/src/function.rs | 82 +++++++++++++++++++++++++++++++++++++++--- vm/src/obj/objint.rs | 50 ++++++++++++++++---------- 5 files changed, 187 insertions(+), 41 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ad743e72f..558bf3dda 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -3,9 +3,9 @@ extern crate proc_macro; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use syn::{Data, DeriveInput, Fields}; +use syn::{Data, DeriveInput, Field, Fields}; -#[proc_macro_derive(FromArgs)] +#[proc_macro_derive(FromArgs, attributes(positional, keyword))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); @@ -13,21 +13,54 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { gen.to_string().parse().unwrap() } +enum ArgType { + Positional, + PositionalKeyword, + Keyword, +} + +fn generate_field(field: &Field) -> TokenStream2 { + let arg_type = if let Some(attr) = field.attrs.first() { + if attr.path.is_ident("positional") { + ArgType::Positional + } else if attr.path.is_ident("keyword") { + ArgType::Keyword + } else { + panic!("Unrecognised attribute") + } + } else { + ArgType::PositionalKeyword + }; + + let name = &field.ident; + match arg_type { + ArgType::Positional => { + quote! { + #name: args.take_positional(vm)?, + } + } + ArgType::PositionalKeyword => { + quote! { + #name: args.take_positional_keyword(vm, stringify!(#name))?, + } + } + ArgType::Keyword => { + quote! { + #name: args.take_keyword(vm, stringify!(#name))?, + } + } + } +} + fn impl_from_args(input: &DeriveInput) -> TokenStream2 { // FIXME: This references types using `crate` instead of `rustpython_vm` // so that it can be used in the latter. How can we support both? + // Can use extern crate self as rustpython_vm; once in stable. + // https://github.com/rust-lang/rust/issues/56409 let fields = match input.data { Data::Struct(ref data) => { match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(|field| { - let name = &field.ident; - quote! { - #name: crate::pyobject::TryFromObject::try_from_object( - vm, - args.take_keyword(stringify!(#name)).unwrap_or_else(|| vm.ctx.none()) - )?, - } - }), + Fields::Named(ref fields) => fields.named.iter().map(generate_field), Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message } } diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 2d36afcb6..1fbbc2bef 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -1,4 +1,4 @@ -from testutils import assert_raises +from testutils import assert_raises, assertRaises # int to int comparisons @@ -58,3 +58,19 @@ assert (2).__mul__(1.0) == NotImplemented assert (2).__rmul__(1.0) == NotImplemented assert (2).__truediv__(1.0) == NotImplemented assert (2).__rtruediv__(1.0) == NotImplemented + + +assert int() == 0 +assert int("101", 2) == 5 +assert int("101", base=2) == 5 +assert int(1) == 1 + +with assertRaises(TypeError): + int(base=2) + +with assertRaises(TypeError): + int(1, base=2) + +with assertRaises(TypeError): + # check that first parameter is truly positional only + int(val_options=1) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 4d9fb7989..cf0c0a3f6 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -559,11 +559,22 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +// Idea: Should we have a 'default' attribute, so we don't have to use OptionalArg's in this case +//#[derive(Debug, FromArgs)] +//pub struct PrintOptions { +// #[keyword] #[default(None)] sep: Option, +// #[keyword] #[default(None)] end: Option, +// #[keyword] #[default(flush)] flush: bool, +//} + #[derive(Debug, FromArgs)] pub struct PrintOptions { - sep: Option, - end: Option, - flush: bool, + #[keyword] + sep: OptionalArg>, + #[keyword] + end: OptionalArg>, + #[keyword] + flush: OptionalArg, } pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { @@ -573,7 +584,7 @@ pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) for object in objects { if first { first = false; - } else if let Some(ref sep) = options.sep { + } else if let OptionalArg::Present(Some(ref sep)) = options.sep { write!(stdout_lock, "{}", sep.value).unwrap(); } else { write!(stdout_lock, " ").unwrap(); @@ -582,13 +593,13 @@ pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) write!(stdout_lock, "{}", s).unwrap(); } - if let Some(end) = options.end { + if let OptionalArg::Present(Some(end)) = options.end { write!(stdout_lock, "{}", end.value).unwrap(); } else { writeln!(stdout_lock).unwrap(); } - if options.flush { + if options.flush.into_option().unwrap_or(false) { stdout_lock.flush().unwrap(); } diff --git a/vm/src/function.rs b/vm/src/function.rs index ed959d85c..49c2f02f5 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -1,12 +1,11 @@ use std::collections::HashMap; use std::ops::RangeInclusive; -use crate::obj::objtype; +use crate::obj::objtype::{isinstance, PyClassRef}; use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use self::OptionalArg::*; -use crate::obj::objtype::PyClassRef; /// The `PyFuncArgs` struct is one of the most used structs then creating /// a rust function that can be called from python. It holds both positional @@ -95,7 +94,7 @@ impl PyFuncArgs { ) -> Result, PyObjectRef> { match self.get_optional_kwarg(key) { Some(kwarg) => { - if objtype::isinstance(&kwarg, &ty) { + if isinstance(&kwarg, &ty) { Ok(Some(kwarg)) } else { let expected_ty_name = vm.to_pystr(&ty)?; @@ -118,7 +117,7 @@ impl PyFuncArgs { } } - pub fn take_keyword(&mut self, name: &str) -> Option { + fn extract_keyword(&mut self, name: &str) -> Option { // TODO: change kwarg representation so this scan isn't necessary if let Some(index) = self .kwargs @@ -131,6 +130,49 @@ impl PyFuncArgs { } } + pub fn take_positional( + &mut self, + vm: &VirtualMachine, + ) -> Result { + if let Some(arg) = self.next_positional() { + H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) + } else if let Some(default) = H::DEFAULT { + Ok(default) + } else { + Err(ArgumentError::TooFewArgs) + } + } + + pub fn take_positional_keyword( + &mut self, + vm: &VirtualMachine, + name: &str, + ) -> Result { + if let Some(arg) = self.next_positional() { + H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) + } else if let Some(arg) = self.extract_keyword(name) { + H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) + } else if let Some(default) = H::DEFAULT { + Ok(default) + } else { + Err(ArgumentError::TooFewArgs) + } + } + + pub fn take_keyword( + &mut self, + vm: &VirtualMachine, + name: &str, + ) -> Result { + if let Some(arg) = self.extract_keyword(name) { + H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) + } else if let Some(default) = H::DEFAULT { + Ok(default) + } else { + Err(ArgumentError::RequiredKeywordArgument(name.to_string())) + } + } + pub fn remaining_keyword<'a>(&'a mut self) -> impl Iterator + 'a { self.kwargs.drain(..) } @@ -164,6 +206,9 @@ impl PyFuncArgs { Err(ArgumentError::InvalidKeywordArgument(name)) => { return Err(vm.new_type_error(format!("{} is an invalid keyword argument", name))); } + Err(ArgumentError::RequiredKeywordArgument(name)) => { + return Err(vm.new_type_error(format!("Required keyqord only argument {}", name))); + } Err(ArgumentError::Exception(ex)) => { return Err(ex); } @@ -192,6 +237,8 @@ pub enum ArgumentError { TooManyArgs, /// The function doesn't accept a keyword argument with the given name. InvalidKeywordArgument(String), + /// The function require a keyword argument with the given name, but one wasn't provided + RequiredKeywordArgument(String), /// An exception was raised while binding arguments to the function /// parameters. Exception(PyObjectRef), @@ -218,6 +265,32 @@ pub trait FromArgs: Sized { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result; } +/// Handling behaviour when the argument is and isn't present, used to implement OptionalArg. +pub trait ArgHandler: Sized { + /// Default value that will be used if the argument isn't present, or None in which case the a + /// appropriate error is returned. + const DEFAULT: Option; + + /// Converts the arg argument when it is present + fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult; +} + +impl ArgHandler for OptionalArg { + const DEFAULT: Option = Some(Missing); + + fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { + T::try_from_object(vm, object).map(|x| Present(x)) + } +} + +impl ArgHandler for T { + const DEFAULT: Option = None; + + fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { + T::try_from_object(vm, object) + } +} + /// A map of keyword arguments to their values. /// /// A built-in function with a `KwArgs` parameter is analagous to a Python @@ -308,6 +381,7 @@ where /// An argument that may or may not be provided by the caller. /// /// This style of argument is not possible in pure Python. +#[derive(Debug)] pub enum OptionalArg { Present(T), Missing, diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index dd40f06a4..82394d691 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -6,7 +6,7 @@ use num_integer::Integer; use num_traits::{Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::OptionalArg; use crate::pyobject::{ IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; @@ -380,25 +380,37 @@ impl PyIntRef { } } -fn int_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(cls, None)], - optional = [(val_option, None)] - ); +#[derive(FromArgs)] +struct IntOptions { + #[positional] + val_options: OptionalArg, + base: OptionalArg, +} - let base = match args.get_optional_kwarg("base") { - Some(argument) => get_value(&argument).to_u32().unwrap(), - None => 10, - }; - let val = match val_option { - Some(val) => to_int(vm, val, base)?, - None => Zero::zero(), - }; - Ok(PyInt::new(val) - .into_ref_with_type(vm, cls.clone().downcast().unwrap())? - .into_object()) +impl IntOptions { + fn get_int_value(self, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(val) = self.val_options { + let base = if let OptionalArg::Present(base) = self.base { + if !objtype::isinstance(&val, &vm.ctx.str_type) { + return Err(vm.new_type_error( + "int() can't convert non-string with explicit base".to_string(), + )); + } + base + } else { + 10 + }; + to_int(vm, &val, base) + } else if let OptionalArg::Present(_) = self.base { + Err(vm.new_type_error("int() missing string argument".to_string())) + } else { + Ok(Zero::zero()) + } + } +} + +fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResult { + PyInt::new(options.get_int_value(vm)?).into_ref_with_type(vm, cls) } // Casting function: From 40dd75d5133626216bf24ea94c1075aa29edda7a Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Tue, 26 Mar 2019 20:22:33 +0100 Subject: [PATCH 040/658] Upgrade lalrpop --- Cargo.lock | 160 ++++++++++++++++++-------------------------- parser/Cargo.toml | 4 +- parser/build.rs | 5 +- parser/src/token.rs | 2 +- 4 files changed, 71 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6193772c4..79ba85f2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,11 +25,6 @@ dependencies = [ "scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "arrayref" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "arrayvec" version = "0.4.7" @@ -46,16 +41,6 @@ dependencies = [ "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "atty" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "atty" version = "0.2.10" @@ -89,15 +74,15 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bit-vec" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -116,16 +101,26 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.3.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-padding" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "byte-tools" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -186,10 +181,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "digest" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -204,14 +199,14 @@ dependencies = [ [[package]] name = "docopt" -version = "0.8.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -221,8 +216,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ena" -version = "0.5.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "env_logger" @@ -299,7 +297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "generic-array" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -323,7 +321,7 @@ dependencies = [ [[package]] name = "itertools" -version = "0.7.8" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -353,45 +351,23 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.15.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lalrpop-snap" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -399,7 +375,7 @@ dependencies = [ [[package]] name = "lalrpop-util" -version = "0.15.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -529,6 +505,11 @@ name = "num-traits" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "opaque-debug" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ordermap" version = "0.3.5" @@ -685,11 +666,6 @@ dependencies = [ "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "regex-syntax" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "regex-syntax" version = "0.5.6" @@ -754,8 +730,8 @@ dependencies = [ name = "rustpython_parser" version = "0.0.1" dependencies = [ - "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -872,13 +848,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -943,11 +919,6 @@ name = "string_cache_shared" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "strsim" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "strsim" version = "0.7.0" @@ -1285,19 +1256,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" -"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b385d69402821a1c254533a011a312531cbcc0e3e24f19bbb4747a5a2daf37e2" -"checksum atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0fd4c0631f06448cc45a6bbb3b710ebb7ff8ccb96a0800c994afe23a70d5df2" "checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" -"checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" -"checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" +"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" +"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" -"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" -"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" +"checksum block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49665c62e0e700857531fa5d3763e91b539ff1abeebd56808d378b495870d60d" +"checksum block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d75255892aeb580d3c566f213a2b6fdc1c66667839f45719ee1d30ebf2aea591" +"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" "checksum caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" @@ -1306,11 +1276,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" +"checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" "checksum dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "88972de891f6118092b643d85a0b28e0678e0f948d7f879aa32f2d5aafe97d2a" -"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" +"checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" -"checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5" +"checksum ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f56c93cc076508c549d9bb747f79aa9b4eb098be7b8cad8830c3137ef52d1e00" "checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" "checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e" "checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9" @@ -1320,16 +1290,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" +"checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606" "checksum js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58cfec35fd4a94f3cf357d5cb7da71c71cd52720c2f2a7320090a8db5f06f655" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf" -"checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f" -"checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c" +"checksum lalrpop 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4e2e80bee40b22bca46665b4ef1f3cd88ed0fb043c971407eac17a0712c02572" +"checksum lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)" = "33b27d8490dbe1f9704b0088d61e8d46edc10d5673a8829836c6ded26a9912c7" "checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" "checksum lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4fac65df7e751b57bb3a334c346239cb4ce2601907d698726ceeb82a54ba4ef" "checksum lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "025babf624c0c2b4bed1373efd684d5d0b2eecd61138d26ec3eec77bf0f2e33d" @@ -1346,6 +1315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" "checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +"checksum opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "93f5bb2e8e8dec81642920ccff6b61f1eb94fa3020c5a325c9851ff604152409" "checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" "checksum phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "05a079dd052e7b674d21cb31cbb6c05efd56a2cd2827db7692e2f1a507ebd998" @@ -1365,7 +1335,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "214a97e49be64fd2c86f568dd0cb2c757d2cc53de95b273b6ad0a1c908482f26" "checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" "checksum regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3d8c9f33201f46669484bacc312b00e7541bed6aaf296dffe2bb4e0ac6b8ce2a" -"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" "checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" @@ -1379,7 +1348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" "checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" "checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae" -"checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" +"checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" "checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" "checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6" @@ -1388,7 +1357,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423" "checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191" "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" "checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index f2f8e5a6b..e3c4e36bd 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -6,10 +6,10 @@ build = "build.rs" edition = "2018" [build-dependencies] -lalrpop="0.15.1" +lalrpop="0.16.3" [dependencies] -lalrpop-util="0.15.1" +lalrpop-util="0.16.3" log="0.4.1" regex="0.2.2" num-bigint = "0.2" diff --git a/parser/build.rs b/parser/build.rs index d35ace0c0..65147ed0e 100644 --- a/parser/build.rs +++ b/parser/build.rs @@ -1,5 +1,8 @@ use lalrpop; fn main() { - lalrpop::process_root().unwrap(); + lalrpop::Configuration::new() + .generate_in_source_tree() + .process() + .unwrap(); } diff --git a/parser/src/token.rs b/parser/src/token.rs index ebe5050fe..d2494a1d9 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -3,7 +3,7 @@ use num_bigint::BigInt; /// Python source code can be tokenized in a sequence of these tokens. -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Tok { Name { name: String }, Int { value: BigInt }, From 89f63e4c9a62409cee511d8763ad5e6985b79e42 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Wed, 27 Mar 2019 18:58:30 +0100 Subject: [PATCH 041/658] Convert objobject.rs to new args style --- vm/src/obj/objfunction.rs | 18 ++++---- vm/src/obj/objobject.rs | 95 ++++++++++----------------------------- 2 files changed, 31 insertions(+), 82 deletions(-) diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 507d7697a..410794696 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,5 +1,4 @@ use crate::frame::Scope; -use crate::function::PyFuncArgs; use crate::obj::objcode::PyCodeRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; @@ -69,16 +68,15 @@ pub fn init(context: &PyContext) { }); } -fn bind_method(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(function, None), (obj, None), (cls, None)] - ); - +fn bind_method( + function: PyObjectRef, + obj: PyObjectRef, + cls: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { if obj.is(&vm.get_none()) && !cls.is(&obj.class()) { - Ok(function.clone()) + Ok(function) } else { - Ok(vm.ctx.new_bound_method(function.clone(), obj.clone())) + Ok(vm.ctx.new_bound_method(function, obj)) } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 74ba2dc3d..c43e31a96 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,6 +1,6 @@ use super::objdict::{self, PyDictRef}; use super::objlist::PyList; -use super::objstr::{self, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; @@ -31,69 +31,31 @@ pub fn new_instance(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { Ok(PyObject::new(PyInstance, cls, dict)) } -fn object_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - Ok(vm.ctx.not_implemented()) +fn object_eq(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_ne(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_ne(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_lt(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_le(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_gt(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_zelf, Some(vm.ctx.object()))]); - - // For now default to non hashable +fn object_hash(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } @@ -147,15 +109,12 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } } -fn object_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.object()))]); - vm.call_method(zelf, "__repr__", vec![]) +fn object_str(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_method(&zelf, "__repr__", vec![]) } -fn object_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.object()))]); - let address = obj.get_id(); - Ok(vm.new_str(format!("<{} object at 0x{:x}>", obj.class().name, address))) +fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { + format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id()) } pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { @@ -235,16 +194,8 @@ fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult } } -fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (obj, Some(vm.ctx.object())), - (name_str, Some(vm.ctx.str_type())) - ] - ); - let name = objstr::get_value(&name_str); +fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { + let name = &name_str.value; trace!("object.__getattribute__({:?}, {:?})", obj, name); let cls = obj.class(); @@ -252,7 +203,7 @@ fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let attr_class = attr.class(); if objtype::class_has_attr(&attr_class, "__set__") { if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") { - return vm.invoke(descriptor, vec![attr, obj.clone(), cls.into_object()]); + return vm.invoke(descriptor, vec![attr, obj, cls.into_object()]); } } } @@ -260,9 +211,9 @@ fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(obj_attr) = object_getattr(&obj, &name) { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { - vm.call_get_descriptor(attr, obj.clone()) + vm.call_get_descriptor(attr, obj) } else if let Some(getter) = objtype::class_get_attr(&cls, "__getattr__") { - vm.invoke(getter, vec![obj.clone(), name_str.clone()]) + vm.invoke(getter, vec![obj, name_str.into_object()]) } else { Err(vm.new_attribute_error(format!("{} has no attribute '{}'", obj, name))) } From 57fa041d08a99007bf947524332fa7cd02f59cea Mon Sep 17 00:00:00 2001 From: Joey Date: Wed, 27 Mar 2019 18:05:33 -0700 Subject: [PATCH 042/658] bytearray: convert to new args style --- vm/src/obj/objbytearray.rs | 228 ++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 127 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 087c99dbf..c609fdb2f 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -6,12 +6,12 @@ use std::ops::{Deref, DerefMut}; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; -use super::objtype::{self, PyClassRef}; +use super::objtype::PyClassRef; #[derive(Debug)] pub struct PyByteArray { @@ -63,22 +63,22 @@ pub fn init(context: &PyContext) { extend_class!(context, bytearray_type, { "__doc__" => context.new_str(bytearray_doc.to_string()), - "__eq__" => context.new_rustfunc(bytearray_eq), - "__len__" => context.new_rustfunc(bytesarray_len), "__new__" => context.new_rustfunc(bytearray_new), - "__repr__" => context.new_rustfunc(bytearray_repr), - "clear" => context.new_rustfunc(bytearray_clear), - "isalnum" => context.new_rustfunc(bytearray_isalnum), - "isalpha" => context.new_rustfunc(bytearray_isalpha), - "isascii" => context.new_rustfunc(bytearray_isascii), - "isdigit" => context.new_rustfunc(bytearray_isdigit), - "islower" => context.new_rustfunc(bytearray_islower), - "isspace" => context.new_rustfunc(bytearray_isspace), - "istitle" =>context.new_rustfunc(bytearray_istitle), - "isupper" => context.new_rustfunc(bytearray_isupper), - "lower" => context.new_rustfunc(bytearray_lower), - "pop" => context.new_rustfunc(bytearray_pop), - "upper" => context.new_rustfunc(bytearray_upper) + "__eq__" => context.new_rustfunc(PyByteArrayRef::eq), + "__len__" => context.new_rustfunc(PyByteArrayRef::len), + "__repr__" => context.new_rustfunc(PyByteArrayRef::repr), + "clear" => context.new_rustfunc(PyByteArrayRef::clear), + "isalnum" => context.new_rustfunc(PyByteArrayRef::isalnum), + "isalpha" => context.new_rustfunc(PyByteArrayRef::isalpha), + "isascii" => context.new_rustfunc(PyByteArrayRef::isascii), + "isdigit" => context.new_rustfunc(PyByteArrayRef::isdigit), + "islower" => context.new_rustfunc(PyByteArrayRef::islower), + "isspace" => context.new_rustfunc(PyByteArrayRef::isspace), + "istitle" =>context.new_rustfunc(PyByteArrayRef::istitle), + "isupper" => context.new_rustfunc(PyByteArrayRef::isupper), + "lower" => context.new_rustfunc(PyByteArrayRef::lower), + "pop" => context.new_rustfunc(PyByteArrayRef::pop), + "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); } @@ -107,89 +107,69 @@ fn bytearray_new( PyByteArray::new(value).into_ref_with_type(vm, cls.clone()) } -fn bytesarray_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytearray_type()))]); +impl PyByteArrayRef { + fn len(self, _vm: &VirtualMachine) -> usize { + self.value.borrow().len() + } - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx + .new_bool(self.value.borrow().as_slice() == other.value.borrow().as_slice()) + } else { + vm.ctx.not_implemented() + } + } -fn bytearray_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytearray_type())), (b, None)] - ); + fn isalnum(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphanumeric()) + } - let result = if objtype::isinstance(b, &vm.ctx.bytearray_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + fn isalpha(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphabetic()) + } -fn bytearray_isalnum(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphanumeric()))) -} + fn isascii(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_ascii()) + } -fn bytearray_isalpha(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphabetic()))) -} + fn isdigit(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_digit(10)) + } -fn bytearray_isascii(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_ascii()))) -} - -fn bytearray_isdigit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_digit(10)))) -} - -fn bytearray_islower(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool( + fn islower(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); !bytes.is_empty() && bytes .iter() .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_lowercase()), - )) -} + .all(|x| char::from(*x).is_lowercase()) + } -fn bytearray_isspace(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_whitespace()))) -} + fn isspace(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_whitespace()) + } -fn bytearray_isupper(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool( + fn isupper(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); !bytes.is_empty() && bytes .iter() .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_uppercase()), - )) -} + .all(|x| char::from(*x).is_uppercase()) + } -fn bytearray_istitle(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); + fn istitle(self, _vm: &VirtualMachine) -> bool { + let bytes = self.value.borrow(); + if bytes.is_empty() { + return false; + } - if bytes.is_empty() { - Ok(vm.new_bool(false)) - } else { let mut iter = bytes.iter().peekable(); let mut prev_cased = false; @@ -198,21 +178,52 @@ fn bytearray_istitle(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let next = if let Some(k) = iter.peek() { char::from(**k) } else if current.is_uppercase() { - return Ok(vm.new_bool(!prev_cased)); + return !prev_cased; } else { - return Ok(vm.new_bool(prev_cased)); + return prev_cased; }; if (is_cased(current) && next.is_uppercase() && !prev_cased) || (!is_cased(current) && next.is_lowercase()) { - return Ok(vm.new_bool(false)); + return false; } prev_cased = is_cased(current); } - Ok(vm.new_bool(true)) + true + } + + fn repr(self, _vm: &VirtualMachine) -> String { + let bytes = self.value.borrow(); + let data = String::from_utf8(bytes.to_vec()).unwrap_or_else(|_| to_hex(&bytes.to_vec())); + format!("bytearray(b'{}')", data) + } + + fn clear(self, _vm: &VirtualMachine) { + self.value.borrow_mut().clear(); + } + + fn pop(self, vm: &VirtualMachine) -> PyResult { + let mut bytes = self.value.borrow_mut(); + bytes + .pop() + .ok_or_else(|| vm.new_index_error("pop from empty bytearray".to_string())) + } + + fn lower(self, _vm: &VirtualMachine) -> PyByteArray { + let bytes = self.value.borrow().clone().to_ascii_lowercase(); + PyByteArray { + value: RefCell::new(bytes), + } + } + + fn upper(self, _vm: &VirtualMachine) -> PyByteArray { + let bytes = self.value.borrow().clone().to_ascii_uppercase(); + PyByteArray { + value: RefCell::new(bytes), + } } } @@ -222,7 +233,7 @@ fn is_cased(c: char) -> bool { } /* -fn bytearray_getitem(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { +fn getitem(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, @@ -239,56 +250,19 @@ fn set_value(obj: &PyObjectRef, value: Vec) { */ /// Return a lowercase hex representation of a bytearray -fn bytearray_to_hex(bytearray: &[u8]) -> String { +fn to_hex(bytearray: &[u8]) -> String { bytearray.iter().fold(String::new(), |mut s, b| { let _ = write!(s, "\\x{:02x}", b); s }) } -fn bytearray_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj); - let data = - String::from_utf8(value.to_vec()).unwrap_or_else(|_| bytearray_to_hex(&value.to_vec())); - Ok(vm.new_str(format!("bytearray(b'{}')", data))) -} - -fn bytearray_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - get_mut_value(zelf).clear(); - Ok(vm.get_none()) -} - -fn bytearray_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let mut value = get_mut_value(obj); - - if let Some(i) = value.pop() { - Ok(vm.ctx.new_int(i)) - } else { - Err(vm.new_index_error("pop from empty bytearray".to_string())) - } -} - -fn bytearray_lower(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj).to_vec().to_ascii_lowercase(); - Ok(vm.ctx.new_bytearray(value)) -} - -fn bytearray_upper(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj).to_vec().to_ascii_uppercase(); - Ok(vm.ctx.new_bytearray(value)) -} - #[cfg(test)] mod tests { use super::*; #[test] fn bytearray_to_hex_formatting() { - assert_eq!(&bytearray_to_hex(&[11u8, 222u8]), "\\x0b\\xde"); + assert_eq!(&to_hex(&[11u8, 222u8]), "\\x0b\\xde"); } } From 62842e6d194dca4ade3383c99be0403a8453542a Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:00:34 -0700 Subject: [PATCH 043/658] convert some builtins --- tests/snippets/builtin_format.py | 5 ++ vm/src/builtins.rs | 125 +++++++++++++------------------ vm/src/function.rs | 2 +- vm/src/vm.rs | 5 +- 4 files changed, 62 insertions(+), 75 deletions(-) diff --git a/tests/snippets/builtin_format.py b/tests/snippets/builtin_format.py index 6c06cbd98..55a6a3da1 100644 --- a/tests/snippets/builtin_format.py +++ b/tests/snippets/builtin_format.py @@ -7,3 +7,8 @@ assert_raises(TypeError, lambda: format(2, 3), 'format called with number') assert format({}) == "{}" assert_raises(TypeError, lambda: format({}, 'b'), 'format_spec not empty for dict') + +class BadFormat: + def __format__(self, spec): + return 42 +assert_raises(TypeError, lambda: format(BadFormat())) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 1de24b86a..9d5980759 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -6,21 +6,22 @@ use std::char; use std::io::{self, Write}; use std::path::PathBuf; -use num_traits::{Signed, ToPrimitive}; +use num_traits::Signed; use crate::compile; use crate::import::import_module; use crate::obj::objbool; use crate::obj::objdict::PyDictRef; -use crate::obj::objint; +use crate::obj::objint::{self, PyIntRef}; use crate::obj::objiter; -use crate::obj::objstr::{self, PyStringRef}; +use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TryFromObject, TypeProtocol, + DictProtocol, IdProtocol, PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; @@ -28,74 +29,53 @@ use crate::obj::objcode::PyCodeRef; #[cfg(not(target_arch = "wasm32"))] use crate::stdlib::io::io_open; -fn builtin_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(x, None)]); +fn builtin_abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { match vm.get_method(x.clone(), "__abs__") { Ok(attrib) => vm.invoke(attrib, PyFuncArgs::new(vec![], vec![])), Err(..) => Err(vm.new_type_error("bad operand for abs".to_string())), } } -fn builtin_all(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let items = vm.extract_elements(iterable)?; - for item in items { - let result = objbool::boolval(vm, item)?; - if !result { - return Ok(vm.new_bool(false)); +fn builtin_all(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + if !item? { + return Ok(false); } } - Ok(vm.new_bool(true)) + Ok(true) } -fn builtin_any(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let iterator = objiter::get_iter(vm, iterable)?; - - while let Some(item) = objiter::get_next_object(vm, &iterator)? { - let result = objbool::boolval(vm, item)?; - if result { - return Ok(vm.new_bool(true)); +fn builtin_any(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + if item? { + return Ok(true); } } - - Ok(vm.new_bool(false)) + Ok(false) } // builtin_ascii -fn builtin_bin(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(number, Some(vm.ctx.int_type()))]); - - let n = objint::get_value(number); - let s = if n.is_negative() { - format!("-0b{:b}", n.abs()) +fn builtin_bin(x: PyIntRef, _vm: &VirtualMachine) -> String { + let x = x.as_bigint(); + if x.is_negative() { + format!("-0b{:b}", x.abs()) } else { - format!("0b{:b}", n) - }; - - Ok(vm.new_str(s)) + format!("0b{:b}", x) + } } // builtin_breakpoint -fn builtin_callable(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, None)]); - let is_callable = objtype::class_has_attr(&obj.class(), "__call__"); - Ok(vm.new_bool(is_callable)) +fn builtin_callable(obj: PyObjectRef, _vm: &VirtualMachine) -> bool { + objtype::class_has_attr(&obj.class(), "__call__") } -fn builtin_chr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.int_type()))]); - - let code_point = objint::get_value(i).to_u32().unwrap(); - - let txt = match char::from_u32(code_point) { +fn builtin_chr(i: u32, _vm: &VirtualMachine) -> String { + match char::from_u32(i) { Some(value) => value.to_string(), None => '_'.to_string(), - }; - - Ok(vm.new_str(txt)) + } } fn builtin_compile( @@ -128,13 +108,8 @@ fn builtin_compile( }) } -fn builtin_delattr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, None), (attr, Some(vm.ctx.str_type()))] - ); - vm.del_attr(obj, attr.clone()) +fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + vm.del_attr(&obj, attr.into_object()) } fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -258,17 +233,26 @@ fn make_scope( Ok(Scope::new(locals, globals)) } -fn builtin_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, None)], - optional = [(format_spec, Some(vm.ctx.str_type()))] - ); - let format_spec = format_spec - .cloned() - .unwrap_or_else(|| vm.new_str("".to_string())); - vm.call_method(obj, "__format__", vec![format_spec]) +fn builtin_format( + value: PyObjectRef, + format_spec: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let format_spec = format_spec.into_option().unwrap_or_else(|| { + PyString { + value: "".to_string(), + } + .into_ref(vm) + }); + + vm.call_method(&value, "__format__", vec![format_spec.into_object()])? + .downcast() + .map_err(|obj| { + vm.new_type_error(format!( + "__format__ must return a str, not {}", + obj.class().name + )) + }) } fn catch_attr_exception(ex: PyObjectRef, default: T, vm: &VirtualMachine) -> PyResult { @@ -644,14 +628,11 @@ fn builtin_sorted(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { Ok(lst) } -fn builtin_sum(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let items = vm.extract_elements(iterable)?; - +fn builtin_sum(iterable: PyIterable, start: OptionalArg, vm: &VirtualMachine) -> PyResult { // Start with zero and add at will: - let mut sum = vm.ctx.new_int(0); - for item in items { - sum = vm._add(sum, item)?; + let mut sum = start.into_option().unwrap_or_else(|| vm.ctx.new_int(0)); + for item in iterable.iter(vm)? { + sum = vm._add(sum, item?)?; } Ok(sum) } diff --git a/vm/src/function.rs b/vm/src/function.rs index 1c61ffc87..a567e7903 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -308,7 +308,7 @@ where /// An argument that may or may not be provided by the caller. /// /// This style of argument is not possible in pure Python. -pub enum OptionalArg { +pub enum OptionalArg { Present(T), Missing, } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 4148184f4..36f4450e6 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -577,8 +577,9 @@ impl VirtualMachine { self.call_method(&obj, "__setattr__", vec![attr_name, attr_value]) } - pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult { - self.call_method(&obj, "__delattr__", vec![attr_name]) + pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult<()> { + self.call_method(&obj, "__delattr__", vec![attr_name])?; + Ok(()) } // get_method should be used for internal access to magic methods (by-passing From b0d7960cc59ed1a9a1a8f794818349024eac3f4f Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:14:37 -0700 Subject: [PATCH 044/658] bytes: move methods to impl block --- vm/src/obj/objbytes.rs | 202 +++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 100 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8d64cc507..5b725be50 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -57,16 +57,16 @@ pub fn init(context: &PyContext) { - an integer"; extend_class!(context, bytes_type, { - "__eq__" => context.new_rustfunc(bytes_eq), - "__lt__" => context.new_rustfunc(bytes_lt), - "__le__" => context.new_rustfunc(bytes_le), - "__gt__" => context.new_rustfunc(bytes_gt), - "__ge__" => context.new_rustfunc(bytes_ge), - "__hash__" => context.new_rustfunc(bytes_hash), "__new__" => context.new_rustfunc(bytes_new), - "__repr__" => context.new_rustfunc(bytes_repr), - "__len__" => context.new_rustfunc(bytes_len), - "__iter__" => context.new_rustfunc(bytes_iter), + "__eq__" => context.new_rustfunc(PyBytesRef::eq), + "__lt__" => context.new_rustfunc(PyBytesRef::lt), + "__le__" => context.new_rustfunc(PyBytesRef::le), + "__gt__" => context.new_rustfunc(PyBytesRef::gt), + "__ge__" => context.new_rustfunc(PyBytesRef::ge), + "__hash__" => context.new_rustfunc(PyBytesRef::hash), + "__repr__" => context.new_rustfunc(PyBytesRef::repr), + "__len__" => context.new_rustfunc(PyBytesRef::len), + "__iter__" => context.new_rustfunc(PyBytesRef::iter), "__doc__" => context.new_str(bytes_doc.to_string()) }); } @@ -93,111 +93,113 @@ fn bytes_new( PyBytes::new(value).into_ref_with_type(vm, cls) } -fn bytes_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); +impl PyBytesRef { + fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() == get_value(b).to_vec() + } else { + false + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() >= get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() > get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() <= get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() < get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); + fn len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + let byte_vec = get_value(a).to_vec(); + Ok(vm.ctx.new_int(byte_vec.len())) + } -fn bytes_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) + fn hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); + let data = get_value(zelf); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + let hash = hasher.finish(); + Ok(vm.ctx.new_int(hash)) + } + + fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); + let value = get_value(obj); + let data = String::from_utf8(value.to_vec()).unwrap(); + Ok(vm.new_str(format!("b'{}'", data))) + } + + fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { + PyIteratorValue { + position: Cell::new(0), + iterated_obj: obj.into_object(), + } + } } pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().value } - -fn bytes_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) -} - -fn bytes_iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { - position: Cell::new(0), - iterated_obj: obj.into_object(), - } -} From e4272126cf5589ecd24a6a89a77daba364968c49 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:15:38 -0700 Subject: [PATCH 045/658] bytes: return NotImplemented where appropriate --- vm/src/obj/objbytes.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 5b725be50..bed57157e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -104,7 +104,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() == get_value(b).to_vec() } else { - false + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -119,7 +119,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() >= get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -134,7 +134,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() > get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -149,7 +149,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() <= get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -164,7 +164,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() < get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } From e0aca86473738ea8cf592fce5b7e4769293b6227 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:17:27 -0700 Subject: [PATCH 046/658] bytes: convert methods to new args style --- vm/src/obj/objbytes.rs | 121 +++++++++++++---------------------------- 1 file changed, 39 insertions(+), 82 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index bed57157e..9390e7070 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,17 +1,16 @@ use std::cell::Cell; +use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::ops::Deref; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; -use super::objtype::{self, PyClassRef}; +use super::objtype::PyClassRef; #[derive(Debug)] pub struct PyBytes { @@ -94,102 +93,60 @@ fn bytes_new( } impl PyBytesRef { - fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value == other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value >= other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value > other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value <= other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value < other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); - - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) + fn len(self, _vm: &VirtualMachine) -> usize { + self.value.len() } - fn hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) + fn hash(self, _vm: &VirtualMachine) -> u64 { + let mut hasher = DefaultHasher::new(); + self.value.hash(&mut hasher); + hasher.finish() } - fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) + fn repr(self, _vm: &VirtualMachine) -> String { + // TODO: don't just unwrap + let data = String::from_utf8(self.value.clone()).unwrap(); + format!("b'{}'", data) } fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { From 657d025592dade327e485b744c41a80d1f673059 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 28 Mar 2019 21:34:11 +1300 Subject: [PATCH 047/658] Change syntax of attributes in FromArgs proc macro --- derive/src/lib.rs | 161 ++++++++++++++++++++++++++++++++++++++----- vm/src/builtins.rs | 26 +++---- vm/src/function.rs | 83 +++------------------- vm/src/obj/objint.rs | 3 +- 4 files changed, 164 insertions(+), 109 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 558bf3dda..f1b9ead08 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -3,9 +3,9 @@ extern crate proc_macro; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use syn::{Data, DeriveInput, Field, Fields}; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; -#[proc_macro_derive(FromArgs, attributes(positional, keyword))] +#[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); @@ -13,40 +13,167 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { gen.to_string().parse().unwrap() } -enum ArgType { +enum ArgKind { Positional, PositionalKeyword, Keyword, } -fn generate_field(field: &Field) -> TokenStream2 { - let arg_type = if let Some(attr) = field.attrs.first() { - if attr.path.is_ident("positional") { - ArgType::Positional - } else if attr.path.is_ident("keyword") { - ArgType::Keyword +impl ArgKind { + fn from_ident(ident: &Ident) -> ArgKind { + if ident == "positional" { + ArgKind::Positional + } else if ident == "positional_keyword" { + ArgKind::PositionalKeyword + } else if ident == "keyword" { + ArgKind::Keyword } else { panic!("Unrecognised attribute") } + } +} + +struct ArgAttribute { + kind: ArgKind, + default: Option, + optional: bool, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option { + if !attr.path.is_ident("pyarg") { + return None; + } + + match attr.parse_meta().unwrap() { + Meta::List(list) => { + let mut iter = list.nested.iter(); + let first_arg = iter.next().expect("at least one argument in pyarg list"); + let kind = match first_arg { + NestedMeta::Meta(Meta::Word(ident)) => ArgKind::from_ident(ident), + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + + let mut attribute = ArgAttribute { + kind, + default: None, + optional: false, + }; + + while let Some(arg) = iter.next() { + attribute.parse_argument(arg); + } + + assert!( + attribute.default.is_none() || !attribute.optional, + "Can't set both a default value and optional" + ); + + Some(attribute) + } + _ => panic!("Bad syntax for pyarg attribute"), + } + } + + fn parse_argument(&mut self, arg: &NestedMeta) { + match arg { + NestedMeta::Meta(Meta::Word(ident)) => { + if ident == "default" { + assert!(self.default.is_none(), "Default already set"); + let expr = syn::parse_str::("Default::default()").unwrap(); + self.default = Some(expr); + } else if ident == "optional" { + self.optional = true; + } else { + panic!("Unrecognised pyarg attribute '{}'", ident); + } + } + NestedMeta::Meta(Meta::NameValue(name_value)) => { + if name_value.ident == "default" { + assert!(self.default.is_none(), "Default already set"); + + match name_value.lit { + Lit::Str(ref val) => { + let expr = val + .parse::() + .expect("a valid expression for default argument"); + self.default = Some(expr); + } + _ => panic!("Expected string value for default argument"), + } + } else if name_value.ident == "optional" { + match name_value.lit { + Lit::Bool(ref val) => { + self.optional = val.value; + } + _ => panic!("Expected boolean value for optional argument"), + } + } else { + panic!("Unrecognised pyarg attribute '{}'", name_value.ident); + } + } + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + } +} + +fn generate_field(field: &Field) -> TokenStream2 { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(ArgAttribute::from_attribute) + .collect::>(); + let attr = if pyarg_attrs.is_empty() { + ArgAttribute { + kind: ArgKind::PositionalKeyword, + default: None, + optional: false, + } + } else if pyarg_attrs.len() == 1 { + pyarg_attrs.remove(0) } else { - ArgType::PositionalKeyword + panic!( + "Multiple pyarg attributes on field '{}'", + field.ident.as_ref().unwrap() + ); }; let name = &field.ident; - match arg_type { - ArgType::Positional => { + let middle = quote! { + .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + }; + let ending = if let Some(default) = attr.default { + quote! { + .unwrap_or_else(|| #default) + } + } else { + let err = match attr.kind { + ArgKind::Positional | ArgKind::PositionalKeyword => { + quote!(crate::function::ArgumentError::TooFewArgs) + } + ArgKind::Keyword => quote!(crate::function::ArgumentError::RequiredKeywordArgument( + stringify!(#name) + )), + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + match attr.kind { + ArgKind::Positional => { quote! { - #name: args.take_positional(vm)?, + #name: args.take_positional()#middle#ending, } } - ArgType::PositionalKeyword => { + ArgKind::PositionalKeyword => { quote! { - #name: args.take_positional_keyword(vm, stringify!(#name))?, + #name: args.take_positional_keyword(stringify!(#name))#middle#ending, } } - ArgType::Keyword => { + ArgKind::Keyword => { quote! { - #name: args.take_keyword(vm, stringify!(#name))?, + #name: args.take_keyword(stringify!(#name))#middle#ending, } } } diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index cf0c0a3f6..5a238208a 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -559,22 +559,14 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -// Idea: Should we have a 'default' attribute, so we don't have to use OptionalArg's in this case -//#[derive(Debug, FromArgs)] -//pub struct PrintOptions { -// #[keyword] #[default(None)] sep: Option, -// #[keyword] #[default(None)] end: Option, -// #[keyword] #[default(flush)] flush: bool, -//} - #[derive(Debug, FromArgs)] pub struct PrintOptions { - #[keyword] - sep: OptionalArg>, - #[keyword] - end: OptionalArg>, - #[keyword] - flush: OptionalArg, + #[pyarg(keyword, default = "None")] + sep: Option, + #[pyarg(keyword, default = "None")] + end: Option, + #[pyarg(keyword, default = "false")] + flush: bool, } pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { @@ -584,7 +576,7 @@ pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) for object in objects { if first { first = false; - } else if let OptionalArg::Present(Some(ref sep)) = options.sep { + } else if let Some(ref sep) = options.sep { write!(stdout_lock, "{}", sep.value).unwrap(); } else { write!(stdout_lock, " ").unwrap(); @@ -593,13 +585,13 @@ pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) write!(stdout_lock, "{}", s).unwrap(); } - if let OptionalArg::Present(Some(end)) = options.end { + if let Some(end) = options.end { write!(stdout_lock, "{}", end.value).unwrap(); } else { writeln!(stdout_lock).unwrap(); } - if options.flush.into_option().unwrap_or(false) { + if options.flush { stdout_lock.flush().unwrap(); } diff --git a/vm/src/function.rs b/vm/src/function.rs index 49c2f02f5..e5267cf45 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -109,7 +109,7 @@ impl PyFuncArgs { } } - pub fn next_positional(&mut self) -> Option { + pub fn take_positional(&mut self) -> Option { if self.args.is_empty() { None } else { @@ -117,7 +117,11 @@ impl PyFuncArgs { } } - fn extract_keyword(&mut self, name: &str) -> Option { + pub fn take_positional_keyword(&mut self, name: &str) -> Option { + self.take_positional().or_else(|| self.take_keyword(name)) + } + + pub fn take_keyword(&mut self, name: &str) -> Option { // TODO: change kwarg representation so this scan isn't necessary if let Some(index) = self .kwargs @@ -130,49 +134,6 @@ impl PyFuncArgs { } } - pub fn take_positional( - &mut self, - vm: &VirtualMachine, - ) -> Result { - if let Some(arg) = self.next_positional() { - H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) - } else if let Some(default) = H::DEFAULT { - Ok(default) - } else { - Err(ArgumentError::TooFewArgs) - } - } - - pub fn take_positional_keyword( - &mut self, - vm: &VirtualMachine, - name: &str, - ) -> Result { - if let Some(arg) = self.next_positional() { - H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) - } else if let Some(arg) = self.extract_keyword(name) { - H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) - } else if let Some(default) = H::DEFAULT { - Ok(default) - } else { - Err(ArgumentError::TooFewArgs) - } - } - - pub fn take_keyword( - &mut self, - vm: &VirtualMachine, - name: &str, - ) -> Result { - if let Some(arg) = self.extract_keyword(name) { - H::from_arg(vm, arg).map_err(|err| ArgumentError::Exception(err)) - } else if let Some(default) = H::DEFAULT { - Ok(default) - } else { - Err(ArgumentError::RequiredKeywordArgument(name.to_string())) - } - } - pub fn remaining_keyword<'a>(&'a mut self) -> impl Iterator + 'a { self.kwargs.drain(..) } @@ -265,32 +226,6 @@ pub trait FromArgs: Sized { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result; } -/// Handling behaviour when the argument is and isn't present, used to implement OptionalArg. -pub trait ArgHandler: Sized { - /// Default value that will be used if the argument isn't present, or None in which case the a - /// appropriate error is returned. - const DEFAULT: Option; - - /// Converts the arg argument when it is present - fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult; -} - -impl ArgHandler for OptionalArg { - const DEFAULT: Option = Some(Missing); - - fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - T::try_from_object(vm, object).map(|x| Present(x)) - } -} - -impl ArgHandler for T { - const DEFAULT: Option = None; - - fn from_arg(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - T::try_from_object(vm, object) - } -} - /// A map of keyword arguments to their values. /// /// A built-in function with a `KwArgs` parameter is analagous to a Python @@ -345,7 +280,7 @@ where { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { let mut varargs = Vec::new(); - while let Some(value) = args.next_positional() { + while let Some(value) = args.take_positional() { varargs.push(T::try_from_object(vm, value)?); } Ok(Args(varargs)) @@ -370,7 +305,7 @@ where } fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { - if let Some(value) = args.next_positional() { + if let Some(value) = args.take_positional() { Ok(T::try_from_object(vm, value)?) } else { Err(ArgumentError::TooFewArgs) @@ -414,7 +349,7 @@ where } fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { - if let Some(value) = args.next_positional() { + if let Some(value) = args.take_positional() { Ok(Present(T::try_from_object(vm, value)?)) } else { Ok(Missing) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 82394d691..2470ea5f9 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -382,8 +382,9 @@ impl PyIntRef { #[derive(FromArgs)] struct IntOptions { - #[positional] + #[pyarg(positional, optional = true)] val_options: OptionalArg, + #[pyarg(positional_keyword, optional = true)] base: OptionalArg, } From c8f4474e7f0e31388384b40f54c508c53bb41079 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 28 Mar 2019 21:46:26 +1300 Subject: [PATCH 048/658] Implement handling of optional attribute in proc macro --- derive/src/lib.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index f1b9ead08..ad8182945 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -146,14 +146,19 @@ fn generate_field(field: &Field) -> TokenStream2 { quote! { .unwrap_or_else(|| #default) } + } else if attr.optional { + quote! { + .map(crate::function::OptionalArg::Present) + .unwrap_or(crate::function::OptionalArg::Missing) + } } else { let err = match attr.kind { - ArgKind::Positional | ArgKind::PositionalKeyword => { - quote!(crate::function::ArgumentError::TooFewArgs) - } - ArgKind::Keyword => quote!(crate::function::ArgumentError::RequiredKeywordArgument( - stringify!(#name) - )), + ArgKind::Positional | ArgKind::PositionalKeyword => quote! { + crate::function::ArgumentError::TooFewArgs + }, + ArgKind::Keyword => quote! { + crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + }, }; quote! { .ok_or_else(|| #err)? From 6fd6cc647b1a804d95a41b3d006acf2d49f28e29 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 28 Mar 2019 22:00:55 +1300 Subject: [PATCH 049/658] Renamed ArgKind, and member to be consistent with python inspect module --- derive/src/lib.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ad8182945..b4048ec83 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -13,20 +13,22 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { gen.to_string().parse().unwrap() } -enum ArgKind { - Positional, - PositionalKeyword, - Keyword, +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +enum ParameterKind { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, } -impl ArgKind { - fn from_ident(ident: &Ident) -> ArgKind { +impl ParameterKind { + fn from_ident(ident: &Ident) -> ParameterKind { if ident == "positional" { - ArgKind::Positional + ParameterKind::PositionalOnly } else if ident == "positional_keyword" { - ArgKind::PositionalKeyword + ParameterKind::PositionalOrKeyword } else if ident == "keyword" { - ArgKind::Keyword + ParameterKind::KeywordOnly } else { panic!("Unrecognised attribute") } @@ -34,7 +36,7 @@ impl ArgKind { } struct ArgAttribute { - kind: ArgKind, + kind: ParameterKind, default: Option, optional: bool, } @@ -50,7 +52,7 @@ impl ArgAttribute { let mut iter = list.nested.iter(); let first_arg = iter.next().expect("at least one argument in pyarg list"); let kind = match first_arg { - NestedMeta::Meta(Meta::Word(ident)) => ArgKind::from_ident(ident), + NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), _ => panic!("Bad syntax for first pyarg attribute argument"), }; @@ -125,7 +127,7 @@ fn generate_field(field: &Field) -> TokenStream2 { .collect::>(); let attr = if pyarg_attrs.is_empty() { ArgAttribute { - kind: ArgKind::PositionalKeyword, + kind: ParameterKind::PositionalOrKeyword, default: None, optional: false, } @@ -153,10 +155,10 @@ fn generate_field(field: &Field) -> TokenStream2 { } } else { let err = match attr.kind { - ArgKind::Positional | ArgKind::PositionalKeyword => quote! { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { crate::function::ArgumentError::TooFewArgs }, - ArgKind::Keyword => quote! { + ParameterKind::KeywordOnly => quote! { crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) }, }; @@ -166,17 +168,17 @@ fn generate_field(field: &Field) -> TokenStream2 { }; match attr.kind { - ArgKind::Positional => { + ParameterKind::PositionalOnly => { quote! { #name: args.take_positional()#middle#ending, } } - ArgKind::PositionalKeyword => { + ParameterKind::PositionalOrKeyword => { quote! { #name: args.take_positional_keyword(stringify!(#name))#middle#ending, } } - ArgKind::Keyword => { + ParameterKind::KeywordOnly => { quote! { #name: args.take_keyword(stringify!(#name))#middle#ending, } From 73cd680d8d761bd6f0616f09bfab1f5e87be9d56 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 28 Mar 2019 17:50:41 +0100 Subject: [PATCH 050/658] Add support for trailing comma in function defs. Parser code simplications. --- parser/src/lexer.rs | 17 +++++++-- parser/src/python.lalrpop | 72 ++++++++++++-------------------------- tests/snippets/function.py | 5 +++ 3 files changed, 42 insertions(+), 52 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index defa5e232..edf21c973 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -466,8 +466,16 @@ where loop { match self.next_char() { Some('\\') => { - if is_raw { + if self.chr0 == Some(quote_char) { + string_content.push(quote_char); + self.next_char(); + } else if is_raw { string_content.push('\\'); + if let Some(c) = self.next_char() { + string_content.push(c) + } else { + return Err(LexicalError::StringError); + } } else { match self.next_char() { Some('\\') => { @@ -711,7 +719,6 @@ where let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); - println!("Emoji: {}", c); return Some(Ok(( tok_start, Tok::Name { @@ -1438,7 +1445,7 @@ mod tests { #[test] fn test_string() { - let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g'"#); + let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#); let tokens = lex_source(&source); assert_eq!( tokens, @@ -1467,6 +1474,10 @@ mod tests { value: String::from("\\g"), is_fstring: false, }, + Tok::String { + value: String::from("raw\'"), + is_fstring: false, + }, ] ); } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 9e5b309b1..7f010278d 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -337,10 +337,7 @@ CompoundStatement: ast::LocatedStatement = { IfStatement: ast::LocatedStatement = { "if" ":" => { // Determine last else: - let mut last = match s3 { - Some(s) => Some(s.2), - None => None, - }; + let mut last = s3.map(|s| s.2); // handle elif: for i in s2.into_iter().rev() { @@ -360,10 +357,7 @@ IfStatement: ast::LocatedStatement = { WhileStatement: ast::LocatedStatement = { "while" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::While { test: e, body: s, orelse: or_else }, @@ -373,10 +367,7 @@ WhileStatement: ast::LocatedStatement = { ForStatement: ast::LocatedStatement = { "for" "in" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::For { @@ -389,14 +380,8 @@ ForStatement: ast::LocatedStatement = { TryStatement: ast::LocatedStatement = { "try" ":" => { - let or_else = match else_suite { - Some(s) => Some(s.2), - None => None, - }; - let finalbody = match finally { - Some(s) => Some(s.2), - None => None, - }; + let or_else = else_suite.map(|s| s.2); + let finalbody = finally.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::Try { @@ -437,10 +422,7 @@ WithStatement: ast::LocatedStatement = { WithItem: ast::WithItem = { => { - let optional_vars = match n { - Some(val) => Some(val.1), - None => None, - }; + let optional_vars = n.map(|val| val.1); ast::WithItem { context_expr: t, optional_vars } }, }; @@ -461,27 +443,19 @@ FuncDef: ast::LocatedStatement = { }; Parameters: ast::Parameters = { - "(" )?> ")" => { - match a { - Some(a) => a, - None => Default::default(), - } + "(" )?> ")" => { + a.unwrap_or_else(Default::default) }, }; -// parameters are (String, None), kwargs are (String, Some(Test)) where Test is -// the default // Note that this is a macro which is used once for function defs, and // once for lambda defs. -TypedArgsList: ast::Parameters = { - > )?> => { +ParameterList: ast::Parameters = { + > )?> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: - let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 { - Some((_, x)) => x, - None => (None, vec![], vec![], None), - }; + let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); ast::Parameters { args: names, @@ -492,7 +466,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > )> => { + > )> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: @@ -510,7 +484,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { let (vararg, kwonlyargs, kw_defaults, kwarg) = params; ast::Parameters { args: vec![], @@ -521,7 +495,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { ast::Parameters { args: vec![], kwonlyargs: vec![], @@ -535,8 +509,8 @@ TypedArgsList: ast::Parameters = { // Use inline here to make sure the "," is not creating an ambiguity. #[inline] -TypedParameters: (Vec, Vec) = { - > )*> => { +ParameterDefs: (Vec, Vec) = { + > )*> => { // Combine first parameters: let mut args = vec![param1]; args.extend(param2.into_iter().map(|x| x.1)); @@ -564,7 +538,7 @@ TypedParameters: (Vec, Vec) = { } }; -TypedParameterDef: (ast::Parameter, Option) = { +ParameterDef: (ast::Parameter, Option) = { => (i, None), "=" => (i, Some(e)), }; @@ -580,8 +554,11 @@ TypedParameter: ast::Parameter = { }, }; +// Use inline here to make sure the "," is not creating an ambiguity. +// TODO: figure out another grammar that makes this inline no longer required. +#[inline] ParameterListStarArgs: (Option>, Vec, Vec>, Option>) = { - "*" )*> )?> => { + "*" )*> )?> => { // Extract keyword arguments: let mut kwonlyargs = vec![]; let mut kw_defaults = vec![]; @@ -590,10 +567,7 @@ ParameterListStarArgs: (Option>, Vec Some(name), - None => None, - }; + let kwarg = kwarg.map(|n| n.1); (Some(va), kwonlyargs, kw_defaults, kwarg) } @@ -684,7 +658,7 @@ Test: ast::Expression = { }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), body:Box::new(b) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index a69178bb7..b1ea96860 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -2,3 +2,8 @@ def foo(): return 42 assert foo() == 42 + +def my_func(a,): + return a+2 + +assert my_func(2) == 4 From 792c6a103e38a35cd21a36ba063d86d26a0f347c Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Wed, 27 Mar 2019 18:56:42 +0100 Subject: [PATCH 051/658] Convert re to new style args --- vm/src/stdlib/re.rs | 208 +++++++++++++++----------------------------- 1 file changed, 68 insertions(+), 140 deletions(-) diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index 4dab1b0c3..fee100bfe 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -6,11 +6,9 @@ */ use regex::{Match, Regex}; -use crate::function::PyFuncArgs; -use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; impl PyValue for Regex { @@ -19,85 +17,6 @@ impl PyValue for Regex { } } -/// Create the python `re` module with all its members. -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let match_type = py_class!(ctx, "Match", ctx.object(), { - "start" => ctx.new_rustfunc(match_start), - "end" => ctx.new_rustfunc(match_end) - }); - - let pattern_type = py_class!(ctx, "Pattern", ctx.object(), { - "match" => ctx.new_rustfunc(pattern_match), - "search" => ctx.new_rustfunc(pattern_search) - }); - - py_module!(ctx, "re", { - "compile" => ctx.new_rustfunc(re_compile), - "Match" => match_type, - "match" => ctx.new_rustfunc(re_match), - "Pattern" => pattern_type, - "search" => ctx.new_rustfunc(re_search) - }) -} - -/// Implement re.match -/// See also: -/// https://docs.python.org/3/library/re.html#re.match -fn re_match(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (pattern, Some(vm.ctx.str_type())), - (string, Some(vm.ctx.str_type())) - ] - ); - let pattern_str = objstr::get_value(&pattern); - let regex = make_regex(vm, &pattern_str)?; - let search_text = objstr::get_value(string); - - do_match(vm, ®ex, search_text) -} - -/// Implement re.search -/// See also: -/// https://docs.python.org/3/library/re.html#re.search -fn re_search(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (pattern, Some(vm.ctx.str_type())), - (string, Some(vm.ctx.str_type())) - ] - ); - - let pattern_str = objstr::get_value(&pattern); - let regex = make_regex(vm, &pattern_str)?; - let search_text = objstr::get_value(string); - - do_search(vm, ®ex, search_text) -} - -fn do_match(vm: &VirtualMachine, regex: &Regex, search_text: String) -> PyResult { - // TODO: implement match! - do_search(vm, regex, search_text) -} - -fn do_search(vm: &VirtualMachine, regex: &Regex, search_text: String) -> PyResult { - match regex.find(&search_text) { - None => Ok(vm.get_none()), - Some(result) => create_match(vm, &result), - } -} - -fn make_regex(vm: &VirtualMachine, pattern: &str) -> PyResult { - match Regex::new(pattern) { - Ok(regex) => Ok(regex), - Err(err) => Err(vm.new_value_error(format!("Error in regex: {:?}", err))), - } -} - /// Inner data for a match object. #[derive(Debug)] struct PyMatch { @@ -111,6 +30,38 @@ impl PyValue for PyMatch { } } +type PyRegexRef = PyRef; +type PyMatchRef = PyRef; + +fn re_match(pattern: PyStringRef, string: PyStringRef, vm: &VirtualMachine) -> PyResult { + let regex = make_regex(vm, &pattern.value)?; + do_match(vm, ®ex, &string.value) +} + +fn re_search(pattern: PyStringRef, string: PyStringRef, vm: &VirtualMachine) -> PyResult { + let regex = make_regex(vm, &pattern.value)?; + do_search(vm, ®ex, &string.value) +} + +fn do_match(vm: &VirtualMachine, regex: &Regex, search_text: &str) -> PyResult { + // TODO: implement match! + do_search(vm, regex, search_text) +} + +fn do_search(vm: &VirtualMachine, regex: &Regex, search_text: &str) -> PyResult { + match regex.find(search_text) { + None => Ok(vm.get_none()), + Some(result) => create_match(vm, &result), + } +} + +fn make_regex(vm: &VirtualMachine, pattern: &str) -> PyResult { + match Regex::new(pattern) { + Ok(regex) => Ok(regex), + Err(err) => Err(vm.new_value_error(format!("Error in regex: {:?}", err))), + } +} + /// Take a found regular expression and convert it to proper match object. fn create_match(vm: &VirtualMachine, match_value: &Match) -> PyResult { // let mo = vm.invoke(match_class, PyFuncArgs::default())?; @@ -124,68 +75,45 @@ fn create_match(vm: &VirtualMachine, match_value: &Match) -> PyResult { .into_object()) } -/// Compile a regular expression into a Pattern object. -/// See also: -/// https://docs.python.org/3/library/re.html#re.compile -fn re_compile(pattern: PyStringRef, vm: &VirtualMachine) -> PyResult> { - let regex = make_regex(vm, &pattern.value)?; - - Ok(regex.into_ref(vm)) +fn re_compile(pattern: PyStringRef, vm: &VirtualMachine) -> PyResult { + make_regex(vm, &pattern.value) } -fn pattern_match(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (text, Some(vm.ctx.str_type()))] - ); - - let regex = get_regex(zelf); - let search_text = objstr::get_value(text); - do_match(vm, ®ex, search_text) -} - -fn pattern_search(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (text, Some(vm.ctx.str_type()))] - ); - - let regex = get_regex(zelf); - let search_text = objstr::get_value(text); - do_search(vm, ®ex, search_text) -} - -/// Returns start of match -/// see: https://docs.python.org/3/library/re.html#re.Match.start -fn match_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - // TODO: implement groups - let m = get_match(zelf); - Ok(vm.new_int(m.start)) -} - -fn match_end(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - // TODO: implement groups - let m = get_match(zelf); - Ok(vm.new_int(m.end)) -} - -/// Retrieve inner rust regex from python object: -fn get_regex(obj: &PyObjectRef) -> &Regex { - // TODO: Regex shouldn't be stored in payload directly, create newtype wrapper - if let Some(regex) = obj.payload::() { - return regex; +impl PyRegexRef { + fn match_(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { + do_match(vm, &self, &text.value) + } + fn search(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { + do_search(vm, &self, &text.value) } - panic!("Inner error getting regex {:?}", obj); } -/// Retrieve inner rust match from python object: -fn get_match(obj: &PyObjectRef) -> &PyMatch { - if let Some(value) = obj.payload::() { - return value; +impl PyMatchRef { + fn start(self, _vm: &VirtualMachine) -> usize { + self.start + } + fn end(self, _vm: &VirtualMachine) -> usize { + self.end } - panic!("Inner error getting match {:?}", obj); +} + +/// Create the python `re` module with all its members. +pub fn make_module(ctx: &PyContext) -> PyObjectRef { + let match_type = py_class!(ctx, "Match", ctx.object(), { + "start" => ctx.new_rustfunc(PyMatchRef::start), + "end" => ctx.new_rustfunc(PyMatchRef::end) + }); + + let pattern_type = py_class!(ctx, "Pattern", ctx.object(), { + "match" => ctx.new_rustfunc(PyRegexRef::match_), + "search" => ctx.new_rustfunc(PyRegexRef::search) + }); + + py_module!(ctx, "re", { + "compile" => ctx.new_rustfunc(re_compile), + "Match" => match_type, + "match" => ctx.new_rustfunc(re_match), + "Pattern" => pattern_type, + "search" => ctx.new_rustfunc(re_search) + }) } From d5e2b3a8b6fd5624be89cb6d126f33e794f23020 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Thu, 28 Mar 2019 22:42:21 +0100 Subject: [PATCH 052/658] Convert iterators next() to new args style Except objiter, as it'll need to be reworked anyway --- vm/src/obj/objenumerate.rs | 24 +++++++----------------- vm/src/obj/objfilter.rs | 19 ++++++------------- vm/src/obj/objmap.rs | 23 ++++++++--------------- vm/src/obj/objzip.rs | 19 ++++++++----------- 4 files changed, 29 insertions(+), 56 deletions(-) diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 68d1e49f6..85781bb7a 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -4,8 +4,8 @@ use std::ops::AddAssign; use num_bigint::BigInt; use num_traits::Zero; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::PyIntRef; @@ -44,18 +44,10 @@ fn enumerate_new( .into_ref_with_type(vm, cls) } -fn enumerate_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(enumerate, Some(vm.ctx.enumerate_type()))] - ); - - if let Some(PyEnumerate { - ref counter, - ref iterator, - }) = enumerate.payload() - { +impl PyEnumerateRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let iterator = &self.iterator; + let counter = &self.counter; let next_obj = objiter::call_next(vm, iterator)?; let result = vm .ctx @@ -64,8 +56,6 @@ fn enumerate_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { AddAssign::add_assign(&mut counter.borrow_mut() as &mut BigInt, 1); Ok(result) - } else { - panic!("enumerate doesn't have correct payload"); } } @@ -74,6 +64,6 @@ pub fn init(context: &PyContext) { objiter::iter_type_init(context, enumerate_type); extend_class!(context, enumerate_type, { "__new__" => context.new_rustfunc(enumerate_new), - "__next__" => context.new_rustfunc(enumerate_next) + "__next__" => context.new_rustfunc(PyEnumerateRef::next) }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index e547d2c43..07c512f6d 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -1,5 +1,4 @@ -use crate::function::PyFuncArgs; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; // Required for arg_check! to use isinstance use super::objbool; @@ -35,14 +34,10 @@ fn filter_new( .into_ref_with_type(vm, cls) } -fn filter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(filter, Some(vm.ctx.filter_type()))]); - - if let Some(PyFilter { - ref predicate, - ref iterator, - }) = filter.payload() - { +impl PyFilterRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let predicate = &self.predicate; + let iterator = &self.iterator; loop { let next_obj = objiter::call_next(vm, iterator)?; let predicate_value = if predicate.is(&vm.get_none()) { @@ -56,8 +51,6 @@ fn filter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { return Ok(next_obj); } } - } else { - panic!("filter doesn't have correct payload"); } } @@ -74,6 +67,6 @@ pub fn init(context: &PyContext) { extend_class!(context, filter_type, { "__new__" => context.new_rustfunc(filter_new), "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(filter_next) + "__next__" => context.new_rustfunc(PyFilterRef::next) }); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index 49cc539e5..fbf0fa035 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -1,5 +1,5 @@ -use crate::function::{Args, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::Args; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; @@ -35,23 +35,16 @@ fn map_new( .into_ref_with_type(vm, cls.clone()) } -fn map_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(map, Some(vm.ctx.map_type()))]); - - if let Some(PyMap { - ref mapper, - ref iterators, - }) = map.payload() - { - let next_objs = iterators +impl PyMapRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let next_objs = self + .iterators .iter() .map(|iterator| objiter::call_next(vm, iterator)) .collect::, _>>()?; // the mapper itself can raise StopIteration which does stop the map iteration - vm.invoke(mapper.clone(), next_objs) - } else { - panic!("map doesn't have correct payload"); + vm.invoke(self.mapper.clone(), next_objs) } } @@ -65,7 +58,7 @@ pub fn init(context: &PyContext) { objiter::iter_type_init(context, map_type); extend_class!(context, map_type, { "__new__" => context.new_rustfunc(map_new), - "__next__" => context.new_rustfunc(map_next), + "__next__" => context.new_rustfunc(PyMapRef::next), "__doc__" => context.new_str(map_doc.to_string()) }); } diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index e628cb218..a12152eee 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -1,5 +1,5 @@ -use crate::function::{Args, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::Args; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; @@ -26,22 +26,19 @@ fn zip_new(cls: PyClassRef, iterables: Args, vm: &VirtualMachine) -> PyResult PyResult { - arg_check!(vm, args, required = [(zip, Some(vm.ctx.zip_type()))]); - - if let Some(PyZip { ref iterators }) = zip.payload() { - if iterators.is_empty() { +impl PyZipRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.iterators.is_empty() { Err(objiter::new_stop_iteration(vm)) } else { - let next_objs = iterators + let next_objs = self + .iterators .iter() .map(|iterator| objiter::call_next(vm, iterator)) .collect::, _>>()?; Ok(vm.ctx.new_tuple(next_objs)) } - } else { - panic!("zip doesn't have correct payload"); } } @@ -50,6 +47,6 @@ pub fn init(context: &PyContext) { objiter::iter_type_init(context, zip_type); extend_class!(context, zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(zip_next) + "__next__" => context.new_rustfunc(PyZipRef::next) }); } From 8f1ec3dd23a359e29f4bd6934b8a0b4d19aaaa6a Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Thu, 28 Mar 2019 23:06:38 +0100 Subject: [PATCH 053/658] Convert slice::{start, stop, stop} to new args style --- vm/src/obj/objslice.rs | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 3d955b3f2..c82a3ca6e 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -66,38 +66,25 @@ fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { .map(|x| x.into_object()) } -fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyResult { +fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { if let Some(value) = value { - Ok(vm.ctx.new_int(value.clone())) + vm.ctx.new_int(value.clone()) } else { - Ok(vm.get_none()) + vm.get_none() } } -fn slice_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { start, .. }) = &slice.payload() { - get_property_value(vm, start) - } else { - panic!("Slice has incorrect payload."); +impl PySliceRef { + fn start(self, vm: &VirtualMachine) -> PyObjectRef { + get_property_value(vm, &self.start) } -} -fn slice_stop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { stop, .. }) = &slice.payload() { - get_property_value(vm, stop) - } else { - panic!("Slice has incorrect payload."); + fn stop(self, vm: &VirtualMachine) -> PyObjectRef { + get_property_value(vm, &self.stop) } -} -fn slice_step(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { step, .. }) = &slice.payload() { - get_property_value(vm, step) - } else { - panic!("Slice has incorrect payload."); + fn step(self, vm: &VirtualMachine) -> PyObjectRef { + get_property_value(vm, &self.step) } } @@ -106,8 +93,8 @@ pub fn init(context: &PyContext) { extend_class!(context, slice_type, { "__new__" => context.new_rustfunc(slice_new), - "start" => context.new_property(slice_start), - "stop" => context.new_property(slice_stop), - "step" => context.new_property(slice_step) + "start" => context.new_property(PySliceRef::start), + "stop" => context.new_property(PySliceRef::stop), + "step" => context.new_property(PySliceRef::step) }); } From ff85838556989d9dccc79f3d8baa1ae44202a65e Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 00:16:34 -0500 Subject: [PATCH 054/658] Add #[py_class] attribute proc macro --- derive/Cargo.toml | 2 +- derive/src/lib.rs | 198 ++++++++++++++- vm/src/builtins.rs | 1 + vm/src/lib.rs | 2 +- vm/src/obj/objstr.rs | 572 +++++++++++++++++++++++-------------------- vm/src/pyobject.rs | 20 ++ 6 files changed, 517 insertions(+), 278 deletions(-) diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c15d47ad5..91fdc150e 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -8,6 +8,6 @@ edition = "2018" proc-macro = true [dependencies] -syn = "0.15.29" +syn = { version = "0.15.29", features = ["full"] } quote = "0.6.11" proc-macro2 = "0.4.27" diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ad743e72f..64853170f 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,21 +1,57 @@ extern crate proc_macro; use proc_macro::TokenStream; -use proc_macro2::TokenStream as TokenStream2; +use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; -use syn::{Data, DeriveInput, Fields}; +use syn::{ + parse_macro_input, AttributeArgs, Data, DeriveInput, Fields, Ident, ImplItem, Item, Lit, Meta, + NestedMeta, +}; -#[proc_macro_derive(FromArgs)] +fn rustpython_path(inside_vm: bool) -> syn::Path { + let path = if inside_vm { + quote!(crate) + } else { + quote!(::rustpython_vm) + }; + syn::parse2(path).unwrap() +} + +/// Does the item have the #[__inside_vm] attribute on it, signifying that the derive target is +/// being derived from inside the `rustpython_vm` crate. +fn rustpython_path_derive(input: &DeriveInput) -> syn::Path { + rustpython_path( + input + .attrs + .iter() + .any(|attr| attr.path.is_ident("__inside_vm")), + ) +} + +fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { + rustpython_path(attr.iter().any(|meta| { + if let syn::NestedMeta::Meta(meta) = meta { + if let syn::Meta::Word(ident) = meta { + ident == "__inside_vm" + } else { + false + } + } else { + false + } + })) +} + +#[proc_macro_derive(FromArgs, attributes(__inside_vm))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); - let gen = impl_from_args(&ast); + let gen = impl_from_args(ast); gen.to_string().parse().unwrap() } -fn impl_from_args(input: &DeriveInput) -> TokenStream2 { - // FIXME: This references types using `crate` instead of `rustpython_vm` - // so that it can be used in the latter. How can we support both? +fn impl_from_args(input: DeriveInput) -> TokenStream2 { + let rp_path = rustpython_path_derive(&input); let fields = match input.data { Data::Struct(ref data) => { match data.fields { @@ -36,7 +72,7 @@ fn impl_from_args(input: &DeriveInput) -> TokenStream2 { let name = &input.ident; quote! { - impl crate::function::FromArgs for #name { + impl #rp_path::function::FromArgs for #name { fn from_args( vm: &crate::vm::VirtualMachine, args: &mut crate::function::PyFuncArgs @@ -46,3 +82,149 @@ fn impl_from_args(input: &DeriveInput) -> TokenStream2 { } } } + +#[proc_macro_attribute] +pub fn py_class(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + impl_py_class(attr, item).into() +} + +enum MethodKind { + Method, + Property, +} + +impl MethodKind { + fn to_ctx_constructor_fn(&self) -> Ident { + let f = match self { + MethodKind::Method => "new_rustfunc", + MethodKind::Property => "new_property", + }; + Ident::new(f, Span::call_site()) + } +} + +struct Method { + fn_name: Ident, + py_name: String, + kind: MethodKind, +} + +fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter().filter_map(|item| { + if let ImplItem::Method(meth) = item { + let mut py_name = None; + let mut kind = MethodKind::Method; + let metas_iter = meth + .attrs + .iter() + .filter_map(|attr| { + if attr.path.is_ident("py_class") { + let meta = attr.parse_meta().expect("Invalid attribute"); + if let Meta::List(list) = meta { + Some(list) + } else { + panic!( + "#[py_class] attribute on a method should be a list, like \ + #[py_class(...)]" + ) + } + } else { + None + } + }) + .flat_map(|attr| attr.nested); + for meta in metas_iter { + if let NestedMeta::Meta(meta) = meta { + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => match ident.to_string().as_str() { + "property" => kind = MethodKind::Property, + _ => {} + }, + _ => {} + } + } + } + let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); + Some(Method { + fn_name: meth.sig.ident.clone(), + py_name, + kind, + }) + } else { + None + } + }) +} + +fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { + let imp = if let Item::Impl(imp) = item { + imp + } else { + return quote!(#item); + }; + let rp_path = rustpython_path_attr(&attr); + let mut class_name = None; + let mut doc = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } else if name_value.ident == "doc" { + if let Lit::Str(s) = name_value.lit { + doc = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.expect("#[py_class] must have a name"); + let doc = match doc { + Some(doc) => quote!(Some(#doc)), + None => quote!(None), + }; + let ty = &imp.self_ty; + let methods = item_impl_to_methods(&imp).map( + |Method { + py_name, + fn_name, + kind, + }| { + let constructor_fn = kind.to_ctx_constructor_fn(); + quote! { + ctx.set_attr(class, #py_name, ctx.#constructor_fn(#ty::#fn_name)); + } + }, + ); + + quote! { + #imp + impl #rp_path::pyobject::IntoPyClass for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + fn _extend_class( + ctx: &#rp_path::pyobject::PyContext, + class: &#rp_path::obj::objtype::PyClassRef, + ) { + #(#methods)* + } + } + } +} diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 9d5980759..6b5dda16b 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -536,6 +536,7 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } #[derive(Debug, FromArgs)] +#[__inside_vm] pub struct PrintOptions { sep: Option, end: Option, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 3892fe2a4..29124ad11 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -30,7 +30,7 @@ extern crate statrs; extern crate rustpython_parser; #[macro_use] -extern crate rustpython_derive; +pub extern crate rustpython_derive; //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index aa62812a2..82ac806e6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -10,8 +10,8 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TryIntoRef, TypeProtocol, + IdProtocol, IntoPyClass, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, + PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -48,80 +48,138 @@ impl TryIntoRef for &str { } } -impl PyStringRef { - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(format!("{}{}", self.value, get_value(&rhs))) +fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + let rep_str = match rep { + OptionalArg::Present(ref st) => &st.value, + OptionalArg::Missing => " ", + }; + if rep_str.len() == 1 { + Ok(rep_str) + } else { + Err(vm.new_type_error("The fill character must be exactly one character long".to_string())) + } +} + +#[py_class( + __inside_vm, + name = "str", + doc = "str(object='') -> str\n\ + str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ + \n\ + Create a new string object from the given object. If encoding or\n\ + errors is specified, then the object must expose a data buffer\n\ + that will be decoded using the given encoding and error handler.\n\ + Otherwise, returns the result of object.__str__() (if defined)\n\ + or repr(object).\n\ + encoding defaults to sys.getdefaultencoding().\n\ + errors defaults to 'strict'." +)] +impl PyString { + // TODO: should with following format + // class str(object='') + // class str(object=b'', encoding='utf-8', errors='strict') + #[py_class(name = "__new__")] + fn new( + cls: PyClassRef, + object: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let string = match object { + OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), + OptionalArg::Missing => vm.new_str("".to_string()), + }; + if string.class().is(&cls) { + TryFromObject::try_from_object(vm, string) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", self, rhs))) + let payload = string.payload::().unwrap(); + payload.clone().into_ref_with_type(vm, cls) + } + } + #[py_class(name = "__add__")] + fn add(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&rhs, &vm.ctx.str_type()) { + Ok(format!("{}{}", zelf.value, get_value(&rhs))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", zelf, rhs))) } } - fn bool(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() + #[py_class(name = "__bool__")] + fn bool(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() } - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + #[py_class(name = "__eq__")] + fn eq(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - self.value == get_value(&rhs) + zelf.value == get_value(&rhs) } else { false } } - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { - self.value.contains(&needle.value) + #[py_class(name = "__contains__")] + fn contains(zelf: PyStringRef, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + zelf.value.contains(&needle.value) } - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - subscript(vm, &self.value, needle) + #[py_class(name = "__getitem__")] + fn getitem(zelf: PyStringRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + subscript(vm, &zelf.value, needle) } - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__gt__")] + fn gt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value > get_value(&rhs)) + Ok(zelf.value > get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__ge__")] + fn ge(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value >= get_value(&rhs)) + Ok(zelf.value >= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__lt__")] + fn lt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value < get_value(&rhs)) + Ok(zelf.value < get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__le__")] + fn le(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value <= get_value(&rhs)) + Ok(zelf.value <= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn hash(self, _vm: &VirtualMachine) -> usize { + #[py_class(name = "__hash__")] + fn hash(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); + zelf.value.hash(&mut hasher); hasher.finish() as usize } - fn len(self, _vm: &VirtualMachine) -> usize { - self.value.chars().count() + #[py_class(name = "__len__")] + fn len(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { + zelf.value.chars().count() } - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__mul__")] + fn mul(zelf: PyStringRef, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &self.value; + let value = &zelf.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); let mut result = String::new(); for _x in 0..multiplier { @@ -129,16 +187,18 @@ impl PyStringRef { } Ok(result) } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) + Err(vm.new_type_error(format!("Cannot multiply {} and {}", zelf, val))) } } - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + #[py_class(name = "__str__")] + fn str(zelf: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + zelf } - fn repr(self, _vm: &VirtualMachine) -> String { - let value = &self.value; + #[py_class(name = "__repr__")] + fn repr(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let value = &zelf.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' } else { @@ -167,31 +227,31 @@ impl PyStringRef { formatted } - fn lower(self, _vm: &VirtualMachine) -> String { - self.value.to_lowercase() + fn lower(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(self, _vm: &VirtualMachine) -> String { - caseless::default_case_fold_str(&self.value) + fn casefold(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + caseless::default_case_fold_str(&zelf.value) } - fn upper(self, _vm: &VirtualMachine) -> String { - self.value.to_uppercase() + fn upper(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.to_uppercase() } - fn capitalize(self, _vm: &VirtualMachine) -> String { - let (first_part, lower_str) = self.value.split_at(1); + fn capitalize(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let (first_part, lower_str) = zelf.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } fn split( - self, - pattern: OptionalArg, + zelf: PyStringRef, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &self.value; + let value = &zelf.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -207,12 +267,12 @@ impl PyStringRef { } fn rsplit( - self, - pattern: OptionalArg, + zelf: PyStringRef, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &self.value; + let value = &zelf.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -227,85 +287,113 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn strip(self, _vm: &VirtualMachine) -> String { - self.value.trim().to_string() + fn strip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim().to_string() } - fn lstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_start().to_string() + fn lstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim_start().to_string() } - fn rstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_end().to_string() + fn rstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim_end().to_string() } fn endswith( - self, + zelf: PyStringRef, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].ends_with(&suffix.value) + if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { + zelf.value[start..end].ends_with(&suffix.value) } else { false } } fn startswith( - self, + zelf: PyStringRef, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].starts_with(&prefix.value) + if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { + zelf.value[start..end].starts_with(&prefix.value) } else { false } } - fn isalnum(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) + fn isalnum(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) } - fn isnumeric(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_numeric) + fn isnumeric(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_numeric) } - fn isdigit(self, _vm: &VirtualMachine) -> bool { + fn isdigit(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, ]; - if self.value.is_empty() { + if zelf.value.is_empty() { false } else { - self.value + zelf.value .chars() .filter(|c| !c.is_digit(10)) .all(|c| valid_unicodes.contains(&(c as u16))) } } - fn isdecimal(self, _vm: &VirtualMachine) -> bool { - if self.value.is_empty() { + fn isdecimal(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + if zelf.value.is_empty() { false } else { - self.value.chars().all(|c| c.is_ascii_digit()) + zelf.value.chars().all(|c| c.is_ascii_digit()) } } - fn title(self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + if args.args.is_empty() { + return Err(vm.new_type_error( + "descriptor 'format' of 'str' object needs an argument".to_string(), + )); + } + + let zelf = &args.args[0]; + if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { + let zelf_typ = zelf.class(); + let actual_type = vm.to_pystr(&zelf_typ)?; + return Err(vm.new_type_error(format!( + "descriptor 'format' requires a 'str' object but received a '{}'", + actual_type + ))); + } + let format_string_text = get_value(zelf); + match FormatString::from_str(format_string_text.as_str()) { + Ok(format_string) => perform_format(vm, &format_string, &args), + Err(err) => match err { + FormatParseError::UnmatchedBracket => { + Err(vm.new_value_error("expected '}' before end of string".to_string())) + } + _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), + }, + } } - fn swapcase(self, _vm: &VirtualMachine) -> String { - let mut swapped_str = String::with_capacity(self.value.len()); - for c in self.value.chars() { + fn title(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + make_title(&zelf.value) + } + + fn swapcase(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let mut swapped_str = String::with_capacity(zelf.value.len()); + for c in zelf.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { swapped_str.push(c.to_ascii_uppercase()); @@ -318,54 +406,54 @@ impl PyStringRef { swapped_str } - fn isalpha(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) + fn isalpha(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) } fn replace( - self, - old: Self, - new: Self, + zelf: PyStringRef, + old: PyStringRef, + new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { match num.into_option() { - Some(num) => self.value.replacen(&old.value, &new.value, num), - None => self.value.replace(&old.value, &new.value), + Some(num) => zelf.value.replacen(&old.value, &new.value, num), + None => zelf.value.replace(&old.value, &new.value), } } // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) + fn isspace(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() - && self + fn isupper(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() + && zelf .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_uppercase) } - fn islower(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() - && self + fn islower(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() + && zelf .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_lowercase) } - fn isascii(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) + fn isascii(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { - let elements = self + fn splitlines(zelf: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let elements = zelf .value .split('\n') .map(|e| vm.ctx.new_str(e.to_string())) @@ -373,13 +461,17 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join( + zelf: PyStringRef, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { let elem = elem?; if idx != 0 { - joined.push_str(&self.value); + joined.push_str(&zelf.value); } joined.push_str(&elem.value) } @@ -388,13 +480,13 @@ impl PyStringRef { } fn find( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => (start + num) as isize, @@ -406,13 +498,13 @@ impl PyStringRef { } fn rfind( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => (start + num) as isize, @@ -424,13 +516,13 @@ impl PyStringRef { } fn index( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => Ok(start + num), @@ -442,13 +534,13 @@ impl PyStringRef { } fn rindex( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => Ok(start + num), @@ -459,8 +551,8 @@ impl PyStringRef { } } - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &self.value; + fn partition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &zelf.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -477,8 +569,8 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &self.value; + fn rpartition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &zelf.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -496,31 +588,31 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn istitle(self, _vm: &VirtualMachine) -> bool { - if self.value.is_empty() { + fn istitle(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + if zelf.value.is_empty() { false } else { - self.value.split(' ').all(|word| word == make_title(word)) + zelf.value.split(' ').all(|word| word == make_title(word)) } } fn count( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> usize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { - self.value[start..end].matches(&sub.value).count() + zelf.value[start..end].matches(&sub.value).count() } else { 0 } } - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { - let value = &self.value; + fn zfill(zelf: PyStringRef, len: usize, _vm: &VirtualMachine) -> String { + let value = &zelf.value; if len <= value.len() { value.to_string() } else { @@ -528,35 +620,36 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { - let rep_str = match rep { - OptionalArg::Present(ref st) => &st.value, - OptionalArg::Missing => " ", - }; - if rep_str.len() == 1 { - Ok(rep_str) - } else { - Err(vm.new_type_error( - "The fill character must be exactly one character long".to_string(), - )) - } - } - - fn ljust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn ljust( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } - fn rjust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn rjust( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } - fn center(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn center( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; Ok(format!( @@ -567,12 +660,12 @@ impl PyStringRef { )) } - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(zelf: PyStringRef, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; let mut col_count = 0 as usize; - for ch in self.value.chars() { + for ch in zelf.value.chars() { // 0x0009 is tab if ch == 0x0009 as char { let num_spaces = tab_size - col_count; @@ -590,8 +683,8 @@ impl PyStringRef { expanded_str } - fn isidentifier(self, _vm: &VirtualMachine) -> bool { - let value = &self.value; + fn isidentifier(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + let value = &zelf.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) && !value.chars().nth(0).unwrap().is_digit(10) @@ -623,75 +716,66 @@ impl IntoPyObject for String { #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let str_type = &context.str_type; - let str_doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'."; - extend_class!(context, str_type, { - "__add__" => context.new_rustfunc(PyStringRef::add), - "__bool__" => context.new_rustfunc(PyStringRef::bool), - "__contains__" => context.new_rustfunc(PyStringRef::contains), - "__doc__" => context.new_str(str_doc.to_string()), - "__eq__" => context.new_rustfunc(PyStringRef::eq), - "__ge__" => context.new_rustfunc(PyStringRef::ge), - "__getitem__" => context.new_rustfunc(PyStringRef::getitem), - "__gt__" => context.new_rustfunc(PyStringRef::gt), - "__hash__" => context.new_rustfunc(PyStringRef::hash), - "__lt__" => context.new_rustfunc(PyStringRef::lt), - "__le__" => context.new_rustfunc(PyStringRef::le), - "__len__" => context.new_rustfunc(PyStringRef::len), - "__mul__" => context.new_rustfunc(PyStringRef::mul), - "__new__" => context.new_rustfunc(str_new), - "__repr__" => context.new_rustfunc(PyStringRef::repr), - "__str__" => context.new_rustfunc(PyStringRef::str), - "capitalize" => context.new_rustfunc(PyStringRef::capitalize), - "casefold" => context.new_rustfunc(PyStringRef::casefold), - "center" => context.new_rustfunc(PyStringRef::center), - "count" => context.new_rustfunc(PyStringRef::count), - "endswith" => context.new_rustfunc(PyStringRef::endswith), - "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), - "find" => context.new_rustfunc(PyStringRef::find), - "format" => context.new_rustfunc(str_format), - "index" => context.new_rustfunc(PyStringRef::index), - "isalnum" => context.new_rustfunc(PyStringRef::isalnum), - "isalpha" => context.new_rustfunc(PyStringRef::isalpha), - "isascii" => context.new_rustfunc(PyStringRef::isascii), - "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), - "isdigit" => context.new_rustfunc(PyStringRef::isdigit), - "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), - "islower" => context.new_rustfunc(PyStringRef::islower), - "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), - "isspace" => context.new_rustfunc(PyStringRef::isspace), - "isupper" => context.new_rustfunc(PyStringRef::isupper), - "istitle" => context.new_rustfunc(PyStringRef::istitle), - "join" => context.new_rustfunc(PyStringRef::join), - "lower" => context.new_rustfunc(PyStringRef::lower), - "ljust" => context.new_rustfunc(PyStringRef::ljust), - "lstrip" => context.new_rustfunc(PyStringRef::lstrip), - "partition" => context.new_rustfunc(PyStringRef::partition), - "replace" => context.new_rustfunc(PyStringRef::replace), - "rfind" => context.new_rustfunc(PyStringRef::rfind), - "rindex" => context.new_rustfunc(PyStringRef::rindex), - "rjust" => context.new_rustfunc(PyStringRef::rjust), - "rpartition" => context.new_rustfunc(PyStringRef::rpartition), - "rsplit" => context.new_rustfunc(PyStringRef::rsplit), - "rstrip" => context.new_rustfunc(PyStringRef::rstrip), - "split" => context.new_rustfunc(PyStringRef::split), - "splitlines" => context.new_rustfunc(PyStringRef::splitlines), - "startswith" => context.new_rustfunc(PyStringRef::startswith), - "strip" => context.new_rustfunc(PyStringRef::strip), - "swapcase" => context.new_rustfunc(PyStringRef::swapcase), - "title" => context.new_rustfunc(PyStringRef::title), - "upper" => context.new_rustfunc(PyStringRef::upper), - "zfill" => context.new_rustfunc(PyStringRef::zfill), - }); + PyString::extend_class(context, str_type); + // extend_class!(context, str_type, { + // "__add__" => context.new_rustfunc(PyStringRef::add), + // "__bool__" => context.new_rustfunc(PyStringRef::bool), + // "__contains__" => context.new_rustfunc(PyStringRef::contains), + // "__doc__" => context.new_str(str_doc.to_string()), + // "__eq__" => context.new_rustfunc(PyStringRef::eq), + // "__ge__" => context.new_rustfunc(PyStringRef::ge), + // "__getitem__" => context.new_rustfunc(PyStringRef::getitem), + // "__gt__" => context.new_rustfunc(PyStringRef::gt), + // "__hash__" => context.new_rustfunc(PyStringRef::hash), + // "__lt__" => context.new_rustfunc(PyStringRef::lt), + // "__le__" => context.new_rustfunc(PyStringRef::le), + // "__len__" => context.new_rustfunc(PyStringRef::len), + // "__mul__" => context.new_rustfunc(PyStringRef::mul), + // "__new__" => context.new_rustfunc(str_new), + // "__repr__" => context.new_rustfunc(PyStringRef::repr), + // "__str__" => context.new_rustfunc(PyStringRef::str), + // "capitalize" => context.new_rustfunc(PyStringRef::capitalize), + // "casefold" => context.new_rustfunc(PyStringRef::casefold), + // "center" => context.new_rustfunc(PyStringRef::center), + // "count" => context.new_rustfunc(PyStringRef::count), + // "endswith" => context.new_rustfunc(PyStringRef::endswith), + // "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), + // "find" => context.new_rustfunc(PyStringRef::find), + // "format" => context.new_rustfunc(str_format), + // "index" => context.new_rustfunc(PyStringRef::index), + // "isalnum" => context.new_rustfunc(PyStringRef::isalnum), + // "isalpha" => context.new_rustfunc(PyStringRef::isalpha), + // "isascii" => context.new_rustfunc(PyStringRef::isascii), + // "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), + // "isdigit" => context.new_rustfunc(PyStringRef::isdigit), + // "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), + // "islower" => context.new_rustfunc(PyStringRef::islower), + // "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), + // "isspace" => context.new_rustfunc(PyStringRef::isspace), + // "isupper" => context.new_rustfunc(PyStringRef::isupper), + // "istitle" => context.new_rustfunc(PyStringRef::istitle), + // "join" => context.new_rustfunc(PyStringRef::join), + // "lower" => context.new_rustfunc(PyStringRef::lower), + // "ljust" => context.new_rustfunc(PyStringRef::ljust), + // "lstrip" => context.new_rustfunc(PyStringRef::lstrip), + // "partition" => context.new_rustfunc(PyStringRef::partition), + // "replace" => context.new_rustfunc(PyStringRef::replace), + // "rfind" => context.new_rustfunc(PyStringRef::rfind), + // "rindex" => context.new_rustfunc(PyStringRef::rindex), + // "rjust" => context.new_rustfunc(PyStringRef::rjust), + // "rpartition" => context.new_rustfunc(PyStringRef::rpartition), + // "rsplit" => context.new_rustfunc(PyStringRef::rsplit), + // "rstrip" => context.new_rustfunc(PyStringRef::rstrip), + // "split" => context.new_rustfunc(PyStringRef::split), + // "splitlines" => context.new_rustfunc(PyStringRef::splitlines), + // "startswith" => context.new_rustfunc(PyStringRef::startswith), + // "strip" => context.new_rustfunc(PyStringRef::strip), + // "swapcase" => context.new_rustfunc(PyStringRef::swapcase), + // "title" => context.new_rustfunc(PyStringRef::title), + // "upper" => context.new_rustfunc(PyStringRef::upper), + // "zfill" => context.new_rustfunc(PyStringRef::zfill), + // }); } pub fn get_value(obj: &PyObjectRef) -> String { @@ -706,34 +790,6 @@ fn count_char(s: &str, c: char) -> usize { s.chars().filter(|x| *x == c).count() } -fn str_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - return Err( - vm.new_type_error("descriptor 'format' of 'str' object needs an argument".to_string()) - ); - } - - let zelf = &args.args[0]; - if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { - let zelf_typ = zelf.class(); - let actual_type = vm.to_pystr(&zelf_typ)?; - return Err(vm.new_type_error(format!( - "descriptor 'format' requires a 'str' object but received a '{}'", - actual_type - ))); - } - let format_string_text = get_value(zelf); - match FormatString::from_str(format_string_text.as_str()) { - Ok(format_string) => perform_format(vm, &format_string, &args), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_string())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), - }, - } -} - fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { let returned_type = vm.ctx.new_str(format_spec.to_string()); let result = vm.call_method(&argument, "__format__", vec![returned_type])?; @@ -797,26 +853,6 @@ fn perform_format( Ok(vm.ctx.new_str(final_string)) } -// TODO: should with following format -// class str(object='') -// class str(object=b'', encoding='utf-8', errors='strict') -fn str_new( - cls: PyClassRef, - object: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let string = match object { - OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), - OptionalArg::Missing => vm.new_str("".to_string()), - }; - if string.class().is(&cls) { - TryFromObject::try_from_object(vm, string) - } else { - let payload = string.payload::().unwrap(); - payload.clone().into_ref_with_type(vm, cls) - } -} - impl PySliceableSequence for String { fn do_slice(&self, range: Range) -> Self { to_graphemes(self) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index aa3348be5..a0e737d4c 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1259,6 +1259,26 @@ where } } +pub trait IntoPyClass { + const NAME: &'static str; + const DOC: Option<&'static str> = None; + + fn _extend_class(ctx: &PyContext, class: &PyClassRef); + + fn extend_class(ctx: &PyContext, class: &PyClassRef) { + Self::_extend_class(ctx, class); + if let Some(doc) = Self::DOC { + ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); + } + } + + fn make_class(ctx: &PyContext) -> PyClassRef { + let py_class = ctx.new_class(Self::NAME, ctx.object()); + Self::extend_class(ctx, &py_class); + py_class + } +} + #[cfg(test)] mod tests { use super::*; From ce9a909b9d34d2f6c1983facd1bcf211ca709b44 Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 29 Mar 2019 19:03:13 +1300 Subject: [PATCH 055/658] Change kind identifier to fully use python terminology --- derive/src/lib.rs | 6 +++--- vm/src/builtins.rs | 6 +++--- vm/src/obj/objint.rs | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index b4048ec83..8f15f0e87 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -23,11 +23,11 @@ enum ParameterKind { impl ParameterKind { fn from_ident(ident: &Ident) -> ParameterKind { - if ident == "positional" { + if ident == "positional_only" { ParameterKind::PositionalOnly - } else if ident == "positional_keyword" { + } else if ident == "positional_or_keyword" { ParameterKind::PositionalOrKeyword - } else if ident == "keyword" { + } else if ident == "keyword_only" { ParameterKind::KeywordOnly } else { panic!("Unrecognised attribute") diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 3a695c517..ed656c0fb 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -537,11 +537,11 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { #[derive(Debug, FromArgs)] pub struct PrintOptions { - #[pyarg(keyword, default = "None")] + #[pyarg(keyword_only, default = "None")] sep: Option, - #[pyarg(keyword, default = "None")] + #[pyarg(keyword_only, default = "None")] end: Option, - #[pyarg(keyword, default = "false")] + #[pyarg(keyword_only, default = "false")] flush: bool, } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 0fad52907..08175dc07 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -374,9 +374,9 @@ impl PyIntRef { #[derive(FromArgs)] struct IntOptions { - #[pyarg(positional, optional = true)] + #[pyarg(positional_only, optional = true)] val_options: OptionalArg, - #[pyarg(positional_keyword, optional = true)] + #[pyarg(positional_or_keyword, optional = true)] base: OptionalArg, } From ab6031871cff7602e6160817465fbf561df8b597 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 14:51:42 -0500 Subject: [PATCH 056/658] impl PyStringRef using py_class --- derive/src/lib.rs | 2 +- vm/src/obj/objstr.rs | 252 +++++++++++++++++++++---------------------- 2 files changed, 125 insertions(+), 129 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index be16403ef..f3464b5f1 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -374,7 +374,7 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { }| { let constructor_fn = kind.to_ctx_constructor_fn(); quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(#ty::#fn_name)); + ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); } }, ); diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 82ac806e6..66590fdb2 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -74,7 +74,7 @@ fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> encoding defaults to sys.getdefaultencoding().\n\ errors defaults to 'strict'." )] -impl PyString { +impl PyStringRef { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') @@ -96,90 +96,90 @@ impl PyString { } } #[py_class(name = "__add__")] - fn add(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(format!("{}{}", zelf.value, get_value(&rhs))) + Ok(format!("{}{}", self.value, get_value(&rhs))) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot add {} and {}", self, rhs))) } } #[py_class(name = "__bool__")] - fn bool(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() } #[py_class(name = "__eq__")] - fn eq(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - zelf.value == get_value(&rhs) + self.value == get_value(&rhs) } else { false } } #[py_class(name = "__contains__")] - fn contains(zelf: PyStringRef, needle: PyStringRef, _vm: &VirtualMachine) -> bool { - zelf.value.contains(&needle.value) + fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + self.value.contains(&needle.value) } #[py_class(name = "__getitem__")] - fn getitem(zelf: PyStringRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - subscript(vm, &zelf.value, needle) + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + subscript(vm, &self.value, needle) } #[py_class(name = "__gt__")] - fn gt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value > get_value(&rhs)) + Ok(self.value > get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__ge__")] - fn ge(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value >= get_value(&rhs)) + Ok(self.value >= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__lt__")] - fn lt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value < get_value(&rhs)) + Ok(self.value < get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__le__")] - fn le(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value <= get_value(&rhs)) + Ok(self.value <= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__hash__")] - fn hash(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { + fn hash(self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); - zelf.value.hash(&mut hasher); + self.value.hash(&mut hasher); hasher.finish() as usize } #[py_class(name = "__len__")] - fn len(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { - zelf.value.chars().count() + fn len(self, _vm: &VirtualMachine) -> usize { + self.value.chars().count() } #[py_class(name = "__mul__")] - fn mul(zelf: PyStringRef, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &zelf.value; + let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); let mut result = String::new(); for _x in 0..multiplier { @@ -187,18 +187,18 @@ impl PyString { } Ok(result) } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", zelf, val))) + Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) } } #[py_class(name = "__str__")] - fn str(zelf: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { - zelf + fn str(self, _vm: &VirtualMachine) -> PyStringRef { + self } #[py_class(name = "__repr__")] - fn repr(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let value = &zelf.value; + fn repr(self, _vm: &VirtualMachine) -> String { + let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' } else { @@ -227,31 +227,31 @@ impl PyString { formatted } - fn lower(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.to_lowercase() + fn lower(self, _vm: &VirtualMachine) -> String { + self.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - caseless::default_case_fold_str(&zelf.value) + fn casefold(self, _vm: &VirtualMachine) -> String { + caseless::default_case_fold_str(&self.value) } - fn upper(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.to_uppercase() + fn upper(self, _vm: &VirtualMachine) -> String { + self.value.to_uppercase() } - fn capitalize(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let (first_part, lower_str) = zelf.value.split_at(1); + fn capitalize(self, _vm: &VirtualMachine) -> String { + let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } fn split( - zelf: PyStringRef, + self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &zelf.value; + let value = &self.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -267,12 +267,12 @@ impl PyString { } fn rsplit( - zelf: PyStringRef, + self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &zelf.value; + let value = &self.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -287,75 +287,75 @@ impl PyString { vm.ctx.new_list(elements) } - fn strip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim().to_string() + fn strip(self, _vm: &VirtualMachine) -> String { + self.value.trim().to_string() } - fn lstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim_start().to_string() + fn lstrip(self, _vm: &VirtualMachine) -> String { + self.value.trim_start().to_string() } - fn rstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim_end().to_string() + fn rstrip(self, _vm: &VirtualMachine) -> String { + self.value.trim_end().to_string() } fn endswith( - zelf: PyStringRef, + self, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { - zelf.value[start..end].ends_with(&suffix.value) + if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { + self.value[start..end].ends_with(&suffix.value) } else { false } } fn startswith( - zelf: PyStringRef, + self, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { - zelf.value[start..end].starts_with(&prefix.value) + if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { + self.value[start..end].starts_with(&prefix.value) } else { false } } - fn isalnum(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) + fn isalnum(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } - fn isnumeric(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_numeric) + fn isnumeric(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_numeric) } - fn isdigit(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + fn isdigit(self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, ]; - if zelf.value.is_empty() { + if self.value.is_empty() { false } else { - zelf.value + self.value .chars() .filter(|c| !c.is_digit(10)) .all(|c| valid_unicodes.contains(&(c as u16))) } } - fn isdecimal(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - if zelf.value.is_empty() { + fn isdecimal(self, _vm: &VirtualMachine) -> bool { + if self.value.is_empty() { false } else { - zelf.value.chars().all(|c| c.is_ascii_digit()) + self.value.chars().all(|c| c.is_ascii_digit()) } } @@ -387,13 +387,13 @@ impl PyString { } } - fn title(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - make_title(&zelf.value) + fn title(self, _vm: &VirtualMachine) -> String { + make_title(&self.value) } - fn swapcase(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let mut swapped_str = String::with_capacity(zelf.value.len()); - for c in zelf.value.chars() { + fn swapcase(self, _vm: &VirtualMachine) -> String { + let mut swapped_str = String::with_capacity(self.value.len()); + for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { swapped_str.push(c.to_ascii_uppercase()); @@ -406,54 +406,54 @@ impl PyString { swapped_str } - fn isalpha(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) + fn isalpha(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } fn replace( - zelf: PyStringRef, + self, old: PyStringRef, new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { match num.into_option() { - Some(num) => zelf.value.replacen(&old.value, &new.value, num), - None => zelf.value.replace(&old.value, &new.value), + Some(num) => self.value.replacen(&old.value, &new.value, num), + None => self.value.replace(&old.value, &new.value), } } // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii_whitespace()) + fn isspace(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() - && zelf + fn isupper(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() + && self .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_uppercase) } - fn islower(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() - && zelf + fn islower(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() + && self .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_lowercase) } - fn isascii(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii()) + fn isascii(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(zelf: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let elements = zelf + fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + let elements = self .value .split('\n') .map(|e| vm.ctx.new_str(e.to_string())) @@ -461,17 +461,13 @@ impl PyString { vm.ctx.new_list(elements) } - fn join( - zelf: PyStringRef, - iterable: PyIterable, - vm: &VirtualMachine, - ) -> PyResult { + fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { let elem = elem?; if idx != 0 { - joined.push_str(&zelf.value); + joined.push_str(&self.value); } joined.push_str(&elem.value) } @@ -480,13 +476,13 @@ impl PyString { } fn find( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => (start + num) as isize, @@ -498,13 +494,13 @@ impl PyString { } fn rfind( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => (start + num) as isize, @@ -516,13 +512,13 @@ impl PyString { } fn index( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => Ok(start + num), @@ -534,13 +530,13 @@ impl PyString { } fn rindex( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => Ok(start + num), @@ -551,8 +547,8 @@ impl PyString { } } - fn partition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &zelf.value; + fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -569,8 +565,8 @@ impl PyString { vm.ctx.new_tuple(new_tup) } - fn rpartition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &zelf.value; + fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -588,31 +584,31 @@ impl PyString { vm.ctx.new_tuple(new_tup) } - fn istitle(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - if zelf.value.is_empty() { + fn istitle(self, _vm: &VirtualMachine) -> bool { + if self.value.is_empty() { false } else { - zelf.value.split(' ').all(|word| word == make_title(word)) + self.value.split(' ').all(|word| word == make_title(word)) } } fn count( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> usize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { - zelf.value[start..end].matches(&sub.value).count() + self.value[start..end].matches(&sub.value).count() } else { 0 } } - fn zfill(zelf: PyStringRef, len: usize, _vm: &VirtualMachine) -> String { - let value = &zelf.value; + fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + let value = &self.value; if len <= value.len() { value.to_string() } else { @@ -621,34 +617,34 @@ impl PyString { } fn ljust( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } fn rjust( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } fn center( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; @@ -660,12 +656,12 @@ impl PyString { )) } - fn expandtabs(zelf: PyStringRef, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; let mut col_count = 0 as usize; - for ch in zelf.value.chars() { + for ch in self.value.chars() { // 0x0009 is tab if ch == 0x0009 as char { let num_spaces = tab_size - col_count; @@ -683,8 +679,8 @@ impl PyString { expanded_str } - fn isidentifier(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - let value = &zelf.value; + fn isidentifier(self, _vm: &VirtualMachine) -> bool { + let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) && !value.chars().nth(0).unwrap().is_digit(10) @@ -717,7 +713,7 @@ impl IntoPyObject for String { pub fn init(context: &PyContext) { let str_type = &context.str_type; - PyString::extend_class(context, str_type); + PyStringRef::extend_class(context, str_type); // extend_class!(context, str_type, { // "__add__" => context.new_rustfunc(PyStringRef::add), // "__bool__" => context.new_rustfunc(PyStringRef::bool), From 8376ec2d982bbe90ace709fe7585298f72743871 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 15:17:58 -0500 Subject: [PATCH 057/658] Use doc comments for python __doc__ --- derive/src/lib.rs | 31 +++++++++++++++++++++++-------- vm/src/obj/objstr.rs | 25 +++++++++++-------------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index f3464b5f1..52cfc7567 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -340,7 +340,6 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { }; let rp_path = rustpython_path_attr(&attr); let mut class_name = None; - let mut doc = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { if let Meta::NameValue(name_value) = meta { @@ -350,19 +349,35 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } else { panic!("#[py_class(name = ...)] must be a string"); } - } else if name_value.ident == "doc" { - if let Lit::Str(s) = name_value.lit { - doc = Some(s.value()); - } else { - panic!("#[py_class(name = ...)] must be a string"); - } } } } } let class_name = class_name.expect("#[py_class] must have a name"); + let mut doc: Option> = None; + for attr in imp.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } else { + panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); + } + } + } let doc = match doc { - Some(doc) => quote!(Some(#doc)), + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } None => quote!(None), }; let ty = &imp.self_ty; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 66590fdb2..5e4035c5a 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -60,20 +60,17 @@ fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> } } -#[py_class( - __inside_vm, - name = "str", - doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'." -)] +#[py_class(__inside_vm, name = "str")] +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." impl PyStringRef { // TODO: should with following format // class str(object='') From d9efe4ea3752b8c0f460786d487cacd74f62e772 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 16:16:31 -0500 Subject: [PATCH 058/658] Require a #[pymethod] for a method to be recognized --- derive/src/lib.rs | 62 ++++++++++++++++---------- vm/src/obj/objstr.rs | 104 ++++++++++++++++++++++++++++++------------- 2 files changed, 112 insertions(+), 54 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 52cfc7567..374463f06 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -249,7 +249,7 @@ fn impl_from_args(input: DeriveInput) -> TokenStream2 { } #[proc_macro_attribute] -pub fn py_class(attr: TokenStream, item: TokenStream) -> TokenStream { +pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); impl_py_class(attr, item).into() @@ -276,31 +276,37 @@ struct Method { kind: MethodKind, } -fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter().filter_map(|item| { +/// Parse an impl block into an iterator of methods +fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter_mut().filter_map(|item| { if let ImplItem::Method(meth) = item { let mut py_name = None; let mut kind = MethodKind::Method; - let metas_iter = meth + let mut pymethod_to_remove = Vec::new(); + let metas = meth .attrs .iter() - .filter_map(|attr| { - if attr.path.is_ident("py_class") { + .enumerate() + .filter_map(|(i, attr)| { + if attr.path.is_ident("pymethod") { let meta = attr.parse_meta().expect("Invalid attribute"); - if let Meta::List(list) = meta { - Some(list) - } else { - panic!( - "#[py_class] attribute on a method should be a list, like \ - #[py_class(...)]" - ) + // remove #[pymethod] because there's no actual proc macro + // implementation for it + pymethod_to_remove.push(i); + match meta { + Meta::List(list) => Some(list), + Meta::Word(_) => None, + Meta::NameValue(_) => panic!( + "#[pymethod = ...] attribute on a method should be a list, like \ + #[pymethod(...)]" + ), } } else { None } }) .flat_map(|attr| attr.nested); - for meta in metas_iter { + for meta in metas { if let NestedMeta::Meta(meta) = meta { match meta { Meta::NameValue(name_value) => { @@ -308,18 +314,27 @@ fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator match ident.to_string().as_str() { - "property" => kind = MethodKind::Property, - _ => {} - }, + Meta::Word(ident) => { + if ident == "property" { + kind = MethodKind::Property + } + } _ => {} } } } + // if there are no #[pymethods]s, then it's not a method, so exclude it from + // the final result + if pymethod_to_remove.is_empty() { + return None; + } + for i in pymethod_to_remove { + meth.attrs.remove(i); + } let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); Some(Method { fn_name: meth.sig.ident.clone(), @@ -333,7 +348,7 @@ fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator TokenStream2 { - let imp = if let Item::Impl(imp) = item { + let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); @@ -347,13 +362,13 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { if let Lit::Str(s) = name_value.lit { class_name = Some(s.value()); } else { - panic!("#[py_class(name = ...)] must be a string"); + panic!("#[pyclass(name = ...)] must be a string"); } } } } } - let class_name = class_name.expect("#[py_class] must have a name"); + let class_name = class_name.expect("#[pyclass] must have a name"); let mut doc: Option> = None; for attr in imp.attrs.iter() { if attr.path.is_ident("doc") { @@ -380,8 +395,9 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } None => quote!(None), }; + let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); let ty = &imp.self_ty; - let methods = item_impl_to_methods(&imp).map( + let methods = methods.iter().map( |Method { py_name, fn_name, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 5e4035c5a..70c420665 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -48,19 +48,7 @@ impl TryIntoRef for &str { } } -fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { - let rep_str = match rep { - OptionalArg::Present(ref st) => &st.value, - OptionalArg::Missing => " ", - }; - if rep_str.len() == 1 { - Ok(rep_str) - } else { - Err(vm.new_type_error("The fill character must be exactly one character long".to_string())) - } -} - -#[py_class(__inside_vm, name = "str")] +#[pyclass(__inside_vm, name = "str")] /// str(object='') -> str /// str(bytes_or_buffer[, encoding[, errors]]) -> str /// @@ -75,7 +63,7 @@ impl PyStringRef { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') - #[py_class(name = "__new__")] + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, object: OptionalArg, @@ -92,7 +80,7 @@ impl PyStringRef { payload.clone().into_ref_with_type(vm, cls) } } - #[py_class(name = "__add__")] + #[pymethod(name = "__add__")] fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) @@ -101,12 +89,12 @@ impl PyStringRef { } } - #[py_class(name = "__bool__")] + #[pymethod(name = "__bool__")] fn bool(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() } - #[py_class(name = "__eq__")] + #[pymethod(name = "__eq__")] fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) @@ -115,17 +103,17 @@ impl PyStringRef { } } - #[py_class(name = "__contains__")] + #[pymethod(name = "__contains__")] fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } - #[py_class(name = "__getitem__")] + #[pymethod(name = "__getitem__")] fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } - #[py_class(name = "__gt__")] + #[pymethod(name = "__gt__")] fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) @@ -134,7 +122,7 @@ impl PyStringRef { } } - #[py_class(name = "__ge__")] + #[pymethod(name = "__ge__")] fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) @@ -143,7 +131,7 @@ impl PyStringRef { } } - #[py_class(name = "__lt__")] + #[pymethod(name = "__lt__")] fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) @@ -152,7 +140,7 @@ impl PyStringRef { } } - #[py_class(name = "__le__")] + #[pymethod(name = "__le__")] fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) @@ -161,19 +149,19 @@ impl PyStringRef { } } - #[py_class(name = "__hash__")] + #[pymethod(name = "__hash__")] fn hash(self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() as usize } - #[py_class(name = "__len__")] + #[pymethod(name = "__len__")] fn len(self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } - #[py_class(name = "__mul__")] + #[pymethod(name = "__mul__")] fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; @@ -188,12 +176,12 @@ impl PyStringRef { } } - #[py_class(name = "__str__")] + #[pymethod(name = "__str__")] fn str(self, _vm: &VirtualMachine) -> PyStringRef { self } - #[py_class(name = "__repr__")] + #[pymethod(name = "__repr__")] fn repr(self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { @@ -224,24 +212,29 @@ impl PyStringRef { formatted } + #[pymethod] fn lower(self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower + #[pymethod] fn casefold(self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } + #[pymethod] fn upper(self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } + #[pymethod] fn capitalize(self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } + #[pymethod] fn split( self, pattern: OptionalArg, @@ -263,6 +256,7 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn rsplit( self, pattern: OptionalArg, @@ -284,18 +278,22 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn strip(self, _vm: &VirtualMachine) -> String { self.value.trim().to_string() } + #[pymethod] fn lstrip(self, _vm: &VirtualMachine) -> String { self.value.trim_start().to_string() } + #[pymethod] fn rstrip(self, _vm: &VirtualMachine) -> String { self.value.trim_end().to_string() } + #[pymethod] fn endswith( self, suffix: PyStringRef, @@ -310,6 +308,7 @@ impl PyStringRef { } } + #[pymethod] fn startswith( self, prefix: PyStringRef, @@ -324,14 +323,17 @@ impl PyStringRef { } } + #[pymethod] fn isalnum(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn isnumeric(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } + #[pymethod] fn isdigit(self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ @@ -348,6 +350,7 @@ impl PyStringRef { } } + #[pymethod] fn isdecimal(self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false @@ -356,6 +359,7 @@ impl PyStringRef { } } + #[pymethod] fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if args.args.is_empty() { return Err(vm.new_type_error( @@ -384,10 +388,12 @@ impl PyStringRef { } } + #[pymethod] fn title(self, _vm: &VirtualMachine) -> String { make_title(&self.value) } + #[pymethod] fn swapcase(self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { @@ -403,10 +409,12 @@ impl PyStringRef { swapped_str } + #[pymethod] fn isalpha(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn replace( self, old: PyStringRef, @@ -422,10 +430,12 @@ impl PyStringRef { // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower + #[pymethod] fn isspace(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } + #[pymethod] fn isupper(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self @@ -435,6 +445,7 @@ impl PyStringRef { .all(char::is_uppercase) } + #[pymethod] fn islower(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self @@ -444,11 +455,13 @@ impl PyStringRef { .all(char::is_lowercase) } + #[pymethod] fn isascii(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet + #[pymethod] fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value @@ -458,6 +471,7 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); @@ -472,6 +486,7 @@ impl PyStringRef { Ok(joined) } + #[pymethod] fn find( self, sub: PyStringRef, @@ -490,6 +505,7 @@ impl PyStringRef { } } + #[pymethod] fn rfind( self, sub: PyStringRef, @@ -508,6 +524,7 @@ impl PyStringRef { } } + #[pymethod] fn index( self, sub: PyStringRef, @@ -526,6 +543,7 @@ impl PyStringRef { } } + #[pymethod] fn rindex( self, sub: PyStringRef, @@ -544,6 +562,7 @@ impl PyStringRef { } } + #[pymethod] fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; @@ -562,6 +581,7 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } + #[pymethod] fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; @@ -581,6 +601,7 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } + #[pymethod] fn istitle(self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false @@ -589,6 +610,7 @@ impl PyStringRef { } } + #[pymethod] fn count( self, sub: PyStringRef, @@ -604,6 +626,7 @@ impl PyStringRef { } } + #[pymethod] fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { @@ -613,6 +636,21 @@ impl PyStringRef { } } + fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + let rep_str = match rep { + OptionalArg::Present(ref st) => &st.value, + OptionalArg::Missing => " ", + }; + if rep_str.len() == 1 { + Ok(rep_str) + } else { + Err(vm.new_type_error( + "The fill character must be exactly one character long".to_string(), + )) + } + } + + #[pymethod] fn ljust( self, len: usize, @@ -620,10 +658,11 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } + #[pymethod] fn rjust( self, len: usize, @@ -631,10 +670,11 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } + #[pymethod] fn center( self, len: usize, @@ -642,7 +682,7 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; Ok(format!( @@ -653,6 +693,7 @@ impl PyStringRef { )) } + #[pymethod] fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); @@ -676,6 +717,7 @@ impl PyStringRef { expanded_str } + #[pymethod] fn isidentifier(self, _vm: &VirtualMachine) -> bool { let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number From c2d04f97d80a0eb2916ac7c4ce630b43a528f26b Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 30 Mar 2019 13:33:50 -0700 Subject: [PATCH 059/658] bytes: add tests for NotImplemented --- tests/snippets/bytes.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/snippets/bytes.py diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py new file mode 100644 index 000000000..19ebef68a --- /dev/null +++ b/tests/snippets/bytes.py @@ -0,0 +1,6 @@ +assert b'foobar'.__eq__(2) == NotImplemented +assert b'foobar'.__ne__(2) == NotImplemented +assert b'foobar'.__gt__(2) == NotImplemented +assert b'foobar'.__ge__(2) == NotImplemented +assert b'foobar'.__lt__(2) == NotImplemented +assert b'foobar'.__le__(2) == NotImplemented From 7d18a1ce604a8a36d57e3b3caf34c7816ee545b3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 30 Mar 2019 21:47:20 +0100 Subject: [PATCH 060/658] Fix #746: invalid slice with start or stop =-1 when step<0 --- tests/snippets/builtin_slice.py | 75 +- tests/snippets/slice_res.py | 2252 +++++++++++++++++++++++++++++++ vm/src/obj/objsequence.rs | 21 +- 3 files changed, 2334 insertions(+), 14 deletions(-) create mode 100644 tests/snippets/slice_res.py diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 1d8a93b47..619356211 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -2,24 +2,24 @@ from testutils import assert_raises a = [] assert a[:] == [] -assert a[:2**100] == [] -assert a[-2**100:] == [] -assert a[::2**100] == [] +assert a[: 2 ** 100] == [] +assert a[-2 ** 100 :] == [] +assert a[:: 2 ** 100] == [] assert a[10:20] == [] assert a[-20:-10] == [] b = [1, 2] assert b[:] == [1, 2] -assert b[:2**100] == [1, 2] -assert b[-2**100:] == [1, 2] -assert b[2**100:] == [] -assert b[::2**100] == [1] +assert b[: 2 ** 100] == [1, 2] +assert b[-2 ** 100 :] == [1, 2] +assert b[2 ** 100 :] == [] +assert b[:: 2 ** 100] == [1] assert b[-10:1] == [1] assert b[0:0] == [] assert b[1:0] == [] -assert_raises(ValueError, lambda: b[::0], 'zero step slice') +assert_raises(ValueError, lambda: b[::0], "zero step slice") assert b[::-1] == [2, 1] assert b[1::-1] == [2, 1] @@ -33,7 +33,7 @@ c = list(range(10)) assert c[9:6:-3] == [9] assert c[9::-3] == [9, 6, 3, 0] assert c[9::-4] == [9, 5, 1] -assert c[8::-2**100] == [8] +assert c[8 :: -2 ** 100] == [8] assert c[7:7:-2] == [] assert c[7:8:-2] == [] @@ -43,6 +43,7 @@ d = "123456" assert d[3::-1] == "4321" assert d[4::-3] == "52" +assert [1, 2, 3, 5, 6][-1:-5:-1] == [6, 5, 3, 2] # #746 slice_a = slice(5) assert slice_a.start is None @@ -71,3 +72,59 @@ class SubScript(object): ss = SubScript() _ = ss[:] ss[:1] = 1 + + +def test_all_slices(): + """ + test all possible slices except big number + """ + MODE = None # set to "build" to rebuild slice_res.py + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + + for i in [start, end, step]: + i.append(None) + + def build(): + # loop used to build slices_res.py with cpython + with open("slice_res.py", "wt") as f: + for s in start: + for e in end: + for t in step: + f.write(str(ll[s:e:t]) + "\n") + + def run(): + # test utility + from slice_res import SLICES_RES + + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == SLICES_RES[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, SLICES_RES[count] + ) + ) + count += 1 + + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") + + if MODE == "build": + build() + else: + run() + + +test_all_slices() diff --git a/tests/snippets/slice_res.py b/tests/snippets/slice_res.py new file mode 100644 index 000000000..38eb87729 --- /dev/null +++ b/tests/snippets/slice_res.py @@ -0,0 +1,2252 @@ +SLICES_RES = [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [2, 3], + [2], + [2], + [2], + [2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [3], + [3], + [3], + [3], + [3], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [2, 3], + [2], + [2], + [2], + [2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [0], + [0], + [0], + [0], + [0], + [3], + [3], + [3], + [3], + [3], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [0], + [0], + [0], + [0], + [0], + [3], + [3], + [3], + [3], + [3], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], +] diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a07b2454d..55c56d47c 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -2,11 +2,10 @@ use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; - use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use super::objbool; use super::objint::PyInt; @@ -86,8 +85,20 @@ pub trait PySliceableSequence { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { From 5625f7e15e49772cb906ba23335356d341a470a4 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 31 Mar 2019 09:29:21 +1300 Subject: [PATCH 061/658] Allow arbitrary in slice, and convert slice.__new__ to new style --- tests/snippets/builtin_slice.py | 6 ++ vm/src/frame.rs | 27 ++----- vm/src/obj/objrange.rs | 6 +- vm/src/obj/objsequence.rs | 11 +-- vm/src/obj/objslice.rs | 127 +++++++++++++++++++------------- 5 files changed, 99 insertions(+), 78 deletions(-) diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 1d8a93b47..03f800fba 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -59,6 +59,12 @@ assert slice_c.start == 1 assert slice_c.stop == 5 assert slice_c.step == 2 +a = object() +slice_d = slice(a, "v", 1.0) +assert slice_d.start is a +assert slice_d.stop == "v" +assert slice_d.step == 1.0 + class SubScript(object): def __getitem__(self, item): diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 88a619eb3..3a5f4c5ab 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -2,8 +2,6 @@ use std::cell::RefCell; use std::fmt; use std::rc::Rc; -use num_bigint::BigInt; - use rustpython_parser::ast; use crate::builtins; @@ -13,7 +11,6 @@ use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::{self, PyDictRef}; -use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; @@ -404,24 +401,14 @@ impl Frame { } bytecode::Instruction::BuildSlice { size } => { assert!(*size == 2 || *size == 3); - let elements = self.pop_multiple(*size); - let mut out: Vec> = elements - .into_iter() - .map(|x| { - if x.is(&vm.ctx.none()) { - None - } else if let Some(i) = x.payload::() { - Some(i.as_bigint().clone()) - } else { - panic!("Expect Int or None as BUILD_SLICE arguments") - } - }) - .collect(); - - let start = out[0].take(); - let stop = out[1].take(); - let step = if out.len() == 3 { out[2].take() } else { None }; + let step = if *size == 3 { + Some(self.pop_value()) + } else { + None + }; + let stop = Some(self.pop_value()); + let start = Some(self.pop_value()); let obj = PySlice { start, stop, step }.into_ref(vm); self.push_value(obj.into_object()); diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 04c9c273b..c1a0b9401 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -264,7 +264,7 @@ impl PyRangeRef { } } Either::B(slice) => { - let new_start = if let Some(int) = slice.start.as_ref() { + let new_start = if let Some(int) = slice.start_index(vm)? { if let Some(i) = self.get(int) { PyInt::new(i).into_ref(vm) } else { @@ -274,7 +274,7 @@ impl PyRangeRef { self.start.clone() }; - let new_end = if let Some(int) = slice.stop.as_ref() { + let new_end = if let Some(int) = slice.stop_index(vm)? { if let Some(i) = self.get(int) { PyInt::new(i).into_ref(vm) } else { @@ -284,7 +284,7 @@ impl PyRangeRef { self.stop.clone() }; - let new_step = if let Some(int) = slice.step.as_ref() { + let new_step = if let Some(int) = slice.step_index(vm)? { PyInt::new(int * self.step.as_bigint()).into_ref(vm) } else { self.step.clone() diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a07b2454d..a293314be 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -65,14 +65,15 @@ pub trait PySliceableSequence { where Self: Sized, { - // TODO: we could potentially avoid this copy and use slice - match slice.payload() { - Some(PySlice { start, stop, step }) => { - let step = step.clone().unwrap_or_else(BigInt::one); + match slice.clone().downcast::() { + Ok(slice) => { + let start = slice.start_index(vm)?; + let stop = slice.stop_index(vm)?; + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); if step.is_zero() { Err(vm.new_value_error("slice step cannot be zero".to_string())) } else if step.is_positive() { - let range = self.get_slice_range(start, stop); + let range = self.get_slice_range(&start, &stop); if range.start < range.end { #[allow(clippy::range_plus_one)] match step.to_i32() { diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index c82a3ca6e..3eee8e4ec 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,18 +1,19 @@ -use num_bigint::BigInt; - use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; -use super::objint; -use crate::obj::objtype::PyClassRef; +use crate::obj::objint::PyInt; +use crate::obj::objtype::{class_has_attr, PyClassRef}; +use num_bigint::BigInt; -#[derive(Debug)] +#[derive(Debug, FromArgs)] pub struct PySlice { - // TODO: should be private - pub start: Option, - pub stop: Option, - pub step: Option, + #[pyarg(positional_only)] + pub start: Option, + #[pyarg(positional_only)] + pub stop: Option, + #[pyarg(positional_only, default)] + pub step: Option, } impl PyValue for PySlice { @@ -24,51 +25,29 @@ impl PyValue for PySlice { pub type PySliceRef = PyRef; fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - no_kwargs!(vm, args); - let (cls, start, stop, step): ( - &PyObjectRef, - Option<&PyObjectRef>, - Option<&PyObjectRef>, - Option<&PyObjectRef>, - ) = match args.args.len() { - 0 | 1 => Err(vm.new_type_error("slice() must have at least one arguments.".to_owned())), + let (cls, slice): (PyClassRef, PySlice) = match args.args.len() { + 0 | 1 => { + return Err(vm.new_type_error("slice() must have at least one arguments.".to_owned())); + } 2 => { - arg_check!( - vm, - args, - required = [ - (cls, Some(vm.ctx.type_type())), - (stop, Some(vm.ctx.int_type())) - ] - ); - Ok((cls, None, Some(stop), None)) + let (cls, stop) = args.bind(vm)?; + ( + cls, + PySlice { + start: None, + stop: Some(stop), + step: None, + }, + ) } - _ => { - arg_check!( - vm, - args, - required = [ - (cls, Some(vm.ctx.type_type())), - (start, Some(vm.ctx.int_type())), - (stop, Some(vm.ctx.int_type())) - ], - optional = [(step, Some(vm.ctx.int_type()))] - ); - Ok((cls, Some(start), Some(stop), step)) - } - }?; - PySlice { - start: start.map(|x| objint::get_value(x).clone()), - stop: stop.map(|x| objint::get_value(x).clone()), - step: step.map(|x| objint::get_value(x).clone()), - } - .into_ref_with_type(vm, cls.clone().downcast().unwrap()) - .map(|x| x.into_object()) + _ => args.bind(vm)?, + }; + slice.into_ref_with_type(vm, cls).map(|x| x.into_object()) } -fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { +fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { if let Some(value) = value { - vm.ctx.new_int(value.clone()) + value.clone() } else { vm.get_none() } @@ -86,6 +65,54 @@ impl PySliceRef { fn step(self, vm: &VirtualMachine) -> PyObjectRef { get_property_value(vm, &self.step) } + + pub fn start_index(&self, vm: &VirtualMachine) -> PyResult> { + if let Some(obj) = &self.start { + to_index_value(vm, obj) + } else { + Ok(None) + } + } + + pub fn stop_index(&self, vm: &VirtualMachine) -> PyResult> { + if let Some(obj) = &self.stop { + to_index_value(vm, obj) + } else { + Ok(None) + } + } + + pub fn step_index(&self, vm: &VirtualMachine) -> PyResult> { + if let Some(obj) = &self.step { + to_index_value(vm, obj) + } else { + Ok(None) + } + } +} + +fn to_index_value(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult> { + if obj.is(&vm.ctx.none) { + return Ok(None); + } + + if let Some(val) = obj.payload::() { + Ok(Some(val.as_bigint().clone())) + } else { + let cls = obj.class(); + if class_has_attr(&cls, "__index__") { + let index_result = vm.call_method(obj, "__index__", vec![])?; + if let Some(val) = index_result.payload::() { + Ok(Some(val.as_bigint().clone())) + } else { + Err(vm.new_type_error("__index__ method returned non integer".to_string())) + } + } else { + Err(vm.new_type_error( + "slice indices must be integers or None or have an __index__ method".to_string(), + )) + } + } } pub fn init(context: &PyContext) { From a49895ef63a8a7d7f83ddbfae6e44b6abca9c450 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 31 Mar 2019 09:55:18 +1300 Subject: [PATCH 062/658] Cleanup slice_new and added more slice tests --- tests/snippets/builtin_slice.py | 13 +++++++++++++ vm/src/function.rs | 7 +++++++ vm/src/obj/objslice.rs | 25 +++++++++++-------------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 03f800fba..c5dbd5d90 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -11,6 +11,7 @@ assert a[-20:-10] == [] b = [1, 2] assert b[:] == [1, 2] +assert b[slice(None)] == [1, 2] assert b[:2**100] == [1, 2] assert b[-2**100:] == [1, 2] assert b[2**100:] == [] @@ -77,3 +78,15 @@ class SubScript(object): ss = SubScript() _ = ss[:] ss[:1] = 1 + + +class CustomIndex: + def __init__(self, x): + self.x = x + + def __index__(self): + return self.x + + +assert c[CustomIndex(1):CustomIndex(3)] == [1, 2] +assert d[CustomIndex(1):CustomIndex(3)] == "23" diff --git a/vm/src/function.rs b/vm/src/function.rs index aae4eefe8..921534549 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::mem; use std::ops::RangeInclusive; use crate::obj::objtype::{isinstance, PyClassRef}; @@ -46,6 +47,12 @@ impl From<(&Args, &KwArgs)> for PyFuncArgs { } } +impl FromArgs for PyFuncArgs { + fn from_args(_vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { + Ok(mem::replace(args, Default::default())) + } +} + impl PyFuncArgs { pub fn new(mut args: Vec, kwarg_names: Vec) -> PyFuncArgs { let mut kwargs = vec![]; diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 3eee8e4ec..352ba5ce5 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -24,25 +24,22 @@ impl PyValue for PySlice { pub type PySliceRef = PyRef; -fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let (cls, slice): (PyClassRef, PySlice) = match args.args.len() { - 0 | 1 => { +fn slice_new(cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let slice: PySlice = match args.args.len() { + 0 => { return Err(vm.new_type_error("slice() must have at least one arguments.".to_owned())); } - 2 => { - let (cls, stop) = args.bind(vm)?; - ( - cls, - PySlice { - start: None, - stop: Some(stop), - step: None, - }, - ) + 1 => { + let stop = args.bind(vm)?; + PySlice { + start: None, + stop: Some(stop), + step: None, + } } _ => args.bind(vm)?, }; - slice.into_ref_with_type(vm, cls).map(|x| x.into_object()) + slice.into_ref_with_type(vm, cls) } fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { From e0f7fbb191490a7ee27b34bb411d2f62f5f96fe1 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 31 Mar 2019 10:53:45 +1300 Subject: [PATCH 063/658] Don't derive FromArgs on payload type. --- vm/src/obj/objslice.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 352ba5ce5..f6f02c8db 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,4 +1,4 @@ -use crate::function::PyFuncArgs; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; @@ -6,13 +6,10 @@ use crate::obj::objint::PyInt; use crate::obj::objtype::{class_has_attr, PyClassRef}; use num_bigint::BigInt; -#[derive(Debug, FromArgs)] +#[derive(Debug)] pub struct PySlice { - #[pyarg(positional_only)] pub start: Option, - #[pyarg(positional_only)] pub stop: Option, - #[pyarg(positional_only, default)] pub step: Option, } @@ -37,7 +34,15 @@ fn slice_new(cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult step: None, } } - _ => args.bind(vm)?, + _ => { + let (start, stop, step): (PyObjectRef, PyObjectRef, OptionalArg) = + args.bind(vm)?; + PySlice { + start: Some(start), + stop: Some(stop), + step: step.into_option(), + } + } }; slice.into_ref_with_type(vm, cls) } From f11f04da709d564114d8fcaec07abb130c7de214 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 18:43:31 -0500 Subject: [PATCH 064/658] Split rustpython_derive into multiple files --- derive/src/from_args.rs | 202 +++++++++++++++++++++ derive/src/lib.rs | 382 +--------------------------------------- derive/src/pyclass.rs | 179 +++++++++++++++++++ vm/src/lib.rs | 4 +- 4 files changed, 390 insertions(+), 377 deletions(-) create mode 100644 derive/src/from_args.rs create mode 100644 derive/src/pyclass.rs diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs new file mode 100644 index 000000000..7845b2880 --- /dev/null +++ b/derive/src/from_args.rs @@ -0,0 +1,202 @@ +use super::rustpython_path_derive; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; + +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +enum ParameterKind { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, +} + +impl ParameterKind { + fn from_ident(ident: &Ident) -> ParameterKind { + if ident == "positional_only" { + ParameterKind::PositionalOnly + } else if ident == "positional_or_keyword" { + ParameterKind::PositionalOrKeyword + } else if ident == "keyword_only" { + ParameterKind::KeywordOnly + } else { + panic!("Unrecognised attribute") + } + } +} + +struct ArgAttribute { + kind: ParameterKind, + default: Option, + optional: bool, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option { + if !attr.path.is_ident("pyarg") { + return None; + } + + match attr.parse_meta().unwrap() { + Meta::List(list) => { + let mut iter = list.nested.iter(); + let first_arg = iter.next().expect("at least one argument in pyarg list"); + let kind = match first_arg { + NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + + let mut attribute = ArgAttribute { + kind, + default: None, + optional: false, + }; + + while let Some(arg) = iter.next() { + attribute.parse_argument(arg); + } + + assert!( + attribute.default.is_none() || !attribute.optional, + "Can't set both a default value and optional" + ); + + Some(attribute) + } + _ => panic!("Bad syntax for pyarg attribute"), + } + } + + fn parse_argument(&mut self, arg: &NestedMeta) { + match arg { + NestedMeta::Meta(Meta::Word(ident)) => { + if ident == "default" { + assert!(self.default.is_none(), "Default already set"); + let expr = syn::parse_str::("Default::default()").unwrap(); + self.default = Some(expr); + } else if ident == "optional" { + self.optional = true; + } else { + panic!("Unrecognised pyarg attribute '{}'", ident); + } + } + NestedMeta::Meta(Meta::NameValue(name_value)) => { + if name_value.ident == "default" { + assert!(self.default.is_none(), "Default already set"); + + match name_value.lit { + Lit::Str(ref val) => { + let expr = val + .parse::() + .expect("a valid expression for default argument"); + self.default = Some(expr); + } + _ => panic!("Expected string value for default argument"), + } + } else if name_value.ident == "optional" { + match name_value.lit { + Lit::Bool(ref val) => { + self.optional = val.value; + } + _ => panic!("Expected boolean value for optional argument"), + } + } else { + panic!("Unrecognised pyarg attribute '{}'", name_value.ident); + } + } + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + } +} + +fn generate_field(field: &Field) -> TokenStream2 { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(ArgAttribute::from_attribute) + .collect::>(); + let attr = if pyarg_attrs.is_empty() { + ArgAttribute { + kind: ParameterKind::PositionalOrKeyword, + default: None, + optional: false, + } + } else if pyarg_attrs.len() == 1 { + pyarg_attrs.remove(0) + } else { + panic!( + "Multiple pyarg attributes on field '{}'", + field.ident.as_ref().unwrap() + ); + }; + + let name = &field.ident; + let middle = quote! { + .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + }; + let ending = if let Some(default) = attr.default { + quote! { + .unwrap_or_else(|| #default) + } + } else if attr.optional { + quote! { + .map(crate::function::OptionalArg::Present) + .unwrap_or(crate::function::OptionalArg::Missing) + } + } else { + let err = match attr.kind { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { + crate::function::ArgumentError::TooFewArgs + }, + ParameterKind::KeywordOnly => quote! { + crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + }, + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + match attr.kind { + ParameterKind::PositionalOnly => { + quote! { + #name: args.take_positional()#middle#ending, + } + } + ParameterKind::PositionalOrKeyword => { + quote! { + #name: args.take_positional_keyword(stringify!(#name))#middle#ending, + } + } + ParameterKind::KeywordOnly => { + quote! { + #name: args.take_keyword(stringify!(#name))#middle#ending, + } + } + } +} + +pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { + let rp_path = rustpython_path_derive(&input); + let fields = match input.data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => fields.named.iter().map(generate_field), + Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message + }; + + let name = &input.ident; + quote! { + impl #rp_path::function::FromArgs for #name { + fn from_args( + vm: &crate::vm::VirtualMachine, + args: &mut crate::function::PyFuncArgs + ) -> Result { + Ok(#name { #(#fields)* }) + } + } + } +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 374463f06..5a26030e4 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,12 +1,11 @@ extern crate proc_macro; use proc_macro::TokenStream; -use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; -use syn::{ - parse_macro_input, Attribute, AttributeArgs, Data, DeriveInput, Expr, Field, Fields, Ident, - ImplItem, Item, Lit, Meta, NestedMeta, -}; +use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; + +mod from_args; +mod pyclass; fn rustpython_path(inside_vm: bool) -> syn::Path { let path = if inside_vm { @@ -46,381 +45,12 @@ fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); - let gen = impl_from_args(ast); - gen.to_string().parse().unwrap() -} - -/// The kind of the python parameter, this corresponds to the value of Parameter.kind -/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) -enum ParameterKind { - PositionalOnly, - PositionalOrKeyword, - KeywordOnly, -} - -impl ParameterKind { - fn from_ident(ident: &Ident) -> ParameterKind { - if ident == "positional_only" { - ParameterKind::PositionalOnly - } else if ident == "positional_or_keyword" { - ParameterKind::PositionalOrKeyword - } else if ident == "keyword_only" { - ParameterKind::KeywordOnly - } else { - panic!("Unrecognised attribute") - } - } -} - -struct ArgAttribute { - kind: ParameterKind, - default: Option, - optional: bool, -} - -impl ArgAttribute { - fn from_attribute(attr: &Attribute) -> Option { - if !attr.path.is_ident("pyarg") { - return None; - } - - match attr.parse_meta().unwrap() { - Meta::List(list) => { - let mut iter = list.nested.iter(); - let first_arg = iter.next().expect("at least one argument in pyarg list"); - let kind = match first_arg { - NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - - let mut attribute = ArgAttribute { - kind, - default: None, - optional: false, - }; - - while let Some(arg) = iter.next() { - attribute.parse_argument(arg); - } - - assert!( - attribute.default.is_none() || !attribute.optional, - "Can't set both a default value and optional" - ); - - Some(attribute) - } - _ => panic!("Bad syntax for pyarg attribute"), - } - } - - fn parse_argument(&mut self, arg: &NestedMeta) { - match arg { - NestedMeta::Meta(Meta::Word(ident)) => { - if ident == "default" { - assert!(self.default.is_none(), "Default already set"); - let expr = syn::parse_str::("Default::default()").unwrap(); - self.default = Some(expr); - } else if ident == "optional" { - self.optional = true; - } else { - panic!("Unrecognised pyarg attribute '{}'", ident); - } - } - NestedMeta::Meta(Meta::NameValue(name_value)) => { - if name_value.ident == "default" { - assert!(self.default.is_none(), "Default already set"); - - match name_value.lit { - Lit::Str(ref val) => { - let expr = val - .parse::() - .expect("a valid expression for default argument"); - self.default = Some(expr); - } - _ => panic!("Expected string value for default argument"), - } - } else if name_value.ident == "optional" { - match name_value.lit { - Lit::Bool(ref val) => { - self.optional = val.value; - } - _ => panic!("Expected boolean value for optional argument"), - } - } else { - panic!("Unrecognised pyarg attribute '{}'", name_value.ident); - } - } - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - } -} - -fn generate_field(field: &Field) -> TokenStream2 { - let mut pyarg_attrs = field - .attrs - .iter() - .filter_map(ArgAttribute::from_attribute) - .collect::>(); - let attr = if pyarg_attrs.is_empty() { - ArgAttribute { - kind: ParameterKind::PositionalOrKeyword, - default: None, - optional: false, - } - } else if pyarg_attrs.len() == 1 { - pyarg_attrs.remove(0) - } else { - panic!( - "Multiple pyarg attributes on field '{}'", - field.ident.as_ref().unwrap() - ); - }; - - let name = &field.ident; - let middle = quote! { - .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? - }; - let ending = if let Some(default) = attr.default { - quote! { - .unwrap_or_else(|| #default) - } - } else if attr.optional { - quote! { - .map(crate::function::OptionalArg::Present) - .unwrap_or(crate::function::OptionalArg::Missing) - } - } else { - let err = match attr.kind { - ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - crate::function::ArgumentError::TooFewArgs - }, - ParameterKind::KeywordOnly => quote! { - crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) - }, - }; - quote! { - .ok_or_else(|| #err)? - } - }; - - match attr.kind { - ParameterKind::PositionalOnly => { - quote! { - #name: args.take_positional()#middle#ending, - } - } - ParameterKind::PositionalOrKeyword => { - quote! { - #name: args.take_positional_keyword(stringify!(#name))#middle#ending, - } - } - ParameterKind::KeywordOnly => { - quote! { - #name: args.take_keyword(stringify!(#name))#middle#ending, - } - } - } -} - -fn impl_from_args(input: DeriveInput) -> TokenStream2 { - let rp_path = rustpython_path_derive(&input); - let fields = match input.data { - Data::Struct(ref data) => { - match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(generate_field), - Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message - } - } - Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message - }; - - let name = &input.ident; - quote! { - impl #rp_path::function::FromArgs for #name { - fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { - Ok(#name { #(#fields)* }) - } - } - } + from_args::impl_from_args(ast).into() } #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - impl_py_class(attr, item).into() -} - -enum MethodKind { - Method, - Property, -} - -impl MethodKind { - fn to_ctx_constructor_fn(&self) -> Ident { - let f = match self { - MethodKind::Method => "new_rustfunc", - MethodKind::Property => "new_property", - }; - Ident::new(f, Span::call_site()) - } -} - -struct Method { - fn_name: Ident, - py_name: String, - kind: MethodKind, -} - -/// Parse an impl block into an iterator of methods -fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter_mut().filter_map(|item| { - if let ImplItem::Method(meth) = item { - let mut py_name = None; - let mut kind = MethodKind::Method; - let mut pymethod_to_remove = Vec::new(); - let metas = meth - .attrs - .iter() - .enumerate() - .filter_map(|(i, attr)| { - if attr.path.is_ident("pymethod") { - let meta = attr.parse_meta().expect("Invalid attribute"); - // remove #[pymethod] because there's no actual proc macro - // implementation for it - pymethod_to_remove.push(i); - match meta { - Meta::List(list) => Some(list), - Meta::Word(_) => None, - Meta::NameValue(_) => panic!( - "#[pymethod = ...] attribute on a method should be a list, like \ - #[pymethod(...)]" - ), - } - } else { - None - } - }) - .flat_map(|attr| attr.nested); - for meta in metas { - if let NestedMeta::Meta(meta) = meta { - match meta { - Meta::NameValue(name_value) => { - if name_value.ident == "name" { - if let Lit::Str(s) = &name_value.lit { - py_name = Some(s.value()); - } else { - panic!("#[pymethod(name = ...)] must be a string"); - } - } - } - Meta::Word(ident) => { - if ident == "property" { - kind = MethodKind::Property - } - } - _ => {} - } - } - } - // if there are no #[pymethods]s, then it's not a method, so exclude it from - // the final result - if pymethod_to_remove.is_empty() { - return None; - } - for i in pymethod_to_remove { - meth.attrs.remove(i); - } - let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); - Some(Method { - fn_name: meth.sig.ident.clone(), - py_name, - kind, - }) - } else { - None - } - }) -} - -fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { - let mut imp = if let Item::Impl(imp) = item { - imp - } else { - return quote!(#item); - }; - let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; - for attr in attr { - if let NestedMeta::Meta(meta) = attr { - if let Meta::NameValue(name_value) = meta { - if name_value.ident == "name" { - if let Lit::Str(s) = name_value.lit { - class_name = Some(s.value()); - } else { - panic!("#[pyclass(name = ...)] must be a string"); - } - } - } - } - } - let class_name = class_name.expect("#[pyclass] must have a name"); - let mut doc: Option> = None; - for attr in imp.attrs.iter() { - if attr.path.is_ident("doc") { - let meta = attr.parse_meta().expect("expected doc attr to be a meta"); - if let Meta::NameValue(name_value) = meta { - if let Lit::Str(s) = name_value.lit { - let val = s.value().trim().to_string(); - match doc { - Some(ref mut doc) => doc.push(val), - None => doc = Some(vec![val]), - } - } else { - panic!("expected #[doc = ...] to be a string") - } - } else { - panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); - } - } - } - let doc = match doc { - Some(doc) => { - let doc = doc.join("\n"); - quote!(Some(#doc)) - } - None => quote!(None), - }; - let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); - let ty = &imp.self_ty; - let methods = methods.iter().map( - |Method { - py_name, - fn_name, - kind, - }| { - let constructor_fn = kind.to_ctx_constructor_fn(); - quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); - } - }, - ); - - quote! { - #imp - impl #rp_path::pyobject::IntoPyClass for #ty { - const NAME: &'static str = #class_name; - const DOC: Option<&'static str> = #doc; - fn _extend_class( - ctx: &#rp_path::pyobject::PyContext, - class: &#rp_path::obj::objtype::PyClassRef, - ) { - #(#methods)* - } - } - } + pyclass::impl_py_class(attr, item).into() } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs new file mode 100644 index 000000000..4497424f2 --- /dev/null +++ b/derive/src/pyclass.rs @@ -0,0 +1,179 @@ +use super::rustpython_path_attr; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; + +enum MethodKind { + Method, + Property, +} + +impl MethodKind { + fn to_ctx_constructor_fn(&self) -> Ident { + let f = match self { + MethodKind::Method => "new_rustfunc", + MethodKind::Property => "new_property", + }; + Ident::new(f, Span::call_site()) + } +} + +struct Method { + fn_name: Ident, + py_name: String, + kind: MethodKind, +} + +impl Method { + fn from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + let mut py_name = None; + let mut kind = MethodKind::Method; + let mut pymethod_to_remove = Vec::new(); + let metas = attrs + .iter() + .enumerate() + .filter_map(|(i, attr)| { + if attr.path.is_ident("pymethod") { + let meta = attr.parse_meta().expect("Invalid attribute"); + // remove #[pymethod] because there's no actual proc macro + // implementation for it + pymethod_to_remove.push(i); + match meta { + Meta::List(list) => Some(list), + Meta::Word(_) => None, + Meta::NameValue(_) => panic!( + "#[pymethod = ...] attribute on a method should be a list, like \ + #[pymethod(...)]" + ), + } + } else { + None + } + }) + .flat_map(|attr| attr.nested); + for meta in metas { + if let NestedMeta::Meta(meta) = meta { + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pymethod(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => { + if ident == "property" { + kind = MethodKind::Property + } + } + _ => {} + } + } + } + // if there are no #[pymethods]s, then it's not a method, so exclude it from + // the final result + if pymethod_to_remove.is_empty() { + return None; + } + for i in pymethod_to_remove { + attrs.remove(i); + } + let py_name = py_name.unwrap_or_else(|| sig.ident.to_string()); + Some(Method { + fn_name: sig.ident.clone(), + py_name, + kind, + }) + } +} + +/// Parse an impl block into an iterator of methods +fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter_mut().filter_map(|item| { + if let ImplItem::Method(meth) = item { + Method::from_syn(&mut meth.attrs, &meth.sig) + } else { + None + } + }) +} + +pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { + let mut imp = if let Item::Impl(imp) = item { + imp + } else { + return quote!(#item); + }; + let rp_path = rustpython_path_attr(&attr); + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[pyclass(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.expect("#[pyclass] must have a name"); + let mut doc: Option> = None; + for attr in imp.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } else { + panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); + let ty = &imp.self_ty; + let methods = methods.iter().map( + |Method { + py_name, + fn_name, + kind, + }| { + let constructor_fn = kind.to_ctx_constructor_fn(); + quote! { + ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); + } + }, + ); + + quote! { + #imp + impl #rp_path::pyobject::IntoPyClass for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + fn _extend_class( + ctx: &#rp_path::pyobject::PyContext, + class: &#rp_path::obj::objtype::PyClassRef, + ) { + #(#methods)* + } + } + } +} diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 29124ad11..4a300dcef 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -30,7 +30,9 @@ extern crate statrs; extern crate rustpython_parser; #[macro_use] -pub extern crate rustpython_derive; +extern crate rustpython_derive; + +pub use rustpython_derive::*; //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; From 15cffc4d2b023ec31aade0c72d8821f72be74265 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:30:14 -0500 Subject: [PATCH 065/658] Clean up a bit --- derive/src/pyclass.rs | 23 ++++++++------- vm/src/obj/objstr.rs | 65 ++----------------------------------------- 2 files changed, 13 insertions(+), 75 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 4497424f2..ad03b58c9 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -89,17 +89,6 @@ impl Method { } } -/// Parse an impl block into an iterator of methods -fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter_mut().filter_map(|item| { - if let ImplItem::Method(meth) = item { - Method::from_syn(&mut meth.attrs, &meth.sig) - } else { - None - } - }) -} - pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp @@ -148,7 +137,17 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } None => quote!(None), }; - let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); + let methods = imp + .items + .iter_mut() + .filter_map(|item| { + if let ImplItem::Method(meth) = item { + Method::from_syn(&mut meth.attrs, &meth.sig) + } else { + None + } + }) + .collect::>(); let ty = &imp.self_ty; let methods = methods.iter().map( |Method { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 70c420665..9520b8af9 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -748,69 +748,8 @@ impl IntoPyObject for String { } } -#[rustfmt::skip] // to avoid line splitting -pub fn init(context: &PyContext) { - let str_type = &context.str_type; - - PyStringRef::extend_class(context, str_type); - // extend_class!(context, str_type, { - // "__add__" => context.new_rustfunc(PyStringRef::add), - // "__bool__" => context.new_rustfunc(PyStringRef::bool), - // "__contains__" => context.new_rustfunc(PyStringRef::contains), - // "__doc__" => context.new_str(str_doc.to_string()), - // "__eq__" => context.new_rustfunc(PyStringRef::eq), - // "__ge__" => context.new_rustfunc(PyStringRef::ge), - // "__getitem__" => context.new_rustfunc(PyStringRef::getitem), - // "__gt__" => context.new_rustfunc(PyStringRef::gt), - // "__hash__" => context.new_rustfunc(PyStringRef::hash), - // "__lt__" => context.new_rustfunc(PyStringRef::lt), - // "__le__" => context.new_rustfunc(PyStringRef::le), - // "__len__" => context.new_rustfunc(PyStringRef::len), - // "__mul__" => context.new_rustfunc(PyStringRef::mul), - // "__new__" => context.new_rustfunc(str_new), - // "__repr__" => context.new_rustfunc(PyStringRef::repr), - // "__str__" => context.new_rustfunc(PyStringRef::str), - // "capitalize" => context.new_rustfunc(PyStringRef::capitalize), - // "casefold" => context.new_rustfunc(PyStringRef::casefold), - // "center" => context.new_rustfunc(PyStringRef::center), - // "count" => context.new_rustfunc(PyStringRef::count), - // "endswith" => context.new_rustfunc(PyStringRef::endswith), - // "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), - // "find" => context.new_rustfunc(PyStringRef::find), - // "format" => context.new_rustfunc(str_format), - // "index" => context.new_rustfunc(PyStringRef::index), - // "isalnum" => context.new_rustfunc(PyStringRef::isalnum), - // "isalpha" => context.new_rustfunc(PyStringRef::isalpha), - // "isascii" => context.new_rustfunc(PyStringRef::isascii), - // "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), - // "isdigit" => context.new_rustfunc(PyStringRef::isdigit), - // "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), - // "islower" => context.new_rustfunc(PyStringRef::islower), - // "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), - // "isspace" => context.new_rustfunc(PyStringRef::isspace), - // "isupper" => context.new_rustfunc(PyStringRef::isupper), - // "istitle" => context.new_rustfunc(PyStringRef::istitle), - // "join" => context.new_rustfunc(PyStringRef::join), - // "lower" => context.new_rustfunc(PyStringRef::lower), - // "ljust" => context.new_rustfunc(PyStringRef::ljust), - // "lstrip" => context.new_rustfunc(PyStringRef::lstrip), - // "partition" => context.new_rustfunc(PyStringRef::partition), - // "replace" => context.new_rustfunc(PyStringRef::replace), - // "rfind" => context.new_rustfunc(PyStringRef::rfind), - // "rindex" => context.new_rustfunc(PyStringRef::rindex), - // "rjust" => context.new_rustfunc(PyStringRef::rjust), - // "rpartition" => context.new_rustfunc(PyStringRef::rpartition), - // "rsplit" => context.new_rustfunc(PyStringRef::rsplit), - // "rstrip" => context.new_rustfunc(PyStringRef::rstrip), - // "split" => context.new_rustfunc(PyStringRef::split), - // "splitlines" => context.new_rustfunc(PyStringRef::splitlines), - // "startswith" => context.new_rustfunc(PyStringRef::startswith), - // "strip" => context.new_rustfunc(PyStringRef::strip), - // "swapcase" => context.new_rustfunc(PyStringRef::swapcase), - // "title" => context.new_rustfunc(PyStringRef::title), - // "upper" => context.new_rustfunc(PyStringRef::upper), - // "zfill" => context.new_rustfunc(PyStringRef::zfill), - // }); +pub fn init(ctx: &PyContext) { + PyStringRef::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { From 1b57f0783466d5e364b4307f6943fa8679426a30 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:56:37 -0500 Subject: [PATCH 066/658] Process lalrpop in cargo's target directory --- parser/.gitignore | 4 ---- parser/build.rs | 5 +---- parser/src/lib.rs | 4 ++-- parser/src/python.lalrpop | 1 - 4 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 parser/.gitignore diff --git a/parser/.gitignore b/parser/.gitignore deleted file mode 100644 index 401e8cc22..000000000 --- a/parser/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -src/python.rs -target/ -Cargo.lock - diff --git a/parser/build.rs b/parser/build.rs index 65147ed0e..19f102721 100644 --- a/parser/build.rs +++ b/parser/build.rs @@ -1,8 +1,5 @@ use lalrpop; fn main() { - lalrpop::Configuration::new() - .generate_in_source_tree() - .process() - .unwrap(); + lalrpop::process_root().unwrap() } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index b10c3d551..4f0105a9a 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,11 +1,11 @@ #[macro_use] extern crate log; +use lalrpop_util::lalrpop_mod; pub mod ast; pub mod error; mod fstring; pub mod lexer; pub mod parser; -#[cfg_attr(rustfmt, rustfmt_skip)] -mod python; +lalrpop_mod!(python); pub mod token; diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 9e5b309b1..437d001fe 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -2,7 +2,6 @@ // See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4 // See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword -#![allow(unknown_lints,clippy)] use std::iter::FromIterator; From 60ad5489f15d8911afead86b4c84907fbe20eeec Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:58:46 -0500 Subject: [PATCH 067/658] Ignore clippy lints in the lalrpop generated file --- parser/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 4f0105a9a..5cc8ca3c4 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -7,5 +7,8 @@ pub mod error; mod fstring; pub mod lexer; pub mod parser; -lalrpop_mod!(python); +lalrpop_mod!( + #[allow(clippy::all)] + python +); pub mod token; From f994f8660e487de93f1ad1080512e4af24e92fee Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:10:24 +0200 Subject: [PATCH 068/658] Function support __doc__ --- tests/snippets/function.py | 3 +++ vm/src/compile.rs | 32 +++++++++++++++++++++++++++++++- vm/src/frame.rs | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index b1ea96860..f327d88cb 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,7 +1,10 @@ def foo(): + """test""" return 42 assert foo() == 42 +assert foo.__doc__ == "test" + def my_func(a,): return a+2 diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 00bafe727..037d9e95c 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -597,7 +597,20 @@ impl Compiler { self.in_loop = false; self.in_function_def = true; let mut flags = self.enter_function(name, args)?; - self.compile_statements(body)?; + + let doc = get_doc(body); + let (new_body, doc_str) = match doc { + Some(val) => { + if let Some((_, body_rest)) = body.split_first() { + (body_rest, val) + } else { + (body, "".to_string()) + } + } + None => (body, "".to_string()), + }; + + self.compile_statements(new_body)?; // Emit None at end: self.emit(Instruction::LoadConst { @@ -644,6 +657,10 @@ impl Compiler { }); } + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { value: doc_str }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1511,6 +1528,19 @@ impl Compiler { } } +fn get_doc(body: &[ast::LocatedStatement]) -> Option { + if let Some(val) = body.get(0) { + if let ast::Statement::Expression { ref expression } = val.node { + if let ast::Expression::String { ref value } = expression { + if let ast::StringGroup::Constant { ref value } = value { + return Some(value.to_string()); + } + } + } + } + None +} + #[cfg(test)] mod tests { use super::Compiler; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 88a619eb3..5a22aabe9 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -568,6 +568,8 @@ impl Frame { .downcast() .expect("Second to top value on the stack must be a code object"); + let doc = self.pop_value(); + let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { @@ -586,6 +588,7 @@ impl Frame { let obj = vm.ctx.new_function(code_obj, scope, defaults); vm.ctx.set_attr(&obj, "__annotations__", annotations); + vm.ctx.set_attr(&obj, "__doc__", doc); self.push_value(obj); Ok(None) From c76b31866d2fae1d6b8112be47b6ccbaca1f5cf5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:15:39 +0200 Subject: [PATCH 069/658] Add more tests for function doc --- tests/snippets/function.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index f327d88cb..7d3f25709 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -5,8 +5,35 @@ def foo(): assert foo() == 42 assert foo.__doc__ == "test" - def my_func(a,): return a+2 assert my_func(2) == 4 + + +def f1(): + + """test1""" + pass + +assert f1.__doc__ == "test1" + +def f2(): + '''test2''' + pass + +assert f2.__doc__ == "test2" + +def f3(): + """ + test3 + """ + pass + +assert f3.__doc__ == "\n test3\n " + +def f4(): + "test4" + pass + +assert f4.__doc__ == "test4" From ab3cde77b4b2b7d71ffe7e5d05b94e3c45c677e3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:45:45 +0200 Subject: [PATCH 070/658] Support class __doc__ --- tests/snippets/class.py | 3 +-- vm/src/builtins.rs | 2 ++ vm/src/compile.rs | 47 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 06e125f21..bd94e0516 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -37,8 +37,7 @@ class Bar: assert x == 3 -# TODO: -# assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == " W00t " bar = Bar(42) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index ed656c0fb..705ec6905 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -769,6 +769,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { let function = args.shift(); let name_arg = args.shift(); + let doc = args.shift(); let bases = args.args.clone(); let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? @@ -801,5 +802,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu vec![name_arg, bases, namespace.into_object()], )?; cells.set_item(&vm.ctx, "__class__", class.clone()); + vm.ctx.set_attr(&class, "__doc__", doc); Ok(class) } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 037d9e95c..58a582ae7 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -706,13 +706,34 @@ impl Compiler { line_number, name.to_string(), )); - self.compile_statements(body)?; + + let doc = get_doc(body); + let (new_body, doc_str) = match doc { + Some(val) => { + if let Some((_, body_rest)) = body.split_first() { + (body_rest, val) + } else { + (body, "".to_string()) + } + } + None => (body, "".to_string()), + }; + + self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -735,6 +756,11 @@ impl Compiler { }, }); + // class doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { value: doc_str }, + }); + for base in bases { self.compile_expression(base)?; } @@ -759,11 +785,11 @@ impl Compiler { }, }); self.emit(Instruction::CallFunction { - typ: CallType::Keyword(2 + keywords.len() + bases.len()), + typ: CallType::Keyword(3 + keywords.len() + bases.len()), }); } else { self.emit(Instruction::CallFunction { - typ: CallType::Positional(2 + bases.len()), + typ: CallType::Positional(3 + bases.len()), }); } @@ -1142,6 +1168,14 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1428,6 +1462,13 @@ impl Compiler { // Fetch code for listcomp function: let code = self.pop_code_object(); + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + // List comprehension code: self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { From 6d53fe692408912c3558e827c296cb98a9f98e68 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:49:40 +0200 Subject: [PATCH 071/658] Add more tests for class doc --- tests/snippets/class.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index bd94e0516..9dbce566d 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -116,3 +116,31 @@ assert isinstance(a, super) assert type(a) is super assert a.conjugate() == 1 + +class T1: + "test1" + +assert T1.__doc__ == "test1" + +class T2: + '''test2''' + +assert T2.__doc__ == "test2" + +class T3: + """ + test3 + """ + +assert T3.__doc__ == "\n test3\n " + +class T4: + + """test4""" + + def t1(self): + """t1""" + pass + +assert T4.__doc__ == "test4" +assert T4.t1.__doc__ == "t1" From 4e99350e5c7e8850d0557b2fac7c19773f7b1dab Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 31 Mar 2019 18:49:04 +0300 Subject: [PATCH 072/658] Use StoreAttr for __doc__ --- vm/src/builtins.rs | 2 - vm/src/compile.rs | 94 ++++++++++++++++++---------------------------- vm/src/frame.rs | 3 -- 3 files changed, 36 insertions(+), 63 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 705ec6905..ed656c0fb 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -769,7 +769,6 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { let function = args.shift(); let name_arg = args.shift(); - let doc = args.shift(); let bases = args.args.clone(); let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? @@ -802,6 +801,5 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu vec![name_arg, bases, namespace.into_object()], )?; cells.set_item(&vm.ctx, "__class__", class.clone()); - vm.ctx.set_attr(&class, "__doc__", doc); Ok(class) } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 58a582ae7..f79980af9 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -598,17 +598,7 @@ impl Compiler { self.in_function_def = true; let mut flags = self.enter_function(name, args)?; - let doc = get_doc(body); - let (new_body, doc_str) = match doc { - Some(val) => { - if let Some((_, body_rest)) = body.split_first() { - (body_rest, val) - } else { - (body, "".to_string()) - } - } - None => (body, "".to_string()), - }; + let (new_body, doc_str) = get_doc(body); self.compile_statements(new_body)?; @@ -657,10 +647,6 @@ impl Compiler { }); } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: doc_str }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -679,6 +665,20 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + + if let Some(doc_string) = doc_str { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + self.emit(Instruction::LoadName { + name: name.to_string(), + }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -707,17 +707,7 @@ impl Compiler { name.to_string(), )); - let doc = get_doc(body); - let (new_body, doc_str) = match doc { - Some(val) => { - if let Some((_, body_rest)) = body.split_first() { - (body_rest, val) - } else { - (body, "".to_string()) - } - } - None => (body, "".to_string()), - }; + let (new_body, doc_str) = get_doc(body); self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { @@ -727,13 +717,6 @@ impl Compiler { let code = self.pop_code_object(); - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -756,11 +739,6 @@ impl Compiler { }, }); - // class doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: doc_str }, - }); - for base in bases { self.compile_expression(base)?; } @@ -785,11 +763,11 @@ impl Compiler { }, }); self.emit(Instruction::CallFunction { - typ: CallType::Keyword(3 + keywords.len() + bases.len()), + typ: CallType::Keyword(2 + keywords.len() + bases.len()), }); } else { self.emit(Instruction::CallFunction { - typ: CallType::Positional(3 + bases.len()), + typ: CallType::Positional(2 + bases.len()), }); } @@ -798,6 +776,19 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + if let Some(doc_string) = doc_str { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + self.emit(Instruction::LoadName { + name: name.to_string(), + }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } self.in_loop = was_in_loop; Ok(()) } @@ -1168,14 +1159,6 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1462,13 +1445,6 @@ impl Compiler { // Fetch code for listcomp function: let code = self.pop_code_object(); - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - // List comprehension code: self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { @@ -1569,17 +1545,19 @@ impl Compiler { } } -fn get_doc(body: &[ast::LocatedStatement]) -> Option { +fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option) { if let Some(val) = body.get(0) { if let ast::Statement::Expression { ref expression } = val.node { if let ast::Expression::String { ref value } = expression { if let ast::StringGroup::Constant { ref value } = value { - return Some(value.to_string()); + if let Some((_, body_rest)) = body.split_first() { + return (body_rest, Some(value.to_string())); + } } } } } - None + (body, None) } #[cfg(test)] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 5a22aabe9..88a619eb3 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -568,8 +568,6 @@ impl Frame { .downcast() .expect("Second to top value on the stack must be a code object"); - let doc = self.pop_value(); - let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { @@ -588,7 +586,6 @@ impl Frame { let obj = vm.ctx.new_function(code_obj, scope, defaults); vm.ctx.set_attr(&obj, "__annotations__", annotations); - vm.ctx.set_attr(&obj, "__doc__", doc); self.push_value(obj); Ok(None) From 53b46a7b3245741a27ec05b30a23f1a9f26538ee Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 17:23:33 -0500 Subject: [PATCH 073/658] Split some of #[pyclass] off into #[pyimpl] --- derive/src/lib.rs | 9 +++- derive/src/pyclass.rs | 112 +++++++++++++++++++++++++----------------- vm/src/obj/objstr.rs | 25 +++++----- vm/src/pyobject.rs | 16 ++++-- 4 files changed, 100 insertions(+), 62 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 5a26030e4..adde078f0 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -52,5 +52,12 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - pyclass::impl_py_class(attr, item).into() + pyclass::impl_pyclass(attr, item).into() +} + +#[proc_macro_attribute] +pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + pyclass::impl_pyimpl(attr, item).into() } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index ad03b58c9..56e216f35 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -89,54 +89,15 @@ impl Method { } } -pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); }; + let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; - for attr in attr { - if let NestedMeta::Meta(meta) = attr { - if let Meta::NameValue(name_value) = meta { - if name_value.ident == "name" { - if let Lit::Str(s) = name_value.lit { - class_name = Some(s.value()); - } else { - panic!("#[pyclass(name = ...)] must be a string"); - } - } - } - } - } - let class_name = class_name.expect("#[pyclass] must have a name"); - let mut doc: Option> = None; - for attr in imp.attrs.iter() { - if attr.path.is_ident("doc") { - let meta = attr.parse_meta().expect("expected doc attr to be a meta"); - if let Meta::NameValue(name_value) = meta { - if let Lit::Str(s) = name_value.lit { - let val = s.value().trim().to_string(); - match doc { - Some(ref mut doc) => doc.push(val), - None => doc = Some(vec![val]), - } - } else { - panic!("expected #[doc = ...] to be a string") - } - } else { - panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); - } - } - } - let doc = match doc { - Some(doc) => { - let doc = doc.join("\n"); - quote!(Some(#doc)) - } - None => quote!(None), - }; + let methods = imp .items .iter_mut() @@ -164,10 +125,8 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #imp - impl #rp_path::pyobject::IntoPyClass for #ty { - const NAME: &'static str = #class_name; - const DOC: Option<&'static str> = #doc; - fn _extend_class( + impl #rp_path::pyobject::PyClassImpl for #ty { + fn impl_extend_class( ctx: &#rp_path::pyobject::PyContext, class: &#rp_path::obj::objtype::PyClassRef, ) { @@ -176,3 +135,64 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } } } + +pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { + let struc = if let Item::Struct(struc) = item { + struc + } else { + panic!("#[pyclass] can only be on a struct declaration"); + }; + + let rp_path = rustpython_path_attr(&attr); + + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[pyclass(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.unwrap_or_else(|| struc.ident.to_string()); + + let mut doc: Option> = None; + for attr in struc.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + + let ty = &struc.ident; + + quote! { + #struc + impl #rp_path::pyobject::PyClassDef for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + } + } +} diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 9520b8af9..922306a9e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -10,7 +10,7 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyClass, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -20,6 +20,17 @@ use super::objsequence::PySliceableSequence; use super::objslice::PySlice; use super::objtype::{self, PyClassRef}; +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." +#[pyclass(name = "str", __inside_vm)] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public @@ -48,17 +59,7 @@ impl TryIntoRef for &str { } } -#[pyclass(__inside_vm, name = "str")] -/// str(object='') -> str -/// str(bytes_or_buffer[, encoding[, errors]]) -> str -/// -/// Create a new string object from the given object. If encoding or -/// errors is specified, then the object must expose a data buffer -/// that will be decoded using the given encoding and error handler. -/// Otherwise, returns the result of object.__str__() (if defined) -/// or repr(object). -/// encoding defaults to sys.getdefaultencoding(). -/// errors defaults to 'strict'." +#[pyimpl(__inside_vm)] impl PyStringRef { // TODO: should with following format // class str(object='') diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index a0e737d4c..801522ea0 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1259,14 +1259,24 @@ where } } -pub trait IntoPyClass { +pub trait PyClassDef { const NAME: &'static str; const DOC: Option<&'static str> = None; +} - fn _extend_class(ctx: &PyContext, class: &PyClassRef); +impl PyClassDef for PyRef +where + T: PyClassDef, +{ + const NAME: &'static str = T::NAME; + const DOC: Option<&'static str> = T::DOC; +} + +pub trait PyClassImpl: PyClassDef { + fn impl_extend_class(ctx: &PyContext, class: &PyClassRef); fn extend_class(ctx: &PyContext, class: &PyClassRef) { - Self::_extend_class(ctx, class); + Self::impl_extend_class(ctx, class); if let Some(doc) = Self::DOC { ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); } From 42768b20e57a1a699340c78d21a5268910dc0b93 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 17:53:53 -0500 Subject: [PATCH 074/658] Small demo improvements --- wasm/demo/snippets/mandelbrot.py | 9 +++++---- wasm/demo/src/style.css | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index 2d664fded..b4010c753 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -2,8 +2,7 @@ w = 50.0 h = 50.0 def mandel(): - """Print a mandelbrot fractal to the console, yielding after each character - is printed""" + """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 while y < h: x = 0.0 @@ -32,12 +31,14 @@ def mandel(): y += 1 yield +# run the mandelbrot + try: from browser import request_animation_frame except: request_animation_frame = None gen = mandel() def gen_cb(_time=None): - gen.__next__() + for _ in range(4): gen.__next__() request_animation_frame(gen_cb) if request_animation_frame: gen_cb() -else: list(gen) +else: any(gen) diff --git a/wasm/demo/src/style.css b/wasm/demo/src/style.css index a8a5de500..8fc18ecce 100644 --- a/wasm/demo/src/style.css +++ b/wasm/demo/src/style.css @@ -1,6 +1,6 @@ textarea { font-family: monospace; - resize: none; + resize: vertical; } #code, From 51b766089f4f9f245499c280ff2a33627c024bb9 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 19:53:52 -0500 Subject: [PATCH 075/658] Allow an enum with #[pyclass] --- derive/src/pyclass.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 56e216f35..7e196a2a1 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -137,10 +137,10 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { } pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { - let struc = if let Item::Struct(struc) = item { - struc - } else { - panic!("#[pyclass] can only be on a struct declaration"); + let (ident, attrs) = match item.clone() { + Item::Struct(struc) => (struc.ident, struc.attrs), + Item::Enum(enu) => (enu.ident, enu.attrs), + _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; let rp_path = rustpython_path_attr(&attr); @@ -159,10 +159,10 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { } } } - let class_name = class_name.unwrap_or_else(|| struc.ident.to_string()); + let class_name = class_name.unwrap_or_else(|| ident.to_string()); let mut doc: Option> = None; - for attr in struc.attrs.iter() { + for attr in attrs.iter() { if attr.path.is_ident("doc") { let meta = attr.parse_meta().expect("expected doc attr to be a meta"); if let Meta::NameValue(name_value) = meta { @@ -186,11 +186,9 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { None => quote!(None), }; - let ty = &struc.ident; - quote! { - #struc - impl #rp_path::pyobject::PyClassDef for #ty { + #item + impl #rp_path::pyobject::PyClassDef for #ident { const NAME: &'static str = #class_name; const DOC: Option<&'static str> = #doc; } From b3ea8155a5558b4c7fec3c1775f4fcaf2449442c Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 22:40:57 -0500 Subject: [PATCH 076/658] Don't clone --- derive/src/pyclass.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 7e196a2a1..ad4ebf66e 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -137,9 +137,9 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { } pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { - let (ident, attrs) = match item.clone() { - Item::Struct(struc) => (struc.ident, struc.attrs), - Item::Enum(enu) => (enu.ident, enu.attrs), + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; From 016ecf204dcb8ff9c956038572f03fbb9bd5d186 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 077/658] Move iterator __contains__ to 'in' implementation --- vm/src/frame.rs | 30 ++++-------------------------- vm/src/obj/objiter.rs | 33 +++++---------------------------- vm/src/vm.rs | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 54 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 88a619eb3..b3cf2149b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1075,36 +1075,14 @@ impl Frame { a.get_id() } - // https://docs.python.org/3/reference/expressions.html#membership-test-operations - fn _membership( - &self, - vm: &VirtualMachine, - needle: PyObjectRef, - haystack: &PyObjectRef, - ) -> PyResult { - vm.call_method(&haystack, "__contains__", vec![needle]) - // TODO: implement __iter__ and __getitem__ cases when __contains__ is - // not implemented. - } - fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(found), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(objbool::boolval(vm, found)?)) } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(!objbool::boolval(vm, found)?)) } fn _is(&self, a: PyObjectRef, b: PyObjectRef) -> bool { diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 5c88a6337..68c8b055f 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -6,7 +6,6 @@ use crate::function::PyFuncArgs; use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; -use super::objbool; use super::objbytearray::PyByteArray; use super::objbytes::PyBytes; use super::objrange::PyRange; @@ -24,6 +23,10 @@ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); // return Err(type_error); + + // TODO: special case when iter_target only has __getitem__ + // see: https://docs.python.org/3/library/functions.html#iter + // also https://docs.python.org/3.8/reference/datamodel.html#special-method-names } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -70,33 +73,8 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -fn contains(vm: &VirtualMachine, args: PyFuncArgs, iter_type: PyClassRef) -> PyResult { - arg_check!( - vm, - args, - required = [(iter, Some(iter_type)), (needle, None)] - ); - loop { - if let Some(element) = get_next_object(vm, iter)? { - let equal = vm._eq(needle.clone(), element.clone())?; - if objbool::get_value(&equal) { - return Ok(vm.new_bool(true)); - } else { - continue; - } - } else { - return Ok(vm.new_bool(false)); - } - } -} - -/// Common setup for iter types, adds __iter__ and __contains__ methods +/// Common setup for iter types, adds __iter__ method pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { - let contains_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| contains(vm, args, cloned_iter_type.clone()) - }; - let iter_func = { let cloned_iter_type = iter_type.clone(); move |vm: &VirtualMachine, args: PyFuncArgs| { @@ -111,7 +89,6 @@ pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { }; extend_class!(context, iter_type, { - "__contains__" => context.new_rustfunc(contains_func), "__iter__" => context.new_rustfunc(iter_func) }); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 36f4450e6..2fad29124 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -867,6 +867,31 @@ impl VirtualMachine { Err(vm.new_unsupported_operand_error(a, b, ">=")) }) } + + // https://docs.python.org/3/reference/expressions.html#membership-test-operations + fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + let iter = objiter::get_iter(self, &haystack)?; + loop { + if let Some(element) = objiter::get_next_object(self, &iter)? { + let equal = self._eq(needle.clone(), element.clone())?; + if objbool::get_value(&equal) { + return Ok(self.new_bool(true)); + } else { + continue; + } + } else { + return Ok(self.new_bool(false)); + } + } + } + + pub fn _membership(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + if let Ok(method) = self.get_method(haystack.clone(), "__contains__") { + self.invoke(method, vec![needle]) + } else { + self._membership_iter_search(haystack, needle) + } + } } impl Default for VirtualMachine { From cba8aa9be52bfc6c9e0f4253de28cd476398bde0 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 078/658] Drop iter_type_init, explicitly define __iter__ for iterators --- vm/src/obj/objenumerate.rs | 8 ++++-- vm/src/obj/objfilter.rs | 9 ++++-- vm/src/obj/objiter.rs | 56 +++++++++----------------------------- vm/src/obj/objmap.rs | 6 +++- vm/src/obj/objzip.rs | 8 ++++-- 5 files changed, 36 insertions(+), 51 deletions(-) diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 85781bb7a..c94aeabbd 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -57,13 +57,17 @@ impl PyEnumerateRef { Ok(result) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let enumerate_type = &context.enumerate_type; - objiter::iter_type_init(context, enumerate_type); extend_class!(context, enumerate_type, { "__new__" => context.new_rustfunc(enumerate_new), - "__next__" => context.new_rustfunc(PyEnumerateRef::next) + "__next__" => context.new_rustfunc(PyEnumerateRef::next), + "__iter__" => context.new_rustfunc(PyEnumerateRef::iter), }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index 07c512f6d..858a0531a 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -52,13 +52,15 @@ impl PyFilterRef { } } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let filter_type = &context.filter_type; - objiter::iter_type_init(context, filter_type); - let filter_doc = "filter(function or None, iterable) --> filter object\n\n\ Return an iterator yielding those items of iterable for which function(item)\n\ @@ -67,6 +69,7 @@ pub fn init(context: &PyContext) { extend_class!(context, filter_type, { "__new__" => context.new_rustfunc(filter_new), "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(PyFilterRef::next) + "__next__" => context.new_rustfunc(PyFilterRef::next), + "__iter__" => context.new_rustfunc(PyFilterRef::iter), }); } diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 68c8b055f..67513a255 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,8 +2,7 @@ * Various types to support iteration. */ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; use super::objbytearray::PyByteArray; @@ -11,7 +10,6 @@ use super::objbytes::PyBytes; use super::objrange::PyRange; use super::objsequence; use super::objtype; -use crate::obj::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -73,41 +71,12 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -/// Common setup for iter types, adds __iter__ method -pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { - let iter_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| { - arg_check!( - vm, - args, - required = [(iter, Some(cloned_iter_type.clone()))] - ); - // Return self: - Ok(iter.clone()) - } - }; +type PyIteratorValueRef = PyRef; - extend_class!(context, iter_type, { - "__iter__" => context.new_rustfunc(iter_func) - }); -} - -// Sequence iterator: -fn iter_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter_target, None)]); - - get_iter(vm, iter_target) -} - -fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter, Some(vm.ctx.iter_type()))]); - - if let Some(PyIteratorValue { - ref position, - iterated_obj: ref iterated_obj_ref, - }) = iter.payload() - { +impl PyIteratorValueRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let position = &self.position; + let iterated_obj_ref = &self.iterated_obj; if let Some(range) = iterated_obj_ref.payload::() { if let Some(int) = range.get(position.get()) { position.set(position.get() + 1); @@ -141,8 +110,10 @@ fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(new_stop_iteration(vm)) } } - } else { - panic!("NOT IMPL"); + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self } } @@ -155,10 +126,9 @@ pub fn init(context: &PyContext) { supply its own iterator, or be a sequence.\n\ In the second form, the callable is called until it returns the sentinel."; - iter_type_init(context, iter_type); extend_class!(context, iter_type, { - "__new__" => context.new_rustfunc(iter_new), - "__next__" => context.new_rustfunc(iter_next), - "__doc__" => context.new_str(iter_doc.to_string()) + "__next__" => context.new_rustfunc(PyIteratorValueRef::next), + "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), + "__doc__" => context.new_str(iter_doc.to_string()), }); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index fbf0fa035..061f4179f 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -46,6 +46,10 @@ impl PyMapRef { // the mapper itself can raise StopIteration which does stop the map iteration vm.invoke(self.mapper.clone(), next_objs) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { @@ -55,10 +59,10 @@ pub fn init(context: &PyContext) { Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted."; - objiter::iter_type_init(context, map_type); extend_class!(context, map_type, { "__new__" => context.new_rustfunc(map_new), "__next__" => context.new_rustfunc(PyMapRef::next), + "__iter__" => context.new_rustfunc(PyMapRef::iter), "__doc__" => context.new_str(map_doc.to_string()) }); } diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index a12152eee..81640a724 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -40,13 +40,17 @@ impl PyZipRef { Ok(vm.ctx.new_tuple(next_objs)) } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let zip_type = &context.zip_type; - objiter::iter_type_init(context, zip_type); extend_class!(context, zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(PyZipRef::next) + "__next__" => context.new_rustfunc(PyZipRef::next), + "__iter__" => context.new_rustfunc(PyZipRef::iter), }); } From bbfca26b27cb03f20931f76d415ef95ff4706dd0 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 079/658] Move PyIteratorValue to objiter.rs --- vm/src/obj/objbytes.rs | 3 ++- vm/src/obj/objdict.rs | 3 ++- vm/src/obj/objiter.rs | 20 +++++++++++++++++++- vm/src/obj/objlist.rs | 5 ++--- vm/src/obj/objrange.rs | 3 ++- vm/src/obj/objset.rs | 5 ++--- vm/src/obj/objtuple.rs | 5 ++--- vm/src/pyobject.rs | 16 +--------------- 8 files changed, 32 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 9390e7070..2f5bdc0d9 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,10 +6,11 @@ use std::ops::Deref; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; +use super::objiter::PyIteratorValue; use super::objtype::PyClassRef; #[derive(Debug)] diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 061c3477e..5a06fbf99 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -5,11 +5,12 @@ use std::ops::{Deref, DerefMut}; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objiter; +use super::objiter::PyIteratorValue; use super::objstr::{self, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 67513a255..c1ffa7773 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,7 +2,9 @@ * Various types to support iteration. */ -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult}; +use std::cell::Cell; + +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objbytearray::PyByteArray; @@ -10,6 +12,7 @@ use super::objbytes::PyBytes; use super::objrange::PyRange; use super::objsequence; use super::objtype; +use super::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -71,6 +74,21 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } +// TODO: This is a workaround and shouldn't exist. +// Each iterable type should have its own distinct iterator type. +// (however, this boilerplate can be reused for "generic iterator" for types with only __getiter__) +#[derive(Debug)] +pub struct PyIteratorValue { + pub position: Cell, + pub iterated_obj: PyObjectRef, +} + +impl PyValue for PyIteratorValue { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.iter_type() + } +} + type PyIteratorValueRef = PyRef; impl PyIteratorValueRef { diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index dd5eb055b..444b5ad9d 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -4,13 +4,12 @@ use std::fmt; use num_traits::ToPrimitive; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; +use super::objiter::PyIteratorValue; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, PySliceableSequence, diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 04c9c273b..5af576bfc 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,10 +6,11 @@ use num_integer::Integer; use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{Either, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; +use super::objiter::PyIteratorValue; use super::objslice::PySliceRef; use super::objtype::PyClassRef; diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index ad82c34de..c048686d7 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,14 +8,13 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - PyContext, PyIteratorValue, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; use super::objiter; +use super::objiter::PyIteratorValue; use super::objtype; use super::objtype::PyClassRef; diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index a199e34b1..e7dc5d0ea 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -3,13 +3,12 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, -}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; +use super::objiter::PyIteratorValue; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index aa3348be5..3574d71d9 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,5 +1,5 @@ use std::any::Any; -use std::cell::{Cell, RefCell}; +use std::cell::RefCell; use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; @@ -1127,20 +1127,6 @@ where } } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -#[derive(Debug)] -pub struct PyIteratorValue { - pub position: Cell, - pub iterated_obj: PyObjectRef, -} - -impl PyValue for PyIteratorValue { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.iter_type() - } -} - impl PyObject where T: Sized + PyObjectPayload, From c918e9d5d3ab343e64a607529742690b144f212d Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 080/658] Split iterators into separate types --- vm/src/obj/objbytearray.rs | 47 +++++++++++++++++++++++++++++++++++- vm/src/obj/objbytes.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/obj/objdict.rs | 24 +++++++++++-------- vm/src/obj/objiter.rs | 49 ++------------------------------------ vm/src/obj/objlist.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/obj/objrange.rs | 49 ++++++++++++++++++++++++++++++++------ vm/src/obj/objset.rs | 9 +++---- vm/src/obj/objtuple.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/pyobject.rs | 35 +++++++++++++++++++++++++++ 9 files changed, 264 insertions(+), 81 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index c609fdb2f..f6244399b 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,6 +1,6 @@ //! Implementation of the python bytearray object. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::fmt::Write; use std::ops::{Deref, DerefMut}; @@ -11,6 +11,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; +use super::objiter; use super::objtype::PyClassRef; #[derive(Debug)] @@ -67,6 +68,7 @@ pub fn init(context: &PyContext) { "__eq__" => context.new_rustfunc(PyByteArrayRef::eq), "__len__" => context.new_rustfunc(PyByteArrayRef::len), "__repr__" => context.new_rustfunc(PyByteArrayRef::repr), + "__iter__" => context.new_rustfunc(PyByteArrayRef::iter), "clear" => context.new_rustfunc(PyByteArrayRef::clear), "isalnum" => context.new_rustfunc(PyByteArrayRef::isalnum), "isalpha" => context.new_rustfunc(PyByteArrayRef::isalpha), @@ -80,6 +82,12 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyByteArrayRef::pop), "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); + + let bytearrayiterator_type = &context.bytearrayiterator_type; + extend_class!(context, bytearrayiterator_type, { + "__next__" => context.new_rustfunc(PyByteArrayIteratorRef::next), + "__iter__" => context.new_rustfunc(PyByteArrayIteratorRef::iter), + }); } fn bytearray_new( @@ -225,6 +233,13 @@ impl PyByteArrayRef { value: RefCell::new(bytes), } } + + fn iter(self, _vm: &VirtualMachine) -> PyByteArrayIterator { + PyByteArrayIterator { + position: Cell::new(0), + bytearray: self, + } + } } // helper function for istitle @@ -266,3 +281,33 @@ mod tests { assert_eq!(&to_hex(&[11u8, 222u8]), "\\x0b\\xde"); } } + +#[derive(Debug)] +pub struct PyByteArrayIterator { + position: Cell, + bytearray: PyByteArrayRef, +} + +impl PyValue for PyByteArrayIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytearrayiterator_type() + } +} + +type PyByteArrayIteratorRef = PyRef; + +impl PyByteArrayIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytearray.value.borrow().len() { + let ret = self.bytearray.value.borrow()[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2f5bdc0d9..4119d688f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -10,7 +10,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objtype::PyClassRef; #[derive(Debug)] @@ -69,6 +69,12 @@ pub fn init(context: &PyContext) { "__iter__" => context.new_rustfunc(PyBytesRef::iter), "__doc__" => context.new_str(bytes_doc.to_string()) }); + + let bytesiterator_type = &context.bytesiterator_type; + extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } fn bytes_new( @@ -150,10 +156,10 @@ impl PyBytesRef { format!("b'{}'", data) } - fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { position: Cell::new(0), - iterated_obj: obj.into_object(), + bytes: self, } } } @@ -161,3 +167,33 @@ impl PyBytesRef { pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().value } + +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, +} + +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } +} + +type PyBytesIteratorRef = PyRef; + +impl PyBytesIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.value.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 5a06fbf99..79c98c2dd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -10,7 +10,7 @@ use crate::pyobject::{ use crate::vm::{ReprGuard, VirtualMachine}; use super::objiter; -use super::objiter::PyIteratorValue; +use super::objlist::PyListIterator; use super::objstr::{self, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -211,7 +211,8 @@ impl PyDictRef { } /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyIteratorValue { + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type, not a list iterator let keys = self .entries .borrow() @@ -220,13 +221,14 @@ impl PyDictRef { .collect(); let key_list = vm.ctx.new_list(keys); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: key_list, + list: key_list.downcast().unwrap(), } } - fn values(self, vm: &VirtualMachine) -> PyIteratorValue { + fn values(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type. `values` should be a live view over the collection, not an iterator. let values = self .entries .borrow() @@ -235,13 +237,14 @@ impl PyDictRef { .collect(); let values_list = vm.ctx.new_list(values); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: values_list, + list: values_list.downcast().unwrap(), } } - fn items(self, vm: &VirtualMachine) -> PyIteratorValue { + fn items(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type. `items` should be a live view over the collection, not an iterator. let items = self .entries .borrow() @@ -250,9 +253,9 @@ impl PyDictRef { .collect(); let items_list = vm.ctx.new_list(items); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: items_list, + list: items_list.downcast().unwrap(), } } @@ -333,6 +336,7 @@ pub fn init(context: &PyContext) { "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), + // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::iter), "get" => context.new_rustfunc(PyDictRef::get), }); diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index c1ffa7773..a99fb0a87 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -7,10 +7,6 @@ use std::cell::Cell; use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objbytearray::PyByteArray; -use super::objbytes::PyBytes; -use super::objrange::PyRange; -use super::objsequence; use super::objtype; use super::objtype::PyClassRef; @@ -92,42 +88,8 @@ impl PyValue for PyIteratorValue { type PyIteratorValueRef = PyRef; impl PyIteratorValueRef { - fn next(self, vm: &VirtualMachine) -> PyResult { - let position = &self.position; - let iterated_obj_ref = &self.iterated_obj; - if let Some(range) = iterated_obj_ref.payload::() { - if let Some(int) = range.get(position.get()) { - position.set(position.get() + 1); - Ok(vm.ctx.new_int(int)) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.len() { - let obj_ref = vm.ctx.new_int(bytes[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.value.borrow().len() { - let obj_ref = vm.ctx.new_int(bytes.value.borrow()[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else { - let elements = objsequence::get_elements(iterated_obj_ref); - if position.get() < elements.len() { - let obj_ref = elements[position.get()].clone(); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } + fn next(self, _vm: &VirtualMachine) -> PyResult { + unimplemented!() } fn iter(self, _vm: &VirtualMachine) -> Self { @@ -138,15 +100,8 @@ impl PyIteratorValueRef { pub fn init(context: &PyContext) { let iter_type = &context.iter_type; - let iter_doc = "iter(iterable) -> iterator\n\ - iter(callable, sentinel) -> iterator\n\n\ - Get an iterator from an object. In the first form, the argument must\n\ - supply its own iterator, or be a sequence.\n\ - In the second form, the callable is called until it returns the sentinel."; - extend_class!(context, iter_type, { "__next__" => context.new_rustfunc(PyIteratorValueRef::next), "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), - "__doc__" => context.new_str(iter_doc.to_string()), }); } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 444b5ad9d..865707201 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -9,7 +9,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, PySliceableSequence, @@ -122,10 +122,10 @@ impl PyListRef { ) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyListIterator { + PyListIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + list: self, } } @@ -411,6 +411,36 @@ fn list_sort(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[derive(Debug)] +pub struct PyListIterator { + pub position: Cell, + pub list: PyListRef, +} + +impl PyValue for PyListIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.listiterator_type() + } +} + +type PyListIteratorRef = PyRef; + +impl PyListIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.list.elements.borrow().len() { + let ret = self.list.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let list_type = &context.list_type; @@ -449,4 +479,10 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyListRef::pop), "remove" => context.new_rustfunc(PyListRef::remove) }); + + let listiterator_type = &context.listiterator_type; + extend_class!(context, listiterator_type, { + "__next__" => context.new_rustfunc(PyListIteratorRef::next), + "__iter__" => context.new_rustfunc(PyListIteratorRef::iter), + }); } diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 5af576bfc..ae946d05e 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -10,7 +10,7 @@ use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objslice::PySliceRef; use super::objtype::PyClassRef; @@ -114,6 +114,12 @@ pub fn init(context: &PyContext) { "stop" => context.new_property(PyRangeRef::stop), "step" => context.new_property(PyRangeRef::step), }); + + let rangeiterator_type = &context.rangeiterator_type; + extend_class!(context, rangeiterator_type, { + "__next__" => context.new_rustfunc(PyRangeIteratorRef::next), + "__iter__" => context.new_rustfunc(PyRangeIteratorRef::iter), + }); } type PyRangeRef = PyRef; @@ -157,14 +163,14 @@ impl PyRangeRef { self.step.clone() } - fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyRangeIterator { + PyRangeIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + range: self, } } - fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { + fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -190,9 +196,9 @@ impl PyRangeRef { step: PyInt::new(-step).into_ref(vm), }; - PyIteratorValue { + PyRangeIterator { position: Cell::new(0), - iterated_obj: reversed.into_ref(vm).into_object(), + range: reversed.into_ref(vm), } } @@ -314,3 +320,32 @@ fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(range.into_object()) } + +#[derive(Debug)] +pub struct PyRangeIterator { + position: Cell, + range: PyRangeRef, +} + +impl PyValue for PyRangeIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.rangeiterator_type() + } +} + +type PyRangeIteratorRef = PyRef; + +impl PyRangeIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if let Some(int) = self.range.get(self.position.get()) { + self.position.set(self.position.get() + 1); + Ok(int) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index c048686d7..fb558c990 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -14,7 +14,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; use super::objiter; -use super::objiter::PyIteratorValue; +use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; @@ -560,12 +560,13 @@ fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(zelf.clone()) } -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyIteratorValue { +fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type let items = zelf.elements.borrow().values().cloned().collect(); let set_list = vm.ctx.new_list(items); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: set_list, + list: set_list.downcast().unwrap(), } } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index e7dc5d0ea..216945284 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -8,7 +8,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; @@ -139,10 +139,10 @@ impl PyTupleRef { Ok(hasher.finish()) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { + PyTupleIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + tuple: self, } } @@ -224,6 +224,36 @@ fn tuple_new( PyTuple::from(elements).into_ref_with_type(vm, cls) } +#[derive(Debug)] +pub struct PyTupleIterator { + position: Cell, + tuple: PyTupleRef, +} + +impl PyValue for PyTupleIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.tupleiterator_type() + } +} + +type PyTupleIteratorRef = PyRef; + +impl PyTupleIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.tuple.elements.borrow().len() { + let ret = self.tuple.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let tuple_type = &context.tuple_type; @@ -251,4 +281,10 @@ If the argument is a tuple, the return value is the same object."; "__doc__" => context.new_str(tuple_doc.to_string()), "index" => context.new_rustfunc(PyTupleRef::index) }); + + let tupleiterator_type = &context.tupleiterator_type; + extend_class!(context, tupleiterator_type, { + "__next__" => context.new_rustfunc(PyTupleIteratorRef::next), + "__iter__" => context.new_rustfunc(PyTupleIteratorRef::iter), + }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 3574d71d9..291d92e25 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -108,7 +108,9 @@ impl fmt::Display for PyObject { #[derive(Debug)] pub struct PyContext { pub bytes_type: PyClassRef, + pub bytesiterator_type: PyClassRef, pub bytearray_type: PyClassRef, + pub bytearrayiterator_type: PyClassRef, pub bool_type: PyClassRef, pub classmethod_type: PyClassRef, pub code_type: PyClassRef, @@ -126,17 +128,20 @@ pub struct PyContext { pub true_value: PyIntRef, pub false_value: PyIntRef, pub list_type: PyClassRef, + pub listiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, pub ellipsis: PyEllipsisRef, pub not_implemented: PyNotImplementedRef, pub tuple_type: PyClassRef, + pub tupleiterator_type: PyClassRef, pub set_type: PyClassRef, pub staticmethod_type: PyClassRef, pub super_type: PyClassRef, pub str_type: PyClassRef, pub range_type: PyClassRef, + pub rangeiterator_type: PyClassRef, pub slice_type: PyClassRef, pub type_type: PyClassRef, pub zip_type: PyClassRef, @@ -248,6 +253,7 @@ impl PyContext { let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); + let listiterator_type = create_type("list_iterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -255,8 +261,11 @@ impl PyContext { let frame_type = create_type("frame", &type_type, &object_type); let complex_type = create_type("complex", &type_type, &object_type); let bytes_type = create_type("bytes", &type_type, &object_type); + let bytesiterator_type = create_type("bytes_iterator", &type_type, &object_type); let bytearray_type = create_type("bytearray", &type_type, &object_type); + let bytearrayiterator_type = create_type("bytearray_iterator", &type_type, &object_type); let tuple_type = create_type("tuple", &type_type, &object_type); + let tupleiterator_type = create_type("tuple_iterator", &type_type, &object_type); let iter_type = create_type("iter", &type_type, &object_type); let enumerate_type = create_type("enumerate", &type_type, &object_type); let filter_type = create_type("filter", &type_type, &object_type); @@ -266,6 +275,7 @@ impl PyContext { let memoryview_type = create_type("memoryview", &type_type, &object_type); let code_type = create_type("code", &type_type, &int_type); let range_type = create_type("range", &type_type, &object_type); + let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); let exceptions = exceptions::ExceptionZoo::new(&type_type, &object_type); @@ -291,7 +301,9 @@ impl PyContext { bool_type, memoryview_type, bytearray_type, + bytearrayiterator_type, bytes_type, + bytesiterator_type, code_type, complex_type, classmethod_type, @@ -300,11 +312,13 @@ impl PyContext { frame_type, staticmethod_type, list_type, + listiterator_type, set_type, frozenset_type, true_value, false_value, tuple_type, + tupleiterator_type, iter_type, ellipsis_type, enumerate_type, @@ -317,6 +331,7 @@ impl PyContext { not_implemented, str_type, range_type, + rangeiterator_type, slice_type, object: object_type, function_type, @@ -373,10 +388,18 @@ impl PyContext { self.bytearray_type.clone() } + pub fn bytearrayiterator_type(&self) -> PyClassRef { + self.bytearrayiterator_type.clone() + } + pub fn bytes_type(&self) -> PyClassRef { self.bytes_type.clone() } + pub fn bytesiterator_type(&self) -> PyClassRef { + self.bytesiterator_type.clone() + } + pub fn code_type(&self) -> PyClassRef { self.code_type.clone() } @@ -405,6 +428,10 @@ impl PyContext { self.list_type.clone() } + pub fn listiterator_type(&self) -> PyClassRef { + self.listiterator_type.clone() + } + pub fn module_type(&self) -> PyClassRef { self.module_type.clone() } @@ -417,6 +444,10 @@ impl PyContext { self.range_type.clone() } + pub fn rangeiterator_type(&self) -> PyClassRef { + self.rangeiterator_type.clone() + } + pub fn slice_type(&self) -> PyClassRef { self.slice_type.clone() } @@ -437,6 +468,10 @@ impl PyContext { self.tuple_type.clone() } + pub fn tupleiterator_type(&self) -> PyClassRef { + self.tupleiterator_type.clone() + } + pub fn iter_type(&self) -> PyClassRef { self.iter_type.clone() } From de5b1c405573e8fa6ef3611ee53a08c234466ee4 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 081/658] Support iter() for types with only __getitem__ --- vm/src/obj/objiter.rs | 55 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index a99fb0a87..6e1dc1d06 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -4,7 +4,7 @@ use std::cell::Cell; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; use super::objtype; @@ -16,14 +16,19 @@ use super::objtype::PyClassRef; * function 'iter' is called. */ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { - vm.call_method(iter_target, "__iter__", vec![]) - // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); - // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); - // return Err(type_error); - - // TODO: special case when iter_target only has __getitem__ - // see: https://docs.python.org/3/library/functions.html#iter - // also https://docs.python.org/3.8/reference/datamodel.html#special-method-names + if let Ok(method) = vm.get_method(iter_target.clone(), "__iter__") { + vm.invoke(method, vec![]) + } else if vm.get_method(iter_target.clone(), "__getitem__").is_ok() { + Ok(PySequenceIterator { + position: Cell::new(0), + obj: iter_target.clone(), + } + .into_ref(vm) + .into_object()) + } else { + let message = format!("Cannot iterate over {}", iter_target.class().name); + return Err(vm.new_type_error(message)); + } } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -70,26 +75,34 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -// (however, this boilerplate can be reused for "generic iterator" for types with only __getiter__) #[derive(Debug)] -pub struct PyIteratorValue { +pub struct PySequenceIterator { pub position: Cell, - pub iterated_obj: PyObjectRef, + pub obj: PyObjectRef, } -impl PyValue for PyIteratorValue { +impl PyValue for PySequenceIterator { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.iter_type() } } -type PyIteratorValueRef = PyRef; +type PySequenceIteratorRef = PyRef; -impl PyIteratorValueRef { - fn next(self, _vm: &VirtualMachine) -> PyResult { - unimplemented!() +impl PySequenceIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let number = vm.ctx.new_int(self.position.get()); + match vm.call_method(&self.obj, "__getitem__", vec![number]) { + Ok(val) => { + self.position.set(self.position.get() + 1); + Ok(val) + } + Err(ref e) if objtype::isinstance(&e, &vm.ctx.exceptions.index_error) => { + Err(new_stop_iteration(vm)) + } + // also catches stop_iteration => stop_iteration + Err(e) => Err(e), + } } fn iter(self, _vm: &VirtualMachine) -> Self { @@ -101,7 +114,7 @@ pub fn init(context: &PyContext) { let iter_type = &context.iter_type; extend_class!(context, iter_type, { - "__next__" => context.new_rustfunc(PyIteratorValueRef::next), - "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), + "__next__" => context.new_rustfunc(PySequenceIteratorRef::next), + "__iter__" => context.new_rustfunc(PySequenceIteratorRef::iter), }); } From 2f2a8438b55acba2cc98cbc9de8eb9cfb79a583e Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 082/658] Add iterable tests --- tests/snippets/iterable.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/snippets/iterable.py diff --git a/tests/snippets/iterable.py b/tests/snippets/iterable.py new file mode 100644 index 000000000..45bb1cf4d --- /dev/null +++ b/tests/snippets/iterable.py @@ -0,0 +1,32 @@ +from testutils import assert_raises + +def test_container(x): + assert 3 in x + assert 4 not in x + assert list(x) == list(iter(x)) + assert list(x) == [0, 1, 2, 3] + assert [*x] == [0, 1, 2, 3] + lst = [] + lst.extend(x) + assert lst == [0, 1, 2, 3] + +class C: + def __iter__(self): + return iter([0, 1, 2, 3]) +test_container(C()) + +class C: + def __getitem__(self, x): + return (0, 1, 2, 3)[x] # raises IndexError on x==4 +test_container(C()) + +class C: + def __getitem__(self, x): + if x > 3: + raise StopIteration + return x +test_container(C()) + +class C: pass +assert_raises(TypeError, lambda: 5 in C()) +assert_raises(TypeError, lambda: iter(C)) From 6276241d59f085965a6ab02ff72171be8c4bd109 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 1 Apr 2019 18:11:37 -0500 Subject: [PATCH 083/658] Redo much of whats_left_to_implement.py --- tests/snippets/whats_left_to_implement.py | 1743 +++++++++++---------- whats_left.sh | 5 + 2 files changed, 879 insertions(+), 869 deletions(-) create mode 100755 whats_left.sh diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index b6c8cbe42..41b092506 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,878 +1,883 @@ -expected_methods = [] +import platform -# TODO: using tuple could have been better -expected_methods.append({'name': 'bool', 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -], 'type': bool}) -expected_methods.append({'name': 'bytearray', 'type': bytearray, 'methods': [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'bytes', 'type': bytes, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'complex', 'type': complex, 'methods': [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', -]}) -expected_methods.append({'name': 'dict', 'type': dict, 'methods': [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', -]}) -expected_methods.append({'name': 'float','type':float,'methods':[ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__set_format__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', -]}) -expected_methods.append({'name': 'frozenset','type':frozenset, 'methods': [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', -]}) -expected_methods.append({'name': 'int', 'type':int, 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -]}) -expected_methods.append({'name': 'iter','type':iter,'methods':[ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', -]}) -expected_methods.append({'name': 'list','type':list,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', -]}) -expected_methods.append({'name': 'memoryview','type':memoryview,'methods':[ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', -]}) -expected_methods.append({'name': 'range','type':range,'methods':[ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', -]}) -expected_methods.append({'name': 'set','type':set,'methods':[ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', -]}) -expected_methods.append({'name': 'string','type':str,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' -]}) -expected_methods.append({'name': 'tuple','type':tuple, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', -]}) -expected_methods.append({'name': 'object', 'type':object, 'methods':[ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' -]}) +expected_methods = { + 'bool': (bool, [ + '__abs__', + '__add__', + '__and__', + '__bool__', + '__ceil__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floor__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__index__', + '__init__', + '__init_subclass__', + '__int__', + '__invert__', + '__le__', + '__lshift__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__or__', + '__pos__', + '__pow__', + '__radd__', + '__rand__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__ror__', + '__round__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + '__xor__', + 'bit_length', + 'conjugate', + 'denominator', + 'from_bytes', + 'imag', + 'numerator', + 'real', + 'to_bytes', + ]), + 'bytearray': (bytearray, [ + '__add__', + '__alloc__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'capitalize', + 'center', + 'clear', + 'copy', + 'count', + 'decode', + 'endswith', + 'expandtabs', + 'extend', + 'find', + 'fromhex', + 'hex', + 'index', + 'insert', + 'isalnum', + 'isalpha', + 'isascii', + 'isdigit', + 'islower', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'pop', + 'remove', + 'replace', + 'reverse', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill', + ]), + 'bytes': (bytes, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'capitalize', + 'center', + 'count', + 'decode', + 'endswith', + 'expandtabs', + 'find', + 'fromhex', + 'hex', + 'index', + 'isalnum', + 'isalpha', + 'isascii', + 'isdigit', + 'islower', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'replace', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill', + ]), + 'complex': (complex, [ + '__abs__', + '__add__', + '__bool__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__int__', + '__le__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rsub__', + '__rtruediv__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + 'conjugate', + 'imag', + 'real', + ]), + 'dict': (dict, [ + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'clear', + 'copy', + 'fromkeys', + 'get', + 'items', + 'keys', + 'pop', + 'popitem', + 'setdefault', + 'update', + 'values', + ]), + 'float': (float, [ + '__abs__', + '__add__', + '__bool__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getformat__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__int__', + '__le__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rmod__', + '__rmul__', + '__round__', + '__rpow__', + '__rsub__', + '__rtruediv__', + '__set_format__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + 'as_integer_ratio', + 'conjugate', + 'fromhex', + 'hex', + 'imag', + 'is_integer', + 'real', + ]), + 'frozenset': (frozenset, [ + '__and__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__or__', + '__rand__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__ror__', + '__rsub__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__xor__', + 'copy', + 'difference', + 'intersection', + 'isdisjoint', + 'issubset', + 'issuperset', + 'symmetric_difference', + 'union', + ]), + 'int': (int, [ + '__abs__', + '__add__', + '__and__', + '__bool__', + '__ceil__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floor__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__index__', + '__init__', + '__init_subclass__', + '__int__', + '__invert__', + '__le__', + '__lshift__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__or__', + '__pos__', + '__pow__', + '__radd__', + '__rand__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__ror__', + '__round__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + '__xor__', + 'bit_length', + 'conjugate', + 'denominator', + 'from_bytes', + 'imag', + 'numerator', + 'real', + 'to_bytes', + ]), + 'iter': (iter, [ + '__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__length_hint__', + '__lt__', + '__ne__', + '__new__', + '__next__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setstate__', + '__sizeof__', + '__str__', + '__subclasshook__', + ]), + 'list': (list, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'pop', + 'remove', + 'reverse', + 'sort', + ]), + 'memoryview': (memoryview, [ + '__class__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__enter__', + '__eq__', + '__exit__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'c_contiguous', + 'cast', + 'contiguous', + 'f_contiguous', + 'format', + 'hex', + 'itemsize', + 'nbytes', + 'ndim', + 'obj', + 'readonly', + 'release', + 'shape', + 'strides', + 'suboffsets', + 'tobytes', + 'tolist', + ]), + 'range': (range, [ + '__bool__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'index', + 'start', + 'step', + 'stop', + ]), + 'set': (set, [ + '__and__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__iand__', + '__init__', + '__init_subclass__', + '__ior__', + '__isub__', + '__iter__', + '__ixor__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__or__', + '__rand__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__ror__', + '__rsub__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__xor__', + 'add', + 'clear', + 'copy', + 'difference', + 'difference_update', + 'discard', + 'intersection', + 'intersection_update', + 'isdisjoint', + 'issubset', + 'issuperset', + 'pop', + 'remove', + 'symmetric_difference', + 'symmetric_difference_update', + 'union', + 'update', + ]), + 'str': (str, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'capitalize', + 'casefold', + 'center', + 'count', + 'encode', + 'endswith', + 'expandtabs', + 'find', + 'format', + 'format_map', + 'index', + 'isalnum', + 'isalpha', + 'isascii', + 'isdecimal', + 'isdigit', + 'isidentifier', + 'islower', + 'isnumeric', + 'isprintable', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'replace', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill' + ]), + 'tuple': (tuple, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'index', + ]), + 'object': (object, [ + '__repr__', + '__hash__', + '__str__', + '__getattribute__', + '__setattr__', + '__delattr__', + '__lt__', + '__le__', + '__eq__', + '__ne__', + '__gt__', + '__ge__', + '__init__', + '__new__', + '__reduce_ex__', + '__reduce__', + '__subclasshook__', + '__init_subclass__', + '__format__', + '__sizeof__', + '__dir__', + '__class__', + '__doc__' + ]), +} not_implemented = [] -for item in expected_methods: - for method in item['methods']: +for name, (ty, methods) in expected_methods.items(): + for method in methods: try: - if not hasattr(item['type'], method): - not_implemented.append((item['name'], method)) + if not hasattr(ty, method): + not_implemented.append((name, method)) except NameError: - not_implemented.append((item['name'], method)) + not_implemented.append((name, method)) -for r in not_implemented: - print(r[0], ".", r[1]) +if not_implemented: + for r in not_implemented: + print(r[0], ".", r[1], sep="") else: print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/whats_left.sh b/whats_left.sh new file mode 100755 index 000000000..c6f191bd5 --- /dev/null +++ b/whats_left.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +cd "$(dirname "$0")" || exit + +cargo run -- tests/snippets/whats_left_to_implement.py From 9ff1e41422578fd7566fba4b15d8fdec0a43b9dc Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 1 Apr 2019 18:28:15 -0500 Subject: [PATCH 084/658] Give values instead of types, list comprehension, remove methods not in CPython --- tests/snippets/whats_left_to_implement.py | 49 +++++++++-------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index 41b092506..a9a2d4de9 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,7 +1,7 @@ import platform expected_methods = { - 'bool': (bool, [ + 'bool': (True, [ '__abs__', '__add__', '__and__', @@ -73,7 +73,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'bytearray': (bytearray, [ + 'bytearray': (bytearray(), [ '__add__', '__alloc__', '__class__', @@ -128,7 +128,6 @@ expected_methods = { 'insert', 'isalnum', 'isalpha', - 'isascii', 'isdigit', 'islower', 'isspace', @@ -160,7 +159,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'bytes': (bytes, [ + 'bytes': (b'', [ '__add__', '__class__', '__contains__', @@ -206,7 +205,6 @@ expected_methods = { 'index', 'isalnum', 'isalpha', - 'isascii', 'isdigit', 'islower', 'isspace', @@ -235,7 +233,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'complex': (complex, [ + 'complex': (0j, [ '__abs__', '__add__', '__bool__', @@ -286,7 +284,7 @@ expected_methods = { 'imag', 'real', ]), - 'dict': (dict, [ + 'dict': ({}, [ '__class__', '__contains__', '__delattr__', @@ -328,7 +326,7 @@ expected_methods = { 'update', 'values', ]), - 'float': (float, [ + 'float': (0.0, [ '__abs__', '__add__', '__bool__', @@ -371,7 +369,6 @@ expected_methods = { '__rpow__', '__rsub__', '__rtruediv__', - '__set_format__', '__setattr__', '__sizeof__', '__str__', @@ -387,7 +384,7 @@ expected_methods = { 'is_integer', 'real', ]), - 'frozenset': (frozenset, [ + 'frozenset': (frozenset(), [ '__and__', '__class__', '__contains__', @@ -431,7 +428,7 @@ expected_methods = { 'symmetric_difference', 'union', ]), - 'int': (int, [ + 'int': (0, [ '__abs__', '__add__', '__and__', @@ -503,7 +500,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'iter': (iter, [ + 'iter': ([].__iter__(), [ '__class__', '__delattr__', '__dir__', @@ -532,7 +529,7 @@ expected_methods = { '__str__', '__subclasshook__', ]), - 'list': (list, [ + 'list': ([], [ '__add__', '__class__', '__contains__', @@ -580,7 +577,7 @@ expected_methods = { 'reverse', 'sort', ]), - 'memoryview': (memoryview, [ + 'memoryview': (memoryview(b''), [ '__class__', '__delattr__', '__delitem__', @@ -628,7 +625,7 @@ expected_methods = { 'tobytes', 'tolist', ]), - 'range': (range, [ + 'range': (range(0), [ '__bool__', '__class__', '__contains__', @@ -664,7 +661,7 @@ expected_methods = { 'step', 'stop', ]), - 'set': (set, [ + 'set': (set(), [ '__and__', '__class__', '__contains__', @@ -721,7 +718,7 @@ expected_methods = { 'union', 'update', ]), - 'str': (str, [ + 'str': ('', [ '__add__', '__class__', '__contains__', @@ -768,7 +765,6 @@ expected_methods = { 'index', 'isalnum', 'isalpha', - 'isascii', 'isdecimal', 'isdigit', 'isidentifier', @@ -801,7 +797,7 @@ expected_methods = { 'upper', 'zfill' ]), - 'tuple': (tuple, [ + 'tuple': ((), [ '__add__', '__class__', '__contains__', @@ -836,7 +832,7 @@ expected_methods = { 'count', 'index', ]), - 'object': (object, [ + 'object': (object(), [ '__repr__', '__hash__', '__str__', @@ -863,15 +859,10 @@ expected_methods = { ]), } -not_implemented = [] - -for name, (ty, methods) in expected_methods.items(): - for method in methods: - try: - if not hasattr(ty, method): - not_implemented.append((name, method)) - except NameError: - not_implemented.append((name, method)) +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] if not_implemented: for r in not_implemented: From 98889d5339f8ec2e5c17ca58becf6a66dfbcd6cb Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 1 Apr 2019 13:38:06 -0700 Subject: [PATCH 085/658] match_class! macro --- vm/src/macros.rs | 87 ++++++++++++++++++++++++++++++++++++++++++ vm/src/obj/objfloat.rs | 6 +++ 2 files changed, 93 insertions(+) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index a63186797..601151a05 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -147,3 +147,90 @@ macro_rules! extend_class { )* } } + +/// Macro to match on the built-in class of a Python object. +/// +/// Like `match`, `match_object!` is an expression, and so must have a default +/// arm that will be evaluated when the object was not an instance of any of +/// the classes specified in other arms. +/// +/// # Examples +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// assert_eq!( +/// "int", +/// match_class!(obj.clone(), +/// PyInt => "int", +/// PyFloat => "float", +/// _ => "neither" +/// ) +/// ); +/// +/// ``` +/// +/// With a binding to the downcasted type: +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// +/// let int_value = match_class!(obj, +/// i @ PyInt => i.as_bigint().clone(), +/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), +/// obj => panic!("non-numeric object {}", obj) +/// ); +/// +/// assert!(int_value.is_zero()); +/// ``` +#[macro_export] +macro_rules! match_class { + // The default arm. + ($obj:expr, _ => $default:expr) => { + $default + }; + + // The default arm, binding the original object to the specified identifier. + ($obj:expr, $binding:ident => $default:expr) => {{ + let $binding = $obj; + $default + }}; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier. + ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { + match $obj.downcast::<$class>() { + Ok($binding) => $expr, + Err(_obj) => match_class!(_obj, $($rest)*), + } + }; + + // An arm taken when the object is an instance of the specified built-in + // class. + ($obj:expr, $class:ty => $expr:expr, $($rest:tt)*) => { + if $obj.payload_is::<$class>() { + $expr + } else { + match_class!($obj, $($rest)*) + } + }; +} diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index ca6bad9d2..4ef7123ee 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -16,6 +16,12 @@ pub struct PyFloat { value: f64, } +impl PyFloat { + pub fn to_f64(&self) -> f64 { + self.value + } +} + impl PyValue for PyFloat { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.float_type() From 8549ea22a3d280498d72b10b0fbfa8378f3dd146 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 1 Apr 2019 20:22:11 -0700 Subject: [PATCH 086/658] Use match_class! in objint::to_int --- vm/src/obj/objint.rs | 37 +++++++++++++++---------------------- vm/src/obj/objstr.rs | 7 +++++++ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 08175dc07..c26cdf4c0 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -12,8 +12,8 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objfloat; -use super::objstr; +use super::objfloat::{self, PyFloat}; +use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -351,7 +351,7 @@ impl PyIntRef { self.value.to_string() } - fn format(self, spec: PyRef, vm: &VirtualMachine) -> PyResult { + fn format(self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -408,29 +408,22 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul // Casting function: pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { - let val = if objtype::isinstance(obj, &vm.ctx.int_type()) { - get_value(obj).clone() - } else if objtype::isinstance(obj, &vm.ctx.float_type()) { - objfloat::get_value(obj).to_bigint().unwrap() - } else if objtype::isinstance(obj, &vm.ctx.str_type()) { - let s = objstr::get_value(obj); - match i32::from_str_radix(&s, base) { - Ok(v) => v.to_bigint().unwrap(), - Err(err) => { - trace!("Error occurred during int conversion {:?}", err); - return Err(vm.new_value_error(format!( + match_class!(obj.clone(), + i @ PyInt => Ok(i.as_bigint().clone()), + f @ PyFloat => Ok(f.to_f64().to_bigint().unwrap()), + s @ PyString => { + i32::from_str_radix(s.as_str(), base) + .map(|i| BigInt::from(i)) + .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s - ))); - } - } - } else { - return Err(vm.new_type_error(format!( + ))) + }, + obj => Err(vm.new_type_error(format!( "int() argument must be a string or a number, not '{}'", obj.class().name - ))); - }; - Ok(val) + ))) + ) } // Retrieve inner int value: diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index aa62812a2..8ac23c160 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -25,6 +25,13 @@ pub struct PyString { // TODO: shouldn't be public pub value: String, } + +impl PyString { + pub fn as_str(&self) -> &str { + &self.value + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { From ea2622ee7b820dc4fb787aa47817b293fcb69c57 Mon Sep 17 00:00:00 2001 From: ben Date: Mon, 1 Apr 2019 19:38:20 +1300 Subject: [PATCH 087/658] Make slice.stop not an option --- vm/src/frame.rs | 11 ++++++++--- vm/src/obj/objslice.rs | 16 ++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 3a5f4c5ab..2cd080824 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -407,10 +407,15 @@ impl Frame { } else { None }; - let stop = Some(self.pop_value()); - let start = Some(self.pop_value()); + let stop = self.pop_value(); + let start = self.pop_value(); - let obj = PySlice { start, stop, step }.into_ref(vm); + let obj = PySlice { + start: Some(start), + stop, + step, + } + .into_ref(vm); self.push_value(obj.into_object()); Ok(None) } diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index f6f02c8db..82e4cef8c 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -9,7 +9,7 @@ use num_bigint::BigInt; #[derive(Debug)] pub struct PySlice { pub start: Option, - pub stop: Option, + pub stop: PyObjectRef, pub step: Option, } @@ -30,7 +30,7 @@ fn slice_new(cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult let stop = args.bind(vm)?; PySlice { start: None, - stop: Some(stop), + stop, step: None, } } @@ -39,7 +39,7 @@ fn slice_new(cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult args.bind(vm)?; PySlice { start: Some(start), - stop: Some(stop), + stop, step: step.into_option(), } } @@ -60,8 +60,8 @@ impl PySliceRef { get_property_value(vm, &self.start) } - fn stop(self, vm: &VirtualMachine) -> PyObjectRef { - get_property_value(vm, &self.stop) + fn stop(self, _vm: &VirtualMachine) -> PyObjectRef { + self.stop.clone() } fn step(self, vm: &VirtualMachine) -> PyObjectRef { @@ -77,11 +77,7 @@ impl PySliceRef { } pub fn stop_index(&self, vm: &VirtualMachine) -> PyResult> { - if let Some(obj) = &self.stop { - to_index_value(vm, obj) - } else { - Ok(None) - } + to_index_value(vm, &self.stop) } pub fn step_index(&self, vm: &VirtualMachine) -> PyResult> { From 75d02a1725ff3b050b395ec79287ca0d85e6948b Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 2 Apr 2019 09:30:18 -0700 Subject: [PATCH 088/658] complex: move to impl block --- vm/src/obj/objcomplex.rs | 242 ++++++++++++++++++++------------------- 1 file changed, 122 insertions(+), 120 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 296830e43..9501ee7f3 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -35,17 +35,17 @@ pub fn init(context: &PyContext) { This is equivalent to (real + imag*1j) where imag defaults to 0."; extend_class!(context, complex_type, { - "__abs__" => context.new_rustfunc(complex_abs), - "__add__" => context.new_rustfunc(complex_add), "__doc__" => context.new_str(complex_doc.to_string()), - "__eq__" => context.new_rustfunc(complex_eq), - "__neg__" => context.new_rustfunc(complex_neg), - "__new__" => context.new_rustfunc(complex_new), - "__radd__" => context.new_rustfunc(complex_radd), - "__repr__" => context.new_rustfunc(complex_repr), - "conjugate" => context.new_rustfunc(complex_conjugate), - "imag" => context.new_property(complex_imag), - "real" => context.new_property(complex_real) + "__abs__" => context.new_rustfunc(PyComplexRef::abs), + "__add__" => context.new_rustfunc(PyComplexRef::add), + "__eq__" => context.new_rustfunc(PyComplexRef::eq), + "__neg__" => context.new_rustfunc(PyComplexRef::neg), + "__new__" => context.new_rustfunc(PyComplexRef::new), + "__radd__" => context.new_rustfunc(PyComplexRef::radd), + "__repr__" => context.new_rustfunc(PyComplexRef::repr), + "conjugate" => context.new_rustfunc(PyComplexRef::conjugate), + "imag" => context.new_property(PyComplexRef::imag), + "real" => context.new_property(PyComplexRef::real) }); } @@ -53,128 +53,130 @@ pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -fn complex_new( - cls: PyClassRef, - real: OptionalArg, - imag: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let real = match real { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; +impl PyComplexRef { + fn new( + cls: PyClassRef, + real: OptionalArg, + imag: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let real = match real { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let imag = match imag { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; + let imag = match imag { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let value = Complex64::new(real, imag); - PyComplex { value }.into_ref_with_type(vm, cls) -} - -fn complex_real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) -} - -fn complex_imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) -} - -fn complex_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) -} - -fn complex_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + let value = Complex64::new(real, imag); + PyComplex { value }.into_ref_with_type(vm, cls) } -} -fn complex_radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + fn real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + let Complex64 { re, .. } = get_value(zelf); + Ok(vm.ctx.new_float(re)) } -} -fn complex_conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + fn imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + let Complex64 { im, .. } = get_value(zelf); + Ok(vm.ctx.new_float(im)) + } - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) -} + fn abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); -fn complex_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); + let Complex64 { re, im } = get_value(zelf); + Ok(vm.ctx.new_float(re.hypot(im))) + } - let z = get_value(zelf); + fn add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(i, Some(vm.ctx.complex_type())), (i2, None)] + ); - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, - None => false, + let v1 = get_value(i); + if objtype::isinstance(i2, &vm.ctx.complex_type()) { + Ok(vm.ctx.new_complex(v1 + get_value(i2))) + } else if objtype::isinstance(i2, &vm.ctx.int_type()) { + Ok(vm.ctx.new_complex(Complex64::new( + v1.re + objint::get_value(i2).to_f64().unwrap(), + v1.im, + ))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) - } else { - return Ok(vm.ctx.not_implemented()); - }; + } - Ok(vm.ctx.new_bool(result)) -} + fn radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(i, Some(vm.ctx.complex_type())), (i2, None)] + ); -fn complex_neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) -} + let v1 = get_value(i); -fn complex_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) - } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + if objtype::isinstance(i2, &vm.ctx.int_type()) { + Ok(vm.ctx.new_complex(Complex64::new( + v1.re + objint::get_value(i2).to_f64().unwrap(), + v1.im, + ))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + } + } + + fn conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + + let v1 = get_value(i); + Ok(vm.ctx.new_complex(v1.conj())) + } + + fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] + ); + + let z = get_value(zelf); + + let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { + z == get_value(other) + } else if objtype::isinstance(other, &vm.ctx.int_type()) { + match objint::get_value(other).to_f64() { + Some(f) => z.im == 0.0f64 && z.re == f, + None => false, + } + } else if objtype::isinstance(other, &vm.ctx.float_type()) { + z.im == 0.0 && z.re == objfloat::get_value(other) + } else { + return Ok(vm.ctx.not_implemented()); + }; + + Ok(vm.ctx.new_bool(result)) + } + + fn neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + Ok(vm.ctx.new_complex(-get_value(zelf))) + } + + fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); + let v = get_value(obj); + let repr = if v.re == 0. { + format!("{}j", v.im) + } else { + format!("({}+{}j)", v.re, v.im) + }; + Ok(vm.new_str(repr)) + } } From 6863c19fb7aa8a87eae9e022080b586f092fcfcb Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 2 Apr 2019 09:51:01 -0700 Subject: [PATCH 089/658] complex: convert to new args style --- vm/src/obj/objcomplex.rs | 127 ++++++++++++++------------------------- 1 file changed, 46 insertions(+), 81 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 9501ee7f3..9f88dd6a4 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,11 +1,11 @@ use num_complex::Complex64; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objfloat; +use super::objfloat::{self, PyFloat}; use super::objint; use super::objtype::{self, PyClassRef}; @@ -74,109 +74,74 @@ impl PyComplexRef { PyComplex { value }.into_ref_with_type(vm, cls) } - fn real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) + fn real(self, _vm: &VirtualMachine) -> PyFloat { + self.value.re.into() } - fn imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) + fn imag(self, _vm: &VirtualMachine) -> PyFloat { + self.value.im.into() } - fn abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) + fn abs(self, _vm: &VirtualMachine) -> PyFloat { + let Complex64 { im, re } = self.value; + re.hypot(im).into() } - fn add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.complex_type()) { + vm.ctx.new_complex(self.value + get_value(&other)) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + vm.ctx.not_implemented() } } - fn radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) + fn radd(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + vm.ctx.not_implemented() } } - fn conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); - - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) + fn conjugate(self, _vm: &VirtualMachine) -> PyComplex { + self.value.conj().into() } - fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); - - let z = get_value(zelf); - - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { + self.value == get_value(&other) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + match objint::get_value(&other).to_f64() { + Some(f) => self.value.im == 0.0f64 && self.value.re == f, None => false, } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) + } else if objtype::isinstance(&other, &vm.ctx.float_type()) { + self.value.im == 0.0 && self.value.re == objfloat::get_value(&other) } else { - return Ok(vm.ctx.not_implemented()); + return vm.ctx.not_implemented(); }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.new_bool(result) } - fn neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) + fn neg(self, _vm: &VirtualMachine) -> PyComplex { + PyComplex::from(-self.value) } - fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) + fn repr(self, _vm: &VirtualMachine) -> String { + let Complex64 { re, im } = self.value; + if re == 0.0 { + format!("{}j", im) } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + format!("({}+{}j)", re, im) + } } } From 2fb3fc92ecaa13d031bdd3dd2e83ccf778660b46 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Tue, 2 Apr 2019 20:50:14 +0200 Subject: [PATCH 090/658] Support chained comparisons --- parser/src/ast.rs | 5 +- parser/src/parser.rs | 36 ++++++++------ parser/src/python.lalrpop | 10 +++- tests/snippets/ast_snippet.py | 7 +++ tests/snippets/comparisons.py | 13 +++++ vm/src/compile.rs | 93 ++++++++++++++++++++++++++++------- vm/src/stdlib/ast.rs | 20 ++++++-- 7 files changed, 141 insertions(+), 43 deletions(-) create mode 100644 tests/snippets/comparisons.py diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 257ceadf2..ece0e9798 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -159,9 +159,8 @@ pub enum Expression { value: Box, }, Compare { - a: Box, - op: Comparison, - b: Box, + vals: Vec, + ops: Vec, }, Attribute { value: Box, diff --git a/parser/src/parser.rs b/parser/src/parser.rs index a119740ec..4c37f2842 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -460,26 +460,30 @@ mod tests { }, ifs: vec![ ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Less, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(5) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(5) + } } - }), + ], + ops: vec![ast::Comparison::Less], }, ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Greater, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(10) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(10) + } } - }), + ], + ops: vec![ast::Comparison::Greater], }, ], } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 83020e926..f4c81f1d8 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -680,7 +680,15 @@ NotTest: ast::Expression = { }; Comparison: ast::Expression = { - => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) }, + => { + let mut vals = vec![e]; + let mut ops = vec![]; + for x in comparisons { + ops.push(x.0); + vals.push(x.1); + } + ast::Expression::Compare { vals, ops } + }, => e, }; diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index 43bf74756..dad767b45 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -21,3 +21,10 @@ print(foo.body[0].value.func.id) assert foo.body[0].value.func.id == 'print' assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 + +n = ast.parse("3 < 4 > 5\n") +assert n.body[0].value.left.n == 3 +assert 'Lt' in str(n.body[0].value.ops[0]) +assert 'Gt' in str(n.body[0].value.ops[1]) +assert n.body[0].value.comparators[0].n == 4 +assert n.body[0].value.comparators[1].n == 5 diff --git a/tests/snippets/comparisons.py b/tests/snippets/comparisons.py new file mode 100644 index 000000000..1a4597ef6 --- /dev/null +++ b/tests/snippets/comparisons.py @@ -0,0 +1,13 @@ + +assert 1 < 2 +assert 1 < 2 < 3 +assert 5 == 5 == 5 +assert (5 == 5) == True +assert 5 == 5 != 4 == 4 > 3 > 2 < 3 <= 3 != 0 == 0 + +assert not 1 > 2 +assert not 5 == 5 == True +assert not 5 == 5 != 5 == 5 +assert not 1 < 2 < 3 > 4 +assert not 1 < 2 > 3 < 4 +assert not 1 > 2 < 3 < 4 diff --git a/vm/src/compile.rs b/vm/src/compile.rs index f79980af9..fe295f424 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -840,6 +840,79 @@ impl Compiler { Ok(()) } + fn compile_chained_comparison( + &mut self, + vals: &[ast::Expression], + ops: &[ast::Comparison], + ) -> Result<(), CompileError> { + assert!(ops.len() > 0); + assert!(vals.len() == ops.len() + 1); + + let to_operator = |op: &ast::Comparison| match op { + ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, + ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, + ast::Comparison::Less => bytecode::ComparisonOperator::Less, + ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, + ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, + ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, + ast::Comparison::In => bytecode::ComparisonOperator::In, + ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, + ast::Comparison::Is => bytecode::ComparisonOperator::Is, + ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, + }; + + // a == b == c == d + // compile into (pseudocode): + // result = a == b + // if result: + // result = b == c + // if result: + // result = c == d + + // initialize lhs outside of loop + self.compile_expression(&vals[0])?; + + let break_label = self.new_label(); + let last_label = self.new_label(); + + // for all comparisons except the last (as the last one doesn't need a conditional jump) + let ops_slice = &ops[0..ops.len()]; + let vals_slice = &vals[1..ops.len()]; + for (op, val) in ops_slice.iter().zip(vals_slice.iter()) { + self.compile_expression(val)?; + // store rhs for the next comparison in chain + self.emit(Instruction::Duplicate); + self.emit(Instruction::Rotate { amount: 3 }); + + self.emit(Instruction::CompareOperation { + op: to_operator(op), + }); + + // if comparison result is false, we break with this value; if true, try the next one. + // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP) + self.emit(Instruction::Duplicate); + self.emit(Instruction::JumpIfFalse { + target: break_label, + }); + self.emit(Instruction::Pop); + } + + // handle the last comparison + self.compile_expression(vals.last().unwrap())?; + self.emit(Instruction::CompareOperation { + op: to_operator(ops.last().unwrap()), + }); + self.emit(Instruction::Jump { target: last_label }); + + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.set_label(break_label); + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::Pop); + + self.set_label(last_label); + Ok(()) + } + fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { @@ -1022,24 +1095,8 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Compare { a, op, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - - let i = match op { - ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, - ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, - ast::Comparison::Less => bytecode::ComparisonOperator::Less, - ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, - ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, - ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, - ast::Comparison::In => bytecode::ComparisonOperator::In, - ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, - ast::Comparison::Is => bytecode::ComparisonOperator::Is, - ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, - }; - let i = Instruction::CompareOperation { op: i }; - self.emit(i); + ast::Expression::Compare { vals, ops } => { + self.compile_chained_comparison(vals, ops)?; } ast::Expression::Number { value } => { let const_value = match value { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 0011407ab..91631b0c9 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -327,14 +327,14 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj node } - ast::Expression::Compare { a, op, b } => { + ast::Expression::Compare { vals, ops } => { let node = create_node(vm, "Compare"); - let py_a = expression_to_ast(vm, a); + let py_a = expression_to_ast(vm, &vals[0]); vm.ctx.set_attr(&node, "left", py_a); // Operator: - let str_op = match op { + let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => "Eq", ast::Comparison::NotEqual => "NotEq", ast::Comparison::Less => "Lt", @@ -346,10 +346,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = vm.ctx.new_list(vec![vm.ctx.new_str(str_op.to_string())]); + let py_ops = vm.ctx.new_list( + ops.iter() + .map(|x| vm.ctx.new_str(to_operator(x).to_string())) + .collect(), + ); + vm.ctx.set_attr(&node, "ops", py_ops); - let py_b = vm.ctx.new_list(vec![expression_to_ast(vm, b)]); + let py_b = vm.ctx.new_list( + vals.iter() + .skip(1) + .map(|x| expression_to_ast(vm, x)) + .collect(), + ); vm.ctx.set_attr(&node, "comparators", py_b); node } From 23f57f757ccb5074e4c1e0c557fcf48bffaf9870 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 3 Apr 2019 14:08:12 +0200 Subject: [PATCH 091/658] Improve trailing comma situation --- parser/src/ast.rs | 6 ++-- parser/src/python.lalrpop | 61 +++++++++++++++--------------------- tests/snippets/function.py | 4 +++ tests/snippets/generators.py | 7 +++++ vm/src/compile.rs | 21 +++---------- vm/src/stdlib/ast.rs | 5 ++- 6 files changed, 45 insertions(+), 59 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index ece0e9798..21c8dd736 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -47,7 +47,7 @@ pub enum Statement { Break, Continue, Return { - value: Option>, + value: Option>, }, Import { import_parts: Vec, @@ -94,7 +94,7 @@ pub enum Statement { }, For { target: Expression, - iter: Vec, + iter: Expression, body: Vec, orelse: Option>, }, @@ -114,12 +114,10 @@ pub enum Statement { bases: Vec, keywords: Vec, decorator_list: Vec, - // TODO: docstring: String, }, FunctionDef { name: String, args: Parameters, - // docstring: String, body: Vec, decorator_list: Vec, returns: Option, diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index f4c81f1d8..151e06de2 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -102,7 +102,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { + => { // TODO: this works in most cases: let rhs = e2.into_iter().next().unwrap(); ast::LocatedStatement { @@ -117,15 +117,7 @@ ExpressionStatement: ast::LocatedStatement = { }; AssignSuffix: ast::Expression = { - "=" => { - if e.len() > 1 { - ast::Expression::Tuple { - elements: e - } - } else { - e.into_iter().next().unwrap() - } - }, + "=" => e, "=" => e, }; @@ -182,7 +174,7 @@ FlowStatement: ast::LocatedStatement = { "return" => { ast::LocatedStatement { location: loc, - node: ast::Statement::Return { value: t }, + node: ast::Statement::Return { value: t.map(Box::new) }, } }, => { @@ -622,22 +614,8 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => { - ast::Expression::Yield { - value: ex.map(|expr| Box::new( - if expr.len() > 1 { - ast::Expression::Tuple { elements: expr } - } else { - expr.into_iter().next().unwrap() - }) - ) - } - }, - "yield" "from" => { - ast::Expression::YieldFrom { - value: Box::new(e), - } - }, + "yield" => ast::Expression::Yield { value: value.map(Box::new) }, + "yield" "from" => ast::Expression::YieldFrom { value: Box::new(e) }, }; Test: ast::Expression = { @@ -660,7 +638,7 @@ LambdaDef: ast::Expression = { "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body:Box::new(b) + body: Box::new(b) } } @@ -817,7 +795,7 @@ Atom: ast::Expression = { // List comprehension: e }, - "(" ")" => { + "(" ")" => { match e { None => ast::Expression::Tuple { elements: Vec::new() }, Some(elements) => { @@ -913,13 +891,26 @@ ExpressionList2: Vec = { }, }; +// TODO: this TestList2 should be renamed or removed. #[inline] -TestList: Vec = { - => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - } +TestList2: Vec = { + > => elements, +}; + +// A test list is one of: +// - a list of expressions +// - a single expression +// - a single expression followed by a trailing comma +TestList: ast::Expression = { + > => { + if e.len() == 1 && trailing_comma.is_none() { + e.into_iter().next().unwrap() + } else { + ast::Expression::Tuple { + elements: e + } + } + } }; // Test diff --git a/tests/snippets/function.py b/tests/snippets/function.py index 7d3f25709..aafc9b53c 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -10,6 +10,10 @@ def my_func(a,): assert my_func(2) == 4 +def fubar(): + return 42, + +assert fubar() == (42,) def f1(): diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 5f64c4905..65044c0da 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -35,3 +35,10 @@ r = list(g3()) # print(r) assert r == [23, 1, 2, 3, 44] +def g4(): + yield + yield 2, + +r = list(g4()) +assert r == [None, (2,)] + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index fe295f424..88818f52f 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -353,19 +353,8 @@ impl Compiler { return Err(CompileError::InvalidReturn); } match value { - Some(e) => { - let size = e.len(); - for v in e { - self.compile_expression(v)?; - } - - // If we have more than 1 return value, make it a tuple: - if size > 1 { - self.emit(Instruction::BuildTuple { - size, - unpack: false, - }); - } + Some(v) => { + self.compile_expression(v)?; } None => { self.emit(Instruction::LoadConst { @@ -796,7 +785,7 @@ impl Compiler { fn compile_for( &mut self, target: &ast::Expression, - iter: &[ast::Expression], + iter: &ast::Expression, body: &[ast::LocatedStatement], orelse: &Option>, ) -> Result<(), CompileError> { @@ -810,9 +799,7 @@ impl Compiler { }); // The thing iterated: - for i in iter { - self.compile_expression(i)?; - } + self.compile_expression(iter)?; // Retrieve Iterator self.emit(Instruction::GetIter); diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 91631b0c9..65fcdf660 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -151,8 +151,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "Return"); let py_value = if let Some(value) = value { - vm.ctx - .new_tuple(value.iter().map(|v| expression_to_ast(vm, v)).collect()) + expression_to_ast(vm, value) } else { vm.ctx.none() }; @@ -189,7 +188,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let py_target = expression_to_ast(vm, target); vm.ctx.set_attr(&node, "target", py_target); - let py_iter = expressions_to_ast(vm, iter); + let py_iter = expression_to_ast(vm, iter); vm.ctx.set_attr(&node, "iter", py_iter); let py_body = statements_to_ast(vm, body); From e078614e05b25b2a6dc08e1c81d1f0804512c6cb Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 3 Apr 2019 08:50:55 -0500 Subject: [PATCH 092/658] Change to classes and remove unnecesary if --- tests/snippets/whats_left_to_implement.py | 37 +++++++++++------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index a9a2d4de9..6abade9df 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,7 +1,7 @@ import platform expected_methods = { - 'bool': (True, [ + 'bool': (bool, [ '__abs__', '__add__', '__and__', @@ -73,7 +73,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'bytearray': (bytearray(), [ + 'bytearray': (bytearray, [ '__add__', '__alloc__', '__class__', @@ -159,7 +159,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'bytes': (b'', [ + 'bytes': (bytes, [ '__add__', '__class__', '__contains__', @@ -233,7 +233,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'complex': (0j, [ + 'complex': (complex, [ '__abs__', '__add__', '__bool__', @@ -284,7 +284,7 @@ expected_methods = { 'imag', 'real', ]), - 'dict': ({}, [ + 'dict': (dict, [ '__class__', '__contains__', '__delattr__', @@ -326,7 +326,7 @@ expected_methods = { 'update', 'values', ]), - 'float': (0.0, [ + 'float': (float, [ '__abs__', '__add__', '__bool__', @@ -384,7 +384,7 @@ expected_methods = { 'is_integer', 'real', ]), - 'frozenset': (frozenset(), [ + 'frozenset': (frozenset, [ '__and__', '__class__', '__contains__', @@ -428,7 +428,7 @@ expected_methods = { 'symmetric_difference', 'union', ]), - 'int': (0, [ + 'int': (int, [ '__abs__', '__add__', '__and__', @@ -529,7 +529,7 @@ expected_methods = { '__str__', '__subclasshook__', ]), - 'list': ([], [ + 'list': (list, [ '__add__', '__class__', '__contains__', @@ -577,7 +577,7 @@ expected_methods = { 'reverse', 'sort', ]), - 'memoryview': (memoryview(b''), [ + 'memoryview': (memoryview, [ '__class__', '__delattr__', '__delitem__', @@ -625,7 +625,7 @@ expected_methods = { 'tobytes', 'tolist', ]), - 'range': (range(0), [ + 'range': (range, [ '__bool__', '__class__', '__contains__', @@ -661,7 +661,7 @@ expected_methods = { 'step', 'stop', ]), - 'set': (set(), [ + 'set': (set, [ '__and__', '__class__', '__contains__', @@ -718,7 +718,7 @@ expected_methods = { 'union', 'update', ]), - 'str': ('', [ + 'str': (str, [ '__add__', '__class__', '__contains__', @@ -797,7 +797,7 @@ expected_methods = { 'upper', 'zfill' ]), - 'tuple': ((), [ + 'tuple': (tuple, [ '__add__', '__class__', '__contains__', @@ -832,7 +832,7 @@ expected_methods = { 'count', 'index', ]), - 'object': (object(), [ + 'object': (object, [ '__repr__', '__hash__', '__str__', @@ -864,10 +864,9 @@ not_implemented = [(name, method) for method in methods if not hasattr(val, method)] -if not_implemented: - for r in not_implemented: - print(r[0], ".", r[1], sep="") -else: +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: print("Not much \\o/") if platform.python_implementation() == "CPython": From 29a137bf0f8722fd449e18997ced1dc964c088ea Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 3 Apr 2019 22:24:19 +0200 Subject: [PATCH 093/658] Drop testlist2. Also add test cases for trailing comma in for statement and tuple addition example. --- parser/src/python.lalrpop | 182 ++++++++++++++------------------------ tests/snippets/for.py | 12 +++ tests/snippets/tuple.py | 4 + 3 files changed, 81 insertions(+), 117 deletions(-) diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 151e06de2..305f97c10 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -102,9 +102,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { - // TODO: this works in most cases: - let rhs = e2.into_iter().next().unwrap(); + => { ast::LocatedStatement { location: loc, node: ast::Statement::AugAssign { @@ -122,18 +120,12 @@ AssignSuffix: ast::Expression = { }; TestOrStarExprList: ast::Expression = { - => { - let mut res = vec![e]; - res.extend(e2.into_iter().map(|x| x.1)); - - // First build tuple from first item: - let expr = if (res.len() > 1) || comma.is_some() { - ast::Expression::Tuple { elements: res } + > => { + if elements.len() == 1 && comma.is_none() { + elements.into_iter().next().unwrap() } else { - res.into_iter().next().unwrap() - }; - - expr + ast::Expression::Tuple { elements } + } } }; @@ -301,15 +293,11 @@ NonlocalStatement: ast::LocatedStatement = { }; AssertStatement: ast::LocatedStatement = { - "assert" => { + "assert" => { ast::LocatedStatement { location: loc, node: ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, - } + test, msg: msg.map(|e| e.1) } } }, @@ -326,7 +314,7 @@ CompoundStatement: ast::LocatedStatement = { }; IfStatement: ast::LocatedStatement = { - "if" ":" => { + "if" ":" => { // Determine last else: let mut last = s3.map(|s| s.2); @@ -341,17 +329,17 @@ IfStatement: ast::LocatedStatement = { ast::LocatedStatement { location: loc, - node: ast::Statement::If { test: t, body: s1, orelse: last } + node: ast::Statement::If { test, body: s1, orelse: last } } }, }; WhileStatement: ast::LocatedStatement = { - "while" ":" => { + "while" ":" => { let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::While { test: e, body: s, orelse: or_else }, + node: ast::Statement::While { test, body, orelse: or_else }, } }, }; @@ -434,9 +422,7 @@ FuncDef: ast::LocatedStatement = { }; Parameters: ast::Parameters = { - "(" )?> ")" => { - a.unwrap_or_else(Default::default) - }, + "(" )?> ")" => a.unwrap_or_else(Default::default), }; // Note that this is a macro which is used once for function defs, and @@ -619,42 +605,41 @@ YieldExpr: ast::Expression = { }; Test: ast::Expression = { - => { - match c { - Some(c) => { - ast::Expression::IfExpression { - test: Box::new(c.1), - body: Box::new(e), - orelse: Box::new(c.3), - } - }, - None => e, + => { + if let Some(c) = condition { + ast::Expression::IfExpression { + test: Box::new(c.1), + body: Box::new(expr), + orelse: Box::new(c.3), + } + } else { + expr } }, - => e, + LambdaDef, }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body: Box::new(b) + body: Box::new(body) } } OrTest: ast::Expression = { - => e, + AndTest, "or" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }, }; AndTest: ast::Expression = { - => e, + NotTest, "and" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }, }; NotTest: ast::Expression = { "not" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }, - => e, + Comparison, }; Comparison: ast::Expression = { @@ -667,7 +652,7 @@ Comparison: ast::Expression = { } ast::Expression::Compare { vals, ops } }, - => e, + Expression, }; CompOp: ast::Comparison = { @@ -685,7 +670,7 @@ CompOp: ast::Comparison = { Expression: ast::Expression = { "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, - => e, + XorExpression, }; XorExpression: ast::Expression = { @@ -735,7 +720,7 @@ Factor: ast::Expression = { "+" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos }, "-" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg }, "~" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv }, - => e, + Power, }; Power: ast::Expression = { @@ -748,7 +733,7 @@ Power: ast::Expression = { }; AtomExpr: ast::Expression = { - => e, + Atom, "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, "." => ast::Expression::Attribute { value: Box::new(e), name: n }, @@ -769,7 +754,7 @@ SubscriptList: ast::Expression = { }; Subscript: ast::Expression = { - => e, + Test, ":" => { let s1 = e1.unwrap_or(ast::Expression::None); let s2 = e2.unwrap_or(ast::Expression::None); @@ -783,30 +768,17 @@ SliceOp: ast::Expression = { } Atom: ast::Expression = { - => ast::Expression::String { value: s }, - => ast::Expression::Bytes { value: b }, - => ast::Expression::Number { value: n }, - => ast::Expression::Identifier { name: i }, + => ast::Expression::String { value }, + => ast::Expression::Bytes { value }, + => ast::Expression::Number { value }, + => ast::Expression::Identifier { name }, "[" "]" => { let elements = e.unwrap_or(Vec::new()); ast::Expression::List { elements } }, - "[" "]" => { - // List comprehension: - e - }, - "(" ")" => { - match e { - None => ast::Expression::Tuple { elements: Vec::new() }, - Some(elements) => { - if elements.len() == 1 && trailing_comma.is_none() { - // This is "(e)", which is equivalent to "e" - elements.into_iter().next().unwrap() - } else { - ast::Expression::Tuple { elements } - } - } - } + "[" "]" => e, + "(" ")" => { + elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() }) }, "(" ")" => { ast::Expression::Comprehension { @@ -825,9 +797,7 @@ Atom: ast::Expression = { }; TestListComp: Vec = { - > <_trailing_comma:","?> => { - e - }, + > <_trailing_comma:","?> => e, }; TestListComp2: ast::Expression = { @@ -840,9 +810,7 @@ TestListComp2: ast::Expression = { }; TestDict: Vec<(ast::Expression, ast::Expression)> = { - > <_trailing_comma:","?> => { - e1 - } + > <_trailing_comma:","?> => elements, }; TestDictComp: ast::Expression = { @@ -859,9 +827,7 @@ DictEntry: (ast::Expression, ast::Expression) = { }; TestSet: Vec = { - > ","? => { - e1 - } + > ","? => e1 }; TestSetComp: ast::Expression = { @@ -873,28 +839,23 @@ TestSetComp: ast::Expression = { } }; +ExpressionOrStarExpression = { + Expression, + StarExpr +}; + ExpressionList: ast::Expression = { - => { - if e.len() == 1 { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements: e } + ast::Expression::Tuple { elements } } }, }; ExpressionList2: Vec = { - ","? => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - }, -}; - -// TODO: this TestList2 should be renamed or removed. -#[inline] -TestList2: Vec = { - > => elements, + > ","? => elements, }; // A test list is one of: @@ -902,13 +863,11 @@ TestList2: Vec = { // - a single expression // - a single expression followed by a trailing comma TestList: ast::Expression = { - > => { - if e.len() == 1 && trailing_comma.is_none() { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { - elements: e - } + ast::Expression::Tuple { elements } } } }; @@ -919,27 +878,16 @@ StarExpr: ast::Expression = { }; // Comprehensions: -CompFor: Vec = { - => c, -}; +CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { - ast::Comprehension { - target: e, - iter: i, - ifs: c2, - } + "for" "in" => { + ast::Comprehension { target, iter, ifs: c2 } } }; -ExpressionNoCond: ast::Expression = { - OrTest, -}; - -ComprehensionIf: ast::Expression = { - "if" => c, -}; +ExpressionNoCond: ast::Expression = OrTest; +ComprehensionIf: ast::Expression = "if" => c; ArgumentList: (Vec, Vec) = { > => { @@ -952,7 +900,7 @@ ArgumentList: (Vec, Vec) = { }, None => { if keywords.len() > 0 { - panic!("positional argument follows keyword argument"); + panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); }, @@ -996,8 +944,8 @@ OneOrMore: Vec = { }; Number: ast::Number = { - => { ast::Number::Integer { value: s } }, - => { ast::Number::Float { value: s } }, + => { ast::Number::Integer { value } }, + => { ast::Number::Float { value } }, => { ast::Number::Complex { real: s.0, imag: s.1 } }, }; diff --git a/tests/snippets/for.py b/tests/snippets/for.py index 07e138e3c..e913ac907 100644 --- a/tests/snippets/for.py +++ b/tests/snippets/for.py @@ -10,3 +10,15 @@ else: x = 3 assert x == 3 + +y = [] +for x, in [(9,), [2]]: + y.append(x) + +assert y == [9, 2], str(y) + +y = [] +for x, *z in [(9,88,'b'), [2, 'bla'], [None]*4]: + y.append(z) + +assert y == [[88, 'b'], ['bla'], [None]*3], str(y) diff --git a/tests/snippets/tuple.py b/tests/snippets/tuple.py index d50f552dc..b7393fe00 100644 --- a/tests/snippets/tuple.py +++ b/tests/snippets/tuple.py @@ -34,3 +34,7 @@ class Foo(object): foo = Foo() assert (foo,) == (foo,) + +a = (1, 2, 3) +a += 1, +assert a == (1, 2, 3, 1) From 88bf2a32c4452a92bb09c4a9b5513701c55f9aa3 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 3 Apr 2019 17:30:04 -0700 Subject: [PATCH 094/658] address feedback --- vm/src/macros.rs | 13 ++++++------- vm/src/obj/objint.rs | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 601151a05..47d3c531a 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -150,9 +150,8 @@ macro_rules! extend_class { /// Macro to match on the built-in class of a Python object. /// -/// Like `match`, `match_object!` is an expression, and so must have a default -/// arm that will be evaluated when the object was not an instance of any of -/// the classes specified in other arms. +/// Like `match`, `match_class!` must be exhaustive, so a default arm with +/// the uncasted object is required. /// /// # Examples /// @@ -173,7 +172,7 @@ macro_rules! extend_class { /// match_class!(obj.clone(), /// PyInt => "int", /// PyFloat => "float", -/// _ => "neither" +/// _ => "neither", /// ) /// ); /// @@ -197,7 +196,7 @@ macro_rules! extend_class { /// let int_value = match_class!(obj, /// i @ PyInt => i.as_bigint().clone(), /// f @ PyFloat => f.to_f64().to_bigint().unwrap(), -/// obj => panic!("non-numeric object {}", obj) +/// obj => panic!("non-numeric object {}", obj), /// ); /// /// assert!(int_value.is_zero()); @@ -205,12 +204,12 @@ macro_rules! extend_class { #[macro_export] macro_rules! match_class { // The default arm. - ($obj:expr, _ => $default:expr) => { + ($obj:expr, _ => $default:expr $(,)?) => { $default }; // The default arm, binding the original object to the specified identifier. - ($obj:expr, $binding:ident => $default:expr) => {{ + ($obj:expr, $binding:ident => $default:expr $(,)?) => {{ let $binding = $obj; $default }}; diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index c26cdf4c0..d7b6a0420 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -407,6 +407,7 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul } // Casting function: +// TODO: this should just call `__int__` on the object pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { match_class!(obj.clone(), i @ PyInt => Ok(i.as_bigint().clone()), From fa53d5cd655be6956fa0086d9441a866938e19f6 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Wed, 3 Apr 2019 21:11:03 -0600 Subject: [PATCH 095/658] impl list.__delitem__ using list.__setitem__ as a guide while it works as well as __setitem__ currently does this impliments no support for slice indexing or negative indexing, adtionaly a bad index panics insted of giving a vm type error! --- vm/src/obj/objlist.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 865707201..1558db6aa 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -306,6 +306,25 @@ impl PyListRef { Ok(vm.ctx.not_implemented()) } } + + fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.borrow_mut(); + if objtype::isinstance(&key, &vm.ctx.int_type()) { + let idx = objint::get_value(&key).to_i32().unwrap(); + if let Some(_) = elements.get_pos(idx) { + elements.remove(idx as usize); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list index out of range".to_string())) + } + + } else { + panic!( + "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + elements, key + ) + } + } } fn list_new( @@ -454,6 +473,7 @@ pub fn init(context: &PyContext) { "__iadd__" => context.new_rustfunc(PyListRef::iadd), "__bool__" => context.new_rustfunc(PyListRef::bool), "__contains__" => context.new_rustfunc(PyListRef::contains), + "__delitem__" => context.new_rustfunc(PyListRef::delitem), "__eq__" => context.new_rustfunc(PyListRef::eq), "__lt__" => context.new_rustfunc(PyListRef::lt), "__gt__" => context.new_rustfunc(PyListRef::gt), From 39d4d64a4cb1f15af9a5900df6af76c98bbffbb4 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 3 Apr 2019 17:11:26 +1300 Subject: [PATCH 096/658] Copy abc and dependent python files over from CPython 3.7 branch --- Lib/_py_abc.py | 147 +++++++++++++++++++++ Lib/_weakrefset.py | 196 ++++++++++++++++++++++++++++ Lib/abc.py | 170 +++++++++++++++++++++++++ PSF-LICENSE | 254 +++++++++++++++++++++++++++++++++++++ tests/snippets/abc_test.py | 4 + 5 files changed, 771 insertions(+) create mode 100644 Lib/_py_abc.py create mode 100644 Lib/_weakrefset.py create mode 100644 Lib/abc.py create mode 100644 PSF-LICENSE create mode 100644 tests/snippets/abc_test.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py new file mode 100644 index 000000000..3c3aa8e3d --- /dev/null +++ b/Lib/_py_abc.py @@ -0,0 +1,147 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py new file mode 100644 index 000000000..304c66f59 --- /dev/null +++ b/Lib/_weakrefset.py @@ -0,0 +1,196 @@ +# Access WeakSet through the weakref module. +# This code is separated-out because it is needed +# by abc.py to load everything else at startup. + +from _weakref import ref + +__all__ = ['WeakSet'] + + +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + l = self._pending_removals + discard = self.data.discard + while l: + discard(l.pop()) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return (self.__class__, (list(self),), + getattr(self, '__dict__', None)) + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + newset = self.copy() + newset.difference_update(other) + return newset + __sub__ = difference + + def difference_update(self, other): + self.__isub__(other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + __and__ = intersection + + def intersection_update(self, other): + self.__iand__(other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __le__ = issubset + + def __lt__(self, other): + return self.data < set(map(ref, other)) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __ge__ = issuperset + + def __gt__(self, other): + return self.data > set(map(ref, other)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(map(ref, other)) + + def symmetric_difference(self, other): + newset = self.copy() + newset.symmetric_difference_update(other) + return newset + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + self.__ixor__(other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) + return self + + def union(self, other): + return self.__class__(e for s in (self, other) for e in s) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 diff --git a/Lib/abc.py b/Lib/abc.py new file mode 100644 index 000000000..709414127 --- /dev/null +++ b/Lib/abc.py @@ -0,0 +1,170 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) according to PEP 3119.""" + + +def abstractmethod(funcobj): + """A decorator indicating abstract methods. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractmethod + def my_abstract_method(self, ...): + ... + """ + funcobj.__isabstractmethod__ = True + return funcobj + + +class abstractclassmethod(classmethod): + """A decorator indicating abstract classmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractclassmethod + def my_abstract_classmethod(cls, ...): + ... + + 'abstractclassmethod' is deprecated. Use 'classmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractstaticmethod(staticmethod): + """A decorator indicating abstract staticmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractstaticmethod + def my_abstract_staticmethod(...): + ... + + 'abstractstaticmethod' is deprecated. Use 'staticmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractproperty(property): + """A decorator indicating abstract properties. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract properties are overridden. + The abstract properties can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractproperty + def my_abstract_property(self): + ... + + This defines a read-only property; you can also define a read-write + abstract property using the 'long' form of property declaration: + + class C(metaclass=ABCMeta): + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) + + 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' + instead. + """ + + __isabstractmethod__ = True + + +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + from _py_abc import ABCMeta, get_cache_token + ABCMeta.__module__ = 'abc' +else: + class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () diff --git a/PSF-LICENSE b/PSF-LICENSE new file mode 100644 index 000000000..1afbedba9 --- /dev/null +++ b/PSF-LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py new file mode 100644 index 000000000..a5b53ea61 --- /dev/null +++ b/tests/snippets/abc_test.py @@ -0,0 +1,4 @@ +import sys +sys.path.append('Lib') + +import abc From 0bd08618afc87e5b787ca3a0b1efe3622bad2e7e Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 3 Apr 2019 20:36:53 +1300 Subject: [PATCH 097/658] Change _py_abc to workaround current issue, and run test with PYTHONPATH linking to Lib directory. --- Lib/_py_abc.py | 2 +- tests/snippets/abc_test.py | 3 --- tests/test_snippets.py | 3 ++- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 3c3aa8e3d..60adace9b 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -33,7 +33,7 @@ class ABCMeta(type): _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, **kwargs): - cls = super().__new__(mcls, name, bases, namespace, **kwargs) + cls = type.__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py index a5b53ea61..494f967be 100644 --- a/tests/snippets/abc_test.py +++ b/tests/snippets/abc_test.py @@ -1,4 +1 @@ -import sys -sys.path.append('Lib') - import abc diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7edbd7b7e..b3bc79ee7 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -27,7 +27,7 @@ TEST_DIRS = { } CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) - +RUSTPYTHON_LIB_DIR = os.path.abspath(os.path.join(ROOT_DIR, "Lib")) @contextlib.contextmanager def pushd(path): @@ -73,6 +73,7 @@ def run_via_rustpython(filename, test_type): env = os.environ.copy() env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' + env['PYTHONPATH'] = RUSTPYTHON_LIB_DIR target = 'release' if env.get('CODE_COVERAGE', 'false') == 'true': From ea872521d6e114cc99cbd7dc0f08ac2f4b2467bf Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 4 Apr 2019 19:45:20 +1300 Subject: [PATCH 098/658] Make object by default hashable --- Lib/_py_abc.py | 2 +- tests/snippets/abc_test.py | 1 - tests/snippets/hash.py | 23 +++++++++++++++++++++++ tests/snippets/test_abc.py | 34 ++++++++++++++++++++++++++++++++++ vm/src/obj/objdict.rs | 5 +++++ vm/src/obj/objlist.rs | 5 +++++ vm/src/obj/objobject.rs | 4 ++-- vm/src/obj/objset.rs | 5 +++++ 8 files changed, 75 insertions(+), 4 deletions(-) delete mode 100644 tests/snippets/abc_test.py create mode 100644 tests/snippets/hash.py create mode 100644 tests/snippets/test_abc.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 60adace9b..07044f2f7 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -43,7 +43,7 @@ class ABCMeta(type): value = getattr(cls, name, None) if getattr(value, "__isabstractmethod__", False): abstracts.add(name) - cls.__abstractmethods__ = frozenset(abstracts) + cls.__abstractmethods__ = set(abstracts) # Set up inheritance registry cls._abc_registry = WeakSet() cls._abc_cache = WeakSet() diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py deleted file mode 100644 index 494f967be..000000000 --- a/tests/snippets/abc_test.py +++ /dev/null @@ -1 +0,0 @@ -import abc diff --git a/tests/snippets/hash.py b/tests/snippets/hash.py new file mode 100644 index 000000000..b108db705 --- /dev/null +++ b/tests/snippets/hash.py @@ -0,0 +1,23 @@ + +from testutils import assertRaises + + +class A: + pass + + +assert type(hash(None)) is int +assert type(hash(object())) is int +assert type(hash(A())) is int +assert type(hash(1)) is int +assert type(hash(1.1)) is int +assert type(hash("")) is int + +with assertRaises(TypeError): + hash({}) + +with assertRaises(TypeError): + hash(set()) + +with assertRaises(TypeError): + hash([]) diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py new file mode 100644 index 000000000..6f0243fcd --- /dev/null +++ b/tests/snippets/test_abc.py @@ -0,0 +1,34 @@ +import abc + +from testutils import assertRaises + + +class CustomInterface(abc.ABC): + @abc.abstractmethod + def a(self): + pass + + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + +# with assertRaises(TypeError): +# CustomInterface() + + +class Concrete: + def a(self): + pass + + +CustomInterface.register(Concrete) + + +class SubConcrete(Concrete): + pass + + +assert issubclass(Concrete, CustomInterface) +assert issubclass(SubConcrete, CustomInterface) + diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 79c98c2dd..28b3f4210 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -295,6 +295,10 @@ impl PyDictRef { } } } + + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } } impl DictProtocol for PyDictRef { @@ -333,6 +337,7 @@ pub fn init(context: &PyContext) { "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 865707201..6f059b592 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -162,6 +162,10 @@ impl PyListRef { Ok(s) } + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } + fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { let new_elements = seq_mul(&self.elements.borrow(), counter); vm.ctx.new_list(new_elements) @@ -466,6 +470,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(PyListRef::len), "__new__" => context.new_rustfunc(list_new), "__repr__" => context.new_rustfunc(PyListRef::repr), + "__hash__" => context.new_rustfunc(PyListRef::hash), "__doc__" => context.new_str(list_doc.to_string()), "append" => context.new_rustfunc(PyListRef::append), "clear" => context.new_rustfunc(PyListRef::clear), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c43e31a96..df164f49b 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -55,8 +55,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py vm.ctx.not_implemented() } -fn object_hash(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("unhashable type".to_string())) +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 { + zelf.get_id() as u64 } fn object_setattr( diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index fb558c990..34d487fbb 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -588,6 +588,10 @@ fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.new_str(s)) } +fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) +} + pub fn init(context: &PyContext) { let set_type = &context.set_type; @@ -600,6 +604,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(set_len), "__new__" => context.new_rustfunc(set_new), "__repr__" => context.new_rustfunc(set_repr), + "__hash__" => context.new_rustfunc(set_hash), "__eq__" => context.new_rustfunc(set_eq), "__ge__" => context.new_rustfunc(set_ge), "__gt__" => context.new_rustfunc(set_gt), From c01dc5310a45fee5b43b98455be66d8d0d88b630 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 4 Apr 2019 20:10:47 +1300 Subject: [PATCH 099/658] Add callback parameter to weakref, at the moment this callback isn't yet implement though. --- tests/snippets/test_abc.py | 5 +++++ vm/src/obj/objweakref.rs | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py index 6f0243fcd..391932c36 100644 --- a/tests/snippets/test_abc.py +++ b/tests/snippets/test_abc.py @@ -13,6 +13,7 @@ class CustomInterface(abc.ABC): return NotImplemented +# TODO raise an error if there are in any abstract methods not fulfilled # with assertRaises(TypeError): # CustomInterface() @@ -31,4 +32,8 @@ class SubConcrete(Concrete): assert issubclass(Concrete, CustomInterface) assert issubclass(SubConcrete, CustomInterface) +assert not issubclass(tuple, CustomInterface) +assert isinstance(Concrete(), CustomInterface) +assert isinstance(SubConcrete(), CustomInterface) +assert not isinstance((), CustomInterface) diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 685e32b70..05d6147ce 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -3,6 +3,7 @@ use crate::pyobject::PyValue; use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; +use crate::function::OptionalArg; use std::rc::{Rc, Weak}; #[derive(Debug)] @@ -32,7 +33,12 @@ pub type PyWeakRef = PyRef; impl PyWeakRef { // TODO callbacks - fn create(cls: PyClassRef, referent: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { PyWeak::downgrade(&referent).into_ref_with_type(vm, cls) } From aba059b81ae9d579134cbd521e624a44e3af3b93 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 02:52:51 -0600 Subject: [PATCH 100/658] impl PySliceableSequenceMut & objseqence::del_item used to impl list.__delitem__ --- vm/src/obj/objlist.rs | 39 +++++++---- vm/src/obj/objsequence.rs | 139 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 15 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 1558db6aa..97a798f35 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,8 +1,11 @@ use std::cell::{Cell, RefCell}; use std::fmt; +use std::ops::DerefMut; + use num_traits::ToPrimitive; + use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -12,7 +15,7 @@ use super::objint; use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, + PySliceableSequence, del_item, }; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -309,21 +312,27 @@ impl PyListRef { fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.borrow_mut(); - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(_) = elements.get_pos(idx) { - elements.remove(idx as usize); - Ok(vm.get_none()) - } else { - Err(vm.new_index_error("list index out of range".to_string())) - } + // if objtype::isinstance(&key, &vm.ctx.int_type()) { + // let idx = objint::get_value(&key).to_i32().unwrap(); + // if let Some(_) = elements.get_pos(idx) { + // elements.remove(idx as usize); + // Ok(vm.get_none()) + // } else { + // Err(vm.new_index_error("list index out of range".to_string())) + // } - } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) - } + // } else { + // panic!( + // "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + // elements, key + // ) + // } + del_item( + vm, + self.as_object(), + elements.deref_mut(), + key, + ) } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a07b2454d..1d700e312 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -137,6 +137,108 @@ impl PySliceableSequence for Vec { } } +pub trait PySliceableSequenceMut : PySliceableSequence { + //fn set_slice(&self, range: Range, seq: T) -> Self; + //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; + //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; + //fn set_stepped_slice_reverse(&self, range: Range, step: usize, seq: T) -> Self; + + fn del_slice(&mut self, range: Range); + fn del_stepped_slice(&mut self, range: Range, step: usize); + fn del_index(&mut self, index: usize); + + fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PyObjectRef) -> PyResult { + // TODO: we could potentially avoid this copy and use slice + match slice.payload() { + Some(PySlice { start, stop, step }) => { + let step = step.clone().unwrap_or_else(BigInt::one); + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(start, stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + }, + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + }, + None => { + self.del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + }, + } + } else { + // TODO what to delete here? + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + }, + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + }, + None => { + self.del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + }, + } + } else { + // TODO what to del here? + Ok(vm.get_none()) + } + } + } + payload => panic!("del_slice_items called with non-slice: {:?}", payload), + } + } +} + +impl PySliceableSequenceMut for Vec { + fn del_slice(&mut self, range: Range) { + self.drain(range); + } + + fn del_stepped_slice(&mut self, range: Range, step: usize) { + // no easy way to delete steped indexes so heres what we'll do + let len = self.len(); + let mut deleted = 0; + let mut indexes = range.step_by(step).peekable(); + for i in 0..len { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + self.swap(i - deleted, i); + } + } + // then truncate the vec (the values to delete should now be the last elements and thus choped off) + self.truncate(len - deleted); + } + + fn del_index(&mut self, index: usize) { + self.remove(index); + } +} + + pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, @@ -183,6 +285,43 @@ pub fn get_item( } } +pub fn del_item ( + vm: &VirtualMachine, + sequence: &PyObjectRef, + elements: &mut T, + subscript: PyObjectRef, +) -> PyResult { + if let Some(i) = subscript.payload::() { + return match i.as_bigint().to_i32() { + Some(value) => { + if let Some(pos_index) = elements.get_pos(value) { + elements.del_index(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + None => { + Err(vm.new_index_error("cannot fit 'int' into an index-sized integer".to_string())) + } + }; + } + + if subscript.payload::().is_some() { + + if sequence.payload::().is_some() { + elements.del_slice_items(vm, &subscript) + } else { + panic!("sequence del_item called for non-mutable-sequence") + } + } else { + Err(vm.new_type_error(format!( + "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + sequence, subscript + ))) + } +} + pub fn seq_equal( vm: &VirtualMachine, zelf: &[PyObjectRef], From c97f342ef772f998cabed97baf6ebc8d916d46fc Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 03:18:58 -0600 Subject: [PATCH 101/658] cleanup and rustfmt --- vm/src/obj/objlist.rs | 27 +++------------------------ vm/src/obj/objsequence.rs | 18 ++++++++---------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 97a798f35..6d7dc770c 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -5,7 +5,6 @@ use std::ops::DerefMut; use num_traits::ToPrimitive; - use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -14,8 +13,8 @@ use super::objbool; use super::objint; use super::objiter; use super::objsequence::{ - get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, del_item, + del_item, get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, + seq_mul, PySliceableSequence, }; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -312,27 +311,7 @@ impl PyListRef { fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.borrow_mut(); - // if objtype::isinstance(&key, &vm.ctx.int_type()) { - // let idx = objint::get_value(&key).to_i32().unwrap(); - // if let Some(_) = elements.get_pos(idx) { - // elements.remove(idx as usize); - // Ok(vm.get_none()) - // } else { - // Err(vm.new_index_error("list index out of range".to_string())) - // } - - // } else { - // panic!( - // "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - // elements, key - // ) - // } - del_item( - vm, - self.as_object(), - elements.deref_mut(), - key, - ) + del_item(vm, self.as_object(), elements.deref_mut(), key) } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 1d700e312..2168166e8 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -137,7 +137,7 @@ impl PySliceableSequence for Vec { } } -pub trait PySliceableSequenceMut : PySliceableSequence { +pub trait PySliceableSequenceMut: PySliceableSequence { //fn set_slice(&self, range: Range, seq: T) -> Self; //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; @@ -162,15 +162,15 @@ pub trait PySliceableSequenceMut : PySliceableSequence { Some(1) => { self.del_slice(range); Ok(vm.get_none()) - }, + } Some(num) => { self.del_stepped_slice(range, num as usize); Ok(vm.get_none()) - }, + } None => { self.del_slice(range.start..range.start + 1); Ok(vm.get_none()) - }, + } } } else { // TODO what to delete here? @@ -187,15 +187,15 @@ pub trait PySliceableSequenceMut : PySliceableSequence { Some(1) => { self.del_slice(range); Ok(vm.get_none()) - }, + } Some(num) => { self.del_stepped_slice(range, num as usize); Ok(vm.get_none()) - }, + } None => { self.del_slice(range.end - 1..range.end); Ok(vm.get_none()) - }, + } } } else { // TODO what to del here? @@ -238,7 +238,6 @@ impl PySliceableSequenceMut for Vec { } } - pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, @@ -285,7 +284,7 @@ pub fn get_item( } } -pub fn del_item ( +pub fn del_item( vm: &VirtualMachine, sequence: &PyObjectRef, elements: &mut T, @@ -308,7 +307,6 @@ pub fn del_item ( } if subscript.payload::().is_some() { - if sequence.payload::().is_some() { elements.del_slice_items(vm, &subscript) } else { From e3994010f3595bc39508d1beb8b9bf9c25aaaacd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 04:04:26 -0600 Subject: [PATCH 102/658] add list.__delitem__ test snippits --- tests/snippets/list.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6b9839321..6efa52cc0 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -170,3 +170,30 @@ def f(x): return x assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] + +# __delitem__ +x = ['a', 'b', 'c'] +del x[0] +assert x == ['b', 'c'] + +x = ['a', 'b', 'c'] +del x[-1] +assert x == ['a', 'b'] + +x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[2:14:3] +assert x == y +assert x == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] +assert y == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] + +x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[-5:] +assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] + +def bad_del_1(): + del ['a', 'b']['a'] +assert_raises(TypeError, bad_del_1) + +def bad_del_2(): + del ['a', 'b'][2] +assert_raises(IndexError, bad_del_2) From d0e6c3bd9266ce0c70e9b8d8d4ffdbc414a8c8e8 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 04:52:33 -0600 Subject: [PATCH 103/658] convert to PySliceRef as it is the only option --- vm/src/obj/objsequence.rs | 113 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 2168166e8..9593e57fe 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -11,7 +11,7 @@ use crate::vm::VirtualMachine; use super::objbool; use super::objint::PyInt; use super::objlist::PyList; -use super::objslice::PySlice; +use super::objslice::{PySlice, PySliceRef}; use super::objtuple::PyTuple; pub trait PySliceableSequence { @@ -147,64 +147,64 @@ pub trait PySliceableSequenceMut: PySliceableSequence { fn del_stepped_slice(&mut self, range: Range, step: usize); fn del_index(&mut self, index: usize); - fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PyObjectRef) -> PyResult { - // TODO: we could potentially avoid this copy and use slice - match slice.payload() { - Some(PySlice { start, stop, step }) => { - let step = step.clone().unwrap_or_else(BigInt::one); - if step.is_zero() { - Err(vm.new_value_error("slice step cannot be zero".to_string())) - } else if step.is_positive() { - let range = self.get_slice_range(start, stop); - if range.start < range.end { - #[allow(clippy::range_plus_one)] - match step.to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.start..range.start + 1); - Ok(vm.get_none()) - } - } - } else { - // TODO what to delete here? + fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PySliceRef) -> PyResult { + + let start = &slice.start; + let stop = &slice.stop; + + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(start, stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self.del_slice(range); Ok(vm.get_none()) } - } else { - // calculate the range for the reverse slice, first the bounds needs to be made - // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); - let range = self.get_slice_range(&stop, &start); - if range.start < range.end { - match (-step).to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.end - 1..range.end); - Ok(vm.get_none()) - } - } - } else { - // TODO what to del here? + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self.del_slice(range.start..range.start + 1); Ok(vm.get_none()) } } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self.del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) } - payload => panic!("del_slice_items called with non-slice: {:?}", payload), } + } } @@ -305,10 +305,15 @@ pub fn del_item( } }; } - + if subscript.payload::().is_some() { if sequence.payload::().is_some() { - elements.del_slice_items(vm, &subscript) + if let Ok(slice) = subscript.downcast::() { + elements.del_slice_items(vm, &slice) + } else { + panic!("PySlice is not a slice? this should not be") + } + } else { panic!("sequence del_item called for non-mutable-sequence") } From 05ede30c519f117998e9282bfed219e049cfd673 Mon Sep 17 00:00:00 2001 From: Joey Date: Thu, 4 Apr 2019 07:52:53 -0700 Subject: [PATCH 104/658] range index --- vm/src/obj/objrange.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index ae946d05e..71adb4118 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,12 +6,14 @@ use num_integer::Integer; use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; use super::objiter; -use super::objslice::PySliceRef; +use super::objslice::{PySlice, PySliceRef}; use super::objtype::PyClassRef; #[derive(Debug, Clone)] @@ -261,16 +263,16 @@ impl PyRangeRef { } } - fn getitem(self, subscript: Either, vm: &VirtualMachine) -> PyResult { + fn getitem(self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { - Either::A(index) => { + RangeIndex::Int(index) => { if let Some(value) = self.get(index.as_bigint()) { Ok(PyInt::new(value).into_ref(vm).into_object()) } else { Err(vm.new_index_error("range object index out of range".to_string())) } } - Either::B(slice) => { + RangeIndex::Slice(slice) => { let new_start = if let Some(int) = slice.start.as_ref() { if let Some(i) = self.get(int) { PyInt::new(i).into_ref(vm) @@ -349,3 +351,21 @@ impl PyRangeIteratorRef { self } } + +pub enum RangeIndex { + Int(PyIntRef), + Slice(PySliceRef), +} + +impl TryFromObject for RangeIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(RangeIndex::Int(i)), + s @ PySlice => Ok(RangeIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } +} From d96a5b11f7574d7214509c750d94e043a79424d3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 4 Apr 2019 21:32:04 +0200 Subject: [PATCH 105/658] add resources to snippets --- .gitignore | 1 + tests/snippets/builtin_slice.py | 66 +- tests/snippets/slice_res.py | 2252 ------------------------------- tests/test_snippets.py | 112 +- 4 files changed, 104 insertions(+), 2327 deletions(-) delete mode 100644 tests/snippets/slice_res.py diff --git a/.gitignore b/.gitignore index 967a51a83..e9dd6d220 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode wasm-pack.log .idea/ +tests/snippets/resources \ No newline at end of file diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 619356211..d61698034 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -78,53 +78,35 @@ def test_all_slices(): """ test all possible slices except big number """ - MODE = None # set to "build" to rebuild slice_res.py - ll = [0, 1, 2, 3] - start = list(range(-7, 7)) - end = list(range(-7, 7)) - step = list(range(-5, 5)) - step.pop(step.index(0)) - for i in [start, end, step]: - i.append(None) + import resources + from resources.cpython_generated_slices import SLICES_RES, START, END, STEP, LL - def build(): - # loop used to build slices_res.py with cpython - with open("slice_res.py", "wt") as f: - for s in start: - for e in end: - for t in step: - f.write(str(ll[s:e:t]) + "\n") + ll = LL + start = START + end = END + step = STEP - def run(): - # test utility - from slice_res import SLICES_RES - - count = 0 - failures = [] - for s in start: - for e in end: - for t in step: - lhs = ll[s:e:t] - try: - assert lhs == SLICES_RES[count] - except AssertionError: - failures.append( - "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( - s, e, t, lhs, SLICES_RES[count] - ) + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == SLICES_RES[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, SLICES_RES[count] ) - count += 1 + ) + count += 1 - if failures: - for f in failures: - print(f) - print(len(failures), "slices failed") - - if MODE == "build": - build() - else: - run() + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") test_all_slices() diff --git a/tests/snippets/slice_res.py b/tests/snippets/slice_res.py deleted file mode 100644 index 38eb87729..000000000 --- a/tests/snippets/slice_res.py +++ /dev/null @@ -1,2252 +0,0 @@ -SLICES_RES = [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [2, 3], - [2], - [2], - [2], - [2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [3], - [3], - [3], - [3], - [3], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [2, 3], - [2], - [2], - [2], - [2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [0], - [0], - [0], - [0], - [0], - [3], - [3], - [3], - [3], - [3], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [0], - [0], - [0], - [0], - [0], - [3], - [3], - [3], - [3], - [3], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], -] diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7edbd7b7e..7a888f093 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -1,4 +1,3 @@ - # This is a python unittest class automatically populating with all tests # in the tests folder. @@ -11,6 +10,8 @@ import logging import subprocess import contextlib import enum +from pathlib import Path +import shutil import compile_code @@ -19,14 +20,15 @@ class _TestType(enum.Enum): functional = 1 -logger = logging.getLogger('tests') -ROOT_DIR = '..' -TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) -TEST_DIRS = { - _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), -} -CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) +logger = logging.getLogger("tests") +ROOT_DIR = ".." +TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) +TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} +CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) +RESOURCES_DIR = os.path.abspath( + os.path.join(TEST_DIRS[_TestType.functional], "resources") +) @contextlib.contextmanager @@ -38,12 +40,12 @@ def pushd(path): def perform_test(filename, method, test_type): - logger.info('Running %s via %s', filename, method) - if method == 'cpython': + logger.info("Running %s via %s", filename, method) + if method == "cpython": run_via_cpython(filename) - elif method == 'cpython_bytecode': + elif method == "cpython_bytecode": run_via_cpython_bytecode(filename, test_type) - elif method == 'rustpython': + elif method == "rustpython": run_via_rustpython(filename, test_type) else: raise NotImplementedError(method) @@ -57,27 +59,27 @@ def run_via_cpython(filename): def run_via_cpython_bytecode(filename, test_type): # Step1: Create bytecode file: - bytecode_filename = filename + '.bytecode' - with open(bytecode_filename, 'w') as f: + bytecode_filename = filename + ".bytecode" + with open(bytecode_filename, "w") as f: compile_code.compile_to_bytecode(filename, out_file=f) # Step2: run cpython bytecode: env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" with pushd(CPYTHON_RUNNER_DIR): - subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) + subprocess.check_call(["cargo", "run", bytecode_filename], env=env) def run_via_rustpython(filename, test_type): env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" - target = 'release' - if env.get('CODE_COVERAGE', 'false') == 'true': - target = 'debug' - binary = os.path.abspath(os.path.join(ROOT_DIR, 'target', target, 'rustpython')) + target = "release" + if env.get("CODE_COVERAGE", "false") == "true": + target = "debug" + binary = os.path.abspath(os.path.join(ROOT_DIR, "target", target, "rustpython")) subprocess.check_call([binary, filename], env=env) @@ -85,15 +87,15 @@ def run_via_rustpython(filename, test_type): def create_test_function(cls, filename, method, test_type): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) - test_function_name = 'test_{}_'.format(method) \ - + os.path.splitext(snippet_filename)[0] \ - .replace('.', '_').replace('-', '_') + test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ + 0 + ].replace(".", "_").replace("-", "_") def test_function(self): perform_test(filename, method, test_type) if hasattr(cls, test_function_name): - raise ValueError('Duplicate test case {}'.format(test_function_name)) + raise ValueError("Duplicate test case {}".format(test_function_name)) setattr(cls, test_function_name, test_function) @@ -103,25 +105,69 @@ def populate(method): for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls + return wrapper def get_test_files(): """ Retrieve test files """ for test_type, test_dir in TEST_DIRS.items(): - for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))): + for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] - if filename.startswith('xfail_'): + if filename.startswith("xfail_"): continue yield test_type, os.path.abspath(filepath) -@populate('cpython') +def generate_slices(path): + # loop used to build slices_res.py with cpython + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + for i in [start, end, step]: + i.append(None) + + slices_res = [] + for s in start: + for e in end: + for t in step: + slices_res.append(ll[s:e:t]) + + path.write_text( + "SLICES_RES={}\nSTART= {}\nEND= {}\nSTEP= {}\nLL={}\n".format( + slices_res, start, end, step, ll + ) + ) + + +@populate("cpython") # @populate('cpython_bytecode') -@populate('rustpython') +@populate("rustpython") class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - subprocess.check_call(['cargo', 'build']) - subprocess.check_call(['cargo', 'build', '--release']) + # setup resource dir + cls.resources = Path(RESOURCES_DIR) + if cls.resources.exists(): + shutil.rmtree( + RESOURCES_DIR, ignore_errors=True + ) # we don't care if dir doesnt exist + cls.resources.mkdir() + (cls.resources / "__init__.py").touch() + + # Here add resource files + cls.slices_resource_path = Path(RESOURCES_DIR) / "cpython_generated_slices.py" + generate_slices(cls.slices_resource_path) + + # cargo stuff + subprocess.check_call(["cargo", "build"]) + subprocess.check_call(["cargo", "build", "--release"]) + + @classmethod + def tearDownClass(cls): + shutil.rmtree( + RESOURCES_DIR, ignore_errors=True + ) # we don't care if dir doesnt exist From 79caaa5a0b025ab18462667f1b6bf3fe14a9ebe9 Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 5 Apr 2019 19:19:55 +1300 Subject: [PATCH 106/658] Move PSF-LICENSE and add Lib README.md --- PSF-LICENSE => Lib/PSF-LICENSE | 0 Lib/README.md | 10 ++++++++++ Lib/re.py | 11 ----------- 3 files changed, 10 insertions(+), 11 deletions(-) rename PSF-LICENSE => Lib/PSF-LICENSE (100%) create mode 100644 Lib/README.md delete mode 100644 Lib/re.py diff --git a/PSF-LICENSE b/Lib/PSF-LICENSE similarity index 100% rename from PSF-LICENSE rename to Lib/PSF-LICENSE diff --git a/Lib/README.md b/Lib/README.md new file mode 100644 index 000000000..b3aade7bf --- /dev/null +++ b/Lib/README.md @@ -0,0 +1,10 @@ +Standard Library for RustPython +=============================== + +This directory contains all of the Python files that make up the standard library for RustPython. + +Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them +to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. +Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation. + +The first target is to run the ``unittest`` module, so we can leverage the CPython test suite. diff --git a/Lib/re.py b/Lib/re.py deleted file mode 100644 index 8511222be..000000000 --- a/Lib/re.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" Regular expressions """ - - -def match(pattern, string, flags=0): - return _compile(pattern, flags).match(string) - - -def _compile(pattern, flags): - p = sre_compile.compile(pattern, flags) - return p From bce4f1e48373f6d4bcdc21fce17c6002505ab6fa Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 3 Apr 2019 17:23:27 +0100 Subject: [PATCH 107/658] Simplify/shrink the dict interface. --- vm/src/frame.rs | 15 ++--- vm/src/import.rs | 6 +- vm/src/obj/objdict.rs | 120 +++++++++------------------------------- vm/src/obj/objobject.rs | 4 +- vm/src/obj/objstr.rs | 6 ++ vm/src/obj/objtype.rs | 8 +-- vm/src/pyobject.rs | 69 +++++++++-------------- vm/src/stdlib/json.rs | 11 ++-- 8 files changed, 78 insertions(+), 161 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index b3cf2149b..6a09f085a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -385,21 +385,21 @@ impl Frame { Ok(None) } bytecode::Instruction::BuildMap { size, unpack } => { - let map_obj = vm.ctx.new_dict().into_object(); + let map_obj = vm.ctx.new_dict(); for _x in 0..*size { let obj = self.pop_value(); if *unpack { // Take all key-value pairs from the dict: let dict_elements = objdict::get_key_value_pairs(&obj); for (key, value) in dict_elements.iter() { - objdict::set_item(&map_obj, vm, key, value); + map_obj.set_item(&vm.ctx, &objstr::get_value(key), value.clone()); } } else { let key = self.pop_value(); - objdict::set_item(&map_obj, vm, &key, &obj); + map_obj.set_item(&vm.ctx, &objstr::get_value(&key), obj) } } - self.push_value(map_obj); + self.push_value(map_obj.into_object()); Ok(None) } bytecode::Instruction::BuildSlice { size } => { @@ -613,10 +613,10 @@ impl Frame { bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { let kw_dict = self.pop_value(); - let dict_elements = objdict::get_elements(&kw_dict).clone(); + let dict_elements = objdict::get_key_value_pairs(&kw_dict).clone(); dict_elements .into_iter() - .map(|elem| (elem.0, (elem.1).1)) + .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { vec![] @@ -1210,7 +1210,8 @@ impl fmt::Debug for Frame { .map(|elem| format!("\n > {:?}", elem)) .collect::(); let dict = self.scope.get_locals(); - let local_str = objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) + let local_str = dict + .get_key_value_pairs() .iter() .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) .collect::(); diff --git a/vm/src/import.rs b/vm/src/import.rs index 936b89680..179dd4321 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use crate::compile; use crate::frame::Scope; use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, PyResult}; +use crate::pyobject::{DictProtocol, ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; @@ -47,11 +47,11 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { // First, see if we already loaded the module: let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; - if let Some(module) = sys_modules.get_item(module_name) { + if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { return Ok(module); } let module = import_uncached_module(vm, current_path, module_name)?; - sys_modules.set_item(&vm.ctx, module_name, module.clone()); + sys_modules.set_item(module_name, module.clone(), vm)?; Ok(module) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 79c98c2dd..15046a405 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,7 +1,6 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::fmt; -use std::ops::{Deref, DerefMut}; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ @@ -37,94 +36,16 @@ impl PyValue for PyDict { } } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap().entries.borrow() -} - -pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut + 'a { - obj.payload::().unwrap().entries.borrow_mut() -} - -pub fn set_item( - dict: &PyObjectRef, - _vm: &VirtualMachine, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // TODO: use vm to call eventual __hash__ and __eq__methods. - let mut elements = get_mut_elements(dict); - set_item_in_content(&mut elements, needle, value); -} - -pub fn set_item_in_content( - elements: &mut DictContentType, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // XXX: Currently, we only support String keys, so we have to unwrap the - // PyObject (and ensure it is a String). - - // TODO: invoke __hash__ function here! - let needle_str = objstr::get_value(needle); - elements.insert(needle_str, (needle.clone(), value.clone())); -} - pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = get_elements(dict); - get_key_value_pairs_from_content(&dict_elements) -} - -pub fn get_key_value_pairs_from_content( - dict_content: &DictContentType, -) -> Vec<(PyObjectRef, PyObjectRef)> { + let dict_elements = dict.payload::().unwrap().entries.borrow(); let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_content.iter() { + for (_str_key, pair) in dict_elements.iter() { let (key, obj) = pair; pairs.push((key.clone(), obj.clone())); } pairs } -pub fn get_item(dict: &PyObjectRef, key: &PyObjectRef) -> Option { - let needle_str = objstr::get_value(key); - get_key_str(dict, &needle_str) -} - -// Special case for the case when requesting a str key from a dict: -pub fn get_key_str(dict: &PyObjectRef, key: &str) -> Option { - let elements = get_elements(dict); - content_get_key_str(&elements, key) -} - -/// Retrieve a key from dict contents: -pub fn content_get_key_str(elements: &DictContentType, key: &str) -> Option { - // TODO: let hash: usize = key; - match elements.get(key) { - Some(v) => Some(v.1.clone()), - None => None, - } -} - -pub fn contains_key_str(dict: &PyObjectRef, key: &str) -> bool { - let elements = get_elements(dict); - content_contains_key_str(&elements, key) -} - -pub fn content_contains_key_str(elements: &DictContentType, key: &str) -> bool { - // TODO: let hash: usize = key; - elements.get(key).is_some() -} - -/// Take a python dictionary and convert it to attributes. -pub fn py_dict_to_attributes(dict: &PyObjectRef) -> PyAttributes { - let mut attrs = PyAttributes::new(); - for (key, value) in get_key_value_pairs(dict) { - let key = objstr::get_value(&key); - attrs.insert(key, value); - } - attrs -} - // Python dict methods: impl PyDictRef { fn new( @@ -137,7 +58,7 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { for (needle, value) in get_key_value_pairs(&dict_obj) { - set_item(dict.as_object(), vm, &needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -156,13 +77,13 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - set_item(dict.as_object(), vm, &needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); } } } for (needle, value) in kwargs.into_iter() { let py_needle = vm.new_str(needle); - set_item(&dict.as_object(), vm, &py_needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&py_needle), value); } Ok(dict) } @@ -259,9 +180,8 @@ impl PyDictRef { } } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, _vm: &VirtualMachine) { - let mut elements = self.entries.borrow_mut(); - set_item_in_content(&mut elements, &needle, &value) + fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { + self.set_item(&vm.ctx, &objstr::get_value(&needle), value) } fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { @@ -295,15 +215,28 @@ impl PyDictRef { } } } + + /// Take a python dictionary and convert it to attributes. + pub fn to_attributes(self) -> PyAttributes { + let mut attrs = PyAttributes::new(); + for (key, value) in self.get_key_value_pairs() { + let key = objstr::get_value(&key); + attrs.insert(key, value); + } + attrs + } } impl DictProtocol for PyDictRef { fn contains_key(&self, k: &str) -> bool { - content_contains_key_str(&self.entries.borrow(), k) + self.entries.borrow().get(k).is_some() } fn get_item(&self, k: &str) -> Option { - content_get_key_str(&self.entries.borrow(), k) + match self.entries.borrow().get(k) { + Some(v) => Some(v.1.clone()), + None => None, + } } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -311,13 +244,14 @@ impl DictProtocol for PyDictRef { } // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - let key = ctx.new_str(key.to_string()); - set_item_in_content(&mut self.entries.borrow_mut(), &key, &v); + fn set_item(&self, ctx: &PyContext, key_str: &str, v: PyObjectRef) { + let key = ctx.new_str(key_str.to_string()); + let elements = &mut self.entries.borrow_mut(); + elements.insert(key_str.to_string(), (key.clone(), v.clone())); } fn del_item(&self, key: &str) { - let mut elements = get_mut_elements(self.as_object()); + let elements = &mut self.entries.borrow_mut(); elements.remove(key).unwrap(); } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c43e31a96..11a8309ab 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,4 +1,4 @@ -use super::objdict::{self, PyDictRef}; +use super::objdict::PyDictRef; use super::objlist::PyList; use super::objstr::PyStringRef; use super::objtype; @@ -233,7 +233,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in objdict::get_key_value_pairs(dict.as_object()) { + for (key, value) in dict.get_key_value_pairs() { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a8264b2de..592a4950f 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -756,6 +756,12 @@ impl IntoPyObject for String { } } +impl IntoPyObject for &str { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.to_string())) + } +} + pub fn init(ctx: &PyContext) { PyStringRef::extend_class(ctx, &ctx.str_type); } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 14958300b..16cfbaa5e 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -9,7 +9,6 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objdict; use super::objdict::PyDictRef; use super::objlist::PyList; use super::objproperty::PropertyBuilder; @@ -244,12 +243,7 @@ pub fn type_new_class( ) -> PyResult { let mut bases: Vec = bases.iter(vm)?.collect::, _>>()?; bases.push(vm.ctx.object()); - new( - typ.clone(), - &name.value, - bases, - objdict::py_dict_to_attributes(dict.as_object()), - ) + new(typ.clone(), &name.value, bases, dict.to_attributes()) } pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d6982aff6..12d6fa1ca 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -684,16 +684,6 @@ impl PyContext { .into_ref() } - // Item set/get: - pub fn set_item(&self, obj: &PyObjectRef, key: &str, v: PyObjectRef) { - if let Some(dict) = obj.payload::() { - let key = self.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - unimplemented!() - }; - } - pub fn set_attr<'a, T: Into<&'a PyObjectRef>, V: Into>( &'a self, obj: T, @@ -948,44 +938,35 @@ pub trait DictProtocol { fn del_item(&self, key: &str); } -impl DictProtocol for PyObjectRef { - fn contains_key(&self, k: &str) -> bool { - if let Some(dict) = self.payload::() { - objdict::content_contains_key_str(&dict.entries.borrow(), k) - } else { - unimplemented!() - } +pub trait ItemProtocol { + // // Move: doesn't really belong in this protocol. + // fn get_key_value_pairs(&self, vm: &VirtualMachine) -> PyResult; + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; +} + +impl ItemProtocol for PyObjectRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } - fn get_item(&self, k: &str) -> Option { - if let Some(dict) = self.payload::() { - objdict::content_get_key_str(&dict.entries.borrow(), k) - } else { - panic!("TODO {:?}", k) - } + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - if self.payload_is::() { - objdict::get_key_value_pairs(self) - } else { - panic!("TODO") - } - } - - // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - if let Some(dict) = self.payload::() { - let key = ctx.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - panic!("TODO {:?}", self); - } - } - - fn del_item(&self, key: &str) { - let mut elements = objdict::get_mut_elements(self); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index fbe7ea6b1..80f0f45b6 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -12,7 +12,8 @@ use crate::obj::{ objtype, }; use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol, + create_type, DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, + TypeProtocol, }; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -63,10 +64,10 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { let elements = objsequence::get_elements(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_elements(self.pyobject); + let pairs = objdict::get_key_value_pairs(self.pyobject); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { - map.serialize_entry(&key, &self.clone_with_object(&e.1))?; + map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; } map.end() } else if self.pyobject.is(&self.vm.get_none()) { @@ -201,9 +202,9 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { let json_decode_error = vm .get_attribute(vm.sys_module.clone(), "modules") .unwrap() - .get_item("json") + .get_item("json", vm) .unwrap() - .get_item("JSONDecodeError") + .get_item("JSONDecodeError", vm) .unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); From f840bdcd7fa8b47434e0470f617d8b488c9dccd3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:47:44 +0100 Subject: [PATCH 108/658] Generalise set_attr. --- vm/src/vm.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 2fad29124..1649d4efc 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -568,13 +568,16 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr( - &self, - obj: &PyObjectRef, - attr_name: PyObjectRef, - attr_value: PyObjectRef, - ) -> PyResult { - self.call_method(&obj, "__setattr__", vec![attr_name, attr_value]) + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: PyObjectRef) -> PyResult + where + K: TryIntoRef, + { + let attr_name = attr_name.try_into_ref(self)?; + self.call_method( + obj, + "__setattr__", + vec![attr_name.into_object(), attr_value], + ) } pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult<()> { From 25c17b482985f3641134f7a06b109845ae19ba49 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:48:23 +0100 Subject: [PATCH 109/658] Change all the ctx.set_attr in ast. --- vm/src/stdlib/ast.rs | 158 +++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 75 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 65fcdf660..f6aa93aad 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -50,7 +50,7 @@ fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { // let ast_node = ctx.new_instance(this.Module); let ast_node = create_node(vm, "program"); let py_body = vm.ctx.new_list(body); - vm.ctx.set_attr(&ast_node, "body", py_body); + vm.set_attr(&ast_node, "body", py_body).unwrap(); ast_node } @@ -80,15 +80,16 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "ClassDef"); // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); + vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) + .unwrap(); // Set body: let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); + vm.set_attr(&node, "decorator_list", py_decorator_list) + .unwrap(); node } ast::Statement::FunctionDef { @@ -101,24 +102,26 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "FunctionDef"); // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); + vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) + .unwrap(); - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); + vm.set_attr(&node, "args", parameters_to_ast(vm, args)) + .unwrap(); // Set body: let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); + vm.set_attr(&node, "decorator_list", py_decorator_list) + .unwrap(); let py_returns = if let Some(hint) = returns { expression_to_ast(vm, hint) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "returns", py_returns); + vm.set_attr(&node, "returns", py_returns).unwrap(); node } ast::Statement::Continue => create_node(vm, "Continue"), @@ -127,13 +130,14 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P ast::Statement::Assert { test, msg } => { let node = create_node(vm, "Pass"); - vm.ctx.set_attr(&node, "test", expression_to_ast(vm, test)); + vm.set_attr(&node, "test", expression_to_ast(vm, test)) + .unwrap(); let py_msg = match msg { Some(msg) => expression_to_ast(vm, msg), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "msg", py_msg); + vm.set_attr(&node, "msg", py_msg).unwrap(); node } @@ -143,7 +147,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let py_targets = vm .ctx .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); - vm.ctx.set_attr(&node, "targets", py_targets); + vm.set_attr(&node, "targets", py_targets).unwrap(); node } @@ -155,7 +159,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -163,17 +167,17 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "If"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -186,20 +190,20 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "For"); let py_target = expression_to_ast(vm, target); - vm.ctx.set_attr(&node, "target", py_target); + vm.set_attr(&node, "target", py_target).unwrap(); let py_iter = expression_to_ast(vm, iter); - vm.ctx.set_attr(&node, "iter", py_iter); + vm.set_attr(&node, "iter", py_iter).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -207,17 +211,17 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "While"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -225,7 +229,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "Expr"); let value = expression_to_ast(vm, expression); - vm.ctx.set_attr(&node, "value", value); + vm.set_attr(&node, "value", value).unwrap(); node } @@ -236,7 +240,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); - vm.ctx.set_attr(&node, "lineno", lineno); + vm.set_attr(&node, "lineno", lineno).unwrap(); node } @@ -255,10 +259,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Call"); let py_func_ast = expression_to_ast(vm, function); - vm.ctx.set_attr(&node, "func", py_func_ast); + vm.set_attr(&node, "func", py_func_ast).unwrap(); let py_args = expressions_to_ast(vm, args); - vm.ctx.set_attr(&node, "args", py_args); + vm.set_attr(&node, "args", py_args).unwrap(); node } @@ -266,7 +270,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "BinOp"); let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "left", py_a); + vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let str_op = match op { @@ -285,10 +289,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Operator::FloorDiv => "FloorDiv", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); let py_b = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "right", py_b); + vm.set_attr(&node, "right", py_b).unwrap(); node } ast::Expression::Unop { op, a } => { @@ -301,10 +305,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::UnaryOperator::Pos => "UAdd", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "operand", py_a); + vm.set_attr(&node, "operand", py_a).unwrap(); node } @@ -315,14 +319,14 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let py_a = expression_to_ast(vm, a); let py_b = expression_to_ast(vm, b); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); node } @@ -330,7 +334,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Compare"); let py_a = expression_to_ast(vm, &vals[0]); - vm.ctx.set_attr(&node, "left", py_a); + vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -351,7 +355,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .collect(), ); - vm.ctx.set_attr(&node, "ops", py_ops); + vm.set_attr(&node, "ops", py_ops).unwrap(); let py_b = vm.ctx.new_list( vals.iter() @@ -359,7 +363,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .map(|x| expression_to_ast(vm, x)) .collect(), ); - vm.ctx.set_attr(&node, "comparators", py_b); + vm.set_attr(&node, "comparators", py_b).unwrap(); node } ast::Expression::Identifier { name } => { @@ -367,16 +371,17 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj // Id: let py_name = vm.ctx.new_str(name.clone()); - vm.ctx.set_attr(&node, "id", py_name); + vm.set_attr(&node, "id", py_name).unwrap(); node } ast::Expression::Lambda { args, body } => { let node = create_node(vm, "Lambda"); - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); + vm.set_attr(&node, "args", parameters_to_ast(vm, args)) + .unwrap(); let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); node } @@ -384,13 +389,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "IfExp"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = expression_to_ast(vm, orelse); - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -404,28 +409,28 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - vm.ctx.set_attr(&node, "n", py_n); + vm.set_attr(&node, "n", py_n).unwrap(); node } ast::Expression::True => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(true)); + vm.set_attr(&node, "value", vm.ctx.new_bool(true)).unwrap(); node } ast::Expression::False => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(false)); + vm.set_attr(&node, "value", vm.ctx.new_bool(false)).unwrap(); node } ast::Expression::None => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.none()); + vm.set_attr(&node, "value", vm.ctx.none()).unwrap(); node } @@ -435,7 +440,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -444,7 +449,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -453,7 +458,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -468,10 +473,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } let py_keys = vm.ctx.new_list(keys); - vm.ctx.set_attr(&node, "keys", py_keys); + vm.set_attr(&node, "keys", py_keys).unwrap(); let py_values = vm.ctx.new_list(values); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); node } @@ -490,7 +495,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .map(|g| comprehension_to_ast(vm, g)) .collect(); let py_generators = vm.ctx.new_list(g); - vm.ctx.set_attr(&node, "generators", py_generators); + vm.set_attr(&node, "generators", py_generators).unwrap(); node } @@ -501,7 +506,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj Some(value) => expression_to_ast(vm, value), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -509,7 +514,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "YieldFrom"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -517,10 +522,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Subscript"); let py_value = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); let py_slice = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "slice", py_slice); + vm.set_attr(&node, "slice", py_slice).unwrap(); node } @@ -528,10 +533,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Attribute"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); let py_attr = vm.ctx.new_str(name.to_string()); - vm.ctx.set_attr(&node, "attr", py_attr); + vm.set_attr(&node, "attr", py_attr).unwrap(); node } @@ -539,7 +544,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Starred"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -547,21 +552,22 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Slice"); let py_value = expressions_to_ast(vm, elements); - vm.ctx.set_attr(&node, "bounds", py_value); + vm.set_attr(&node, "bounds", py_value).unwrap(); node } ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { let node = create_node(vm, "Bytes"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())); + vm.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())) + .unwrap(); node } }; // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); - vm.ctx.set_attr(&node, "lineno", lineno); + vm.set_attr(&node, "lineno", lineno).unwrap(); node } @@ -569,12 +575,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { let node = create_node(vm, "arguments"); - vm.ctx.set_attr( + vm.set_attr( &node, "args", vm.ctx .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), - ); + ) + .unwrap(); node } @@ -583,14 +590,14 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObject let node = create_node(vm, "arg"); let py_arg = vm.ctx.new_str(parameter.arg.to_string()); - vm.ctx.set_attr(&node, "arg", py_arg); + vm.set_attr(&node, "arg", py_arg).unwrap(); let py_annotation = if let Some(annotation) = ¶meter.annotation { expression_to_ast(vm, annotation) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "annotation", py_annotation); + vm.set_attr(&node, "annotation", py_annotation).unwrap(); node } @@ -599,13 +606,13 @@ fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) let node = create_node(vm, "comprehension"); let py_target = expression_to_ast(vm, &comprehension.target); - vm.ctx.set_attr(&node, "target", py_target); + vm.set_attr(&node, "target", py_target).unwrap(); let py_iter = expression_to_ast(vm, &comprehension.iter); - vm.ctx.set_attr(&node, "iter", py_iter); + vm.set_attr(&node, "iter", py_iter).unwrap(); let py_ifs = expressions_to_ast(vm, &comprehension.ifs); - vm.ctx.set_attr(&node, "ifs", py_ifs); + vm.set_attr(&node, "ifs", py_ifs).unwrap(); node } @@ -614,13 +621,14 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef match string { ast::StringGroup::Constant { value } => { let node = create_node(vm, "Str"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_str(value.clone())); + vm.set_attr(&node, "s", vm.ctx.new_str(value.clone())) + .unwrap(); node } ast::StringGroup::FormattedValue { value, .. } => { let node = create_node(vm, "FormattedValue"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } ast::StringGroup::Joined { values } => { @@ -631,7 +639,7 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef .map(|value| string_to_ast(vm, value)) .collect(), ); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); node } } From c8eda3733d211a2b77b5a40f70bc1ce8fbc61821 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:49:10 +0100 Subject: [PATCH 110/658] Eliminate ctx.set_attr. --- derive/src/pyclass.rs | 2 +- vm/src/builtins.rs | 17 ++++++++------- vm/src/exceptions.rs | 4 ++-- vm/src/frame.rs | 16 +++++++------- vm/src/import.rs | 2 +- vm/src/macros.rs | 22 +++++++++---------- vm/src/obj/objbytes.rs | 4 +--- vm/src/obj/objcode.rs | 3 +-- vm/src/obj/objdict.rs | 45 ++++++++++++++++++++++++++++----------- vm/src/obj/objellipsis.rs | 3 +-- vm/src/obj/objmemory.rs | 2 +- vm/src/obj/objobject.rs | 8 +++---- vm/src/obj/objstr.rs | 6 ++++++ vm/src/obj/objsuper.rs | 2 +- vm/src/obj/objtype.rs | 7 ++++++ vm/src/pyobject.rs | 28 ++++-------------------- vm/src/stdlib/io.rs | 16 +++++++------- vm/src/stdlib/json.rs | 8 ++++--- vm/src/stdlib/os.rs | 20 ++++++++--------- vm/src/sysmodule.rs | 11 +++++----- vm/src/vm.rs | 24 +++++++++------------ wasm/lib/src/convert.rs | 2 +- wasm/lib/src/vm_class.rs | 3 +-- 23 files changed, 129 insertions(+), 126 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index ad4ebf66e..e533e8009 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -118,7 +118,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { }| { let constructor_fn = kind.to_ctx_constructor_fn(); quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); + class.set_str_attr(#py_name, ctx.#constructor_fn(Self::#fn_name)); } }, ); diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 205d0c66e..7264c9c6e 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -669,7 +669,12 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_vars pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "__builtins__", { + #[cfg(target_arch = "wasm32")] + let open = ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = ctx.new_rustfunc(io_open); + + py_module!(ctx, "__builtins__", { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -715,6 +720,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "min" => ctx.new_rustfunc(builtin_min), "object" => ctx.object(), "oct" => ctx.new_rustfunc(builtin_oct), + "open" => open, "ord" => ctx.new_rustfunc(builtin_ord), "next" => ctx.new_rustfunc(builtin_next), "pow" => ctx.new_rustfunc(builtin_pow), @@ -759,12 +765,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), - }); - - #[cfg(not(target_arch = "wasm32"))] - ctx.set_attr(&py_mod, "open", ctx.new_rustfunc(io_open)); - - py_mod + }) } pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { @@ -801,6 +802,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu "__call__", vec![name_arg, bases, namespace.into_object()], )?; - cells.set_item(&vm.ctx, "__class__", class.clone()); + cells.set_item("__class__", class.clone(), vm); Ok(class) } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index fb757cd0f..cec4b3125 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -13,8 +13,8 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { vm.new_str("No msg".to_string()) }; let traceback = vm.ctx.new_list(Vec::new()); - vm.ctx.set_attr(&zelf, "msg", msg); - vm.ctx.set_attr(&zelf, "__traceback__", traceback); + vm.set_attr(&zelf, "msg", msg)?; + vm.set_attr(&zelf, "__traceback__", traceback)?; Ok(vm.get_none()) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 6a09f085a..1a0cc4de7 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -132,21 +132,21 @@ pub trait NameProtocol { impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item(name, vm) { return Some(value); } } - if let Some(value) = self.globals.get_item(name) { + if let Some(value) = self.globals.get_item(name, vm) { return Some(value); } vm.get_attribute(vm.builtins.clone(), name).ok() } - fn load_cell(&self, _vm: &VirtualMachine, name: &str) -> Option { + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item(name, vm) { return Some(value); } } @@ -154,7 +154,7 @@ impl NameProtocol for Scope { } fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(&vm.ctx, key, value) + self.get_locals().set_item(key, value, vm) } fn delete_name(&self, _vm: &VirtualMachine, key: &str) { @@ -392,11 +392,11 @@ impl Frame { // Take all key-value pairs from the dict: let dict_elements = objdict::get_key_value_pairs(&obj); for (key, value) in dict_elements.iter() { - map_obj.set_item(&vm.ctx, &objstr::get_value(key), value.clone()); + map_obj.set_item(key.clone(), value.clone(), vm); } } else { let key = self.pop_value(); - map_obj.set_item(&vm.ctx, &objstr::get_value(&key), obj) + map_obj.set_item(key, obj, vm) } } self.push_value(map_obj.into_object()); @@ -585,7 +585,7 @@ impl Frame { let scope = self.scope.clone(); let obj = vm.ctx.new_function(code_obj, scope, defaults); - vm.ctx.set_attr(&obj, "__annotations__", annotations); + vm.set_attr(&obj, "__annotations__", annotations)?; self.push_value(obj); Ok(None) diff --git a/vm/src/import.rs b/vm/src/import.rs index 179dd4321..28b3d02ef 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -39,7 +39,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); + attrs.set_item("__name__", vm.new_str(module.to_string()), vm); vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 47d3c531a..0c37e0dfc 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -114,15 +114,15 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => { - { - let py_mod = $ctx.new_module($module_name, $ctx.new_dict()); - $( - $ctx.set_attr(&py_mod, $name, $value); - )* - py_mod - } - } + ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let mut attributes = $crate::pyobject::PyAttributes::new(); + $( + let value: PyObjectRef = $value.into(); + attributes.insert($name.to_string(), value); + )* + let module_dict = $crate::obj::objdict::PyDictRef::from_attributes($ctx, attributes); + $ctx.new_module($module_name, module_dict) + }}; } #[macro_export] @@ -131,7 +131,7 @@ macro_rules! py_class { { let py_class = $ctx.new_class($class_name, $class_base); $( - $ctx.set_attr(&py_class, $name, $value); + py_class.set_str_attr($name, $value); )* py_class } @@ -143,7 +143,7 @@ macro_rules! extend_class { ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { let class = $class; $( - $ctx.set_attr(class, $name, $value); + class.set_str_attr($name, $value); )* } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 4119d688f..f24c9273c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -43,8 +43,6 @@ impl PyValue for PyBytes { // Fill bytes class methods: pub fn init(context: &PyContext) { - let bytes_type = context.bytes_type.as_object(); - let bytes_doc = "bytes(iterable_of_ints) -> bytes\n\ bytes(string, encoding[, errors]) -> bytes\n\ @@ -56,7 +54,7 @@ pub fn init(context: &PyContext) { - any object implementing the buffer API.\n \ - an integer"; - extend_class!(context, bytes_type, { + extend_class!(context, &context.bytes_type, { "__new__" => context.new_rustfunc(bytes_new), "__eq__" => context.new_rustfunc(PyBytesRef::eq), "__lt__" => context.new_rustfunc(PyBytesRef::lt), diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index a26645b91..60549cd3a 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -80,8 +80,7 @@ impl PyCodeRef { } pub fn init(context: &PyContext) { - let code_type = context.code_type.as_object(); - extend_class!(context, code_type, { + extend_class!(context, &context.code_type, { "__new__" => context.new_rustfunc(PyCodeRef::new), "__repr__" => context.new_rustfunc(PyCodeRef::repr), diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 15046a405..e110e71f8 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -4,7 +4,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, IntoPyObject, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -58,7 +58,7 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { for (needle, value) in get_key_value_pairs(&dict_obj) { - dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); + dict.set_item(needle, value, vm); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -77,13 +77,12 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); + dict.set_item(needle, value, vm); } } } for (needle, value) in kwargs.into_iter() { - let py_needle = vm.new_str(needle); - dict.set_item(&vm.ctx, &objstr::get_value(&py_needle), value); + dict.set_item(vm.new_str(needle), value, vm); } Ok(dict) } @@ -181,7 +180,7 @@ impl PyDictRef { } fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(&vm.ctx, &objstr::get_value(&needle), value) + self.set_item(needle, value, vm) } fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { @@ -216,6 +215,23 @@ impl PyDictRef { } } + // Used during module initialisation when vm isn't available. + pub fn from_attributes(ctx: &PyContext, attributes: PyAttributes) -> PyDictRef { + let dict = ctx.new_dict(); + for (key_str, value) in attributes.into_iter() { + dict.unsafe_str_insert(&key_str, value, ctx); + } + dict + } + + // Pub needed for some nasty edge cases. + // It will be unsafe if there are entries in the dictionary that compare equal. + pub fn unsafe_str_insert(&self, key: &str, value: PyObjectRef, ctx: &PyContext) { + self.entries + .borrow_mut() + .insert(key.to_string(), (ctx.new_str(key.to_string()), value)); + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -228,12 +244,14 @@ impl PyDictRef { } impl DictProtocol for PyDictRef { - fn contains_key(&self, k: &str) -> bool { - self.entries.borrow().get(k).is_some() + fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); + self.entries.borrow().get(key_str).is_some() } - fn get_item(&self, k: &str) -> Option { - match self.entries.borrow().get(k) { + fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { + let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); + match self.entries.borrow().get(key_str) { Some(v) => Some(v.1.clone()), None => None, } @@ -244,10 +262,11 @@ impl DictProtocol for PyDictRef { } // Item set/get: - fn set_item(&self, ctx: &PyContext, key_str: &str, v: PyObjectRef) { - let key = ctx.new_str(key_str.to_string()); + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { + let key = key.into_pyobject(vm).unwrap(); + let key_str = &objstr::get_value(&key); let elements = &mut self.entries.borrow_mut(); - elements.insert(key_str.to_string(), (key.clone(), v.clone())); + elements.insert(key_str.to_string(), (key.clone(), value)); } fn del_item(&self, key: &str) { diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 8ce843352..6591d8173 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -3,8 +3,7 @@ use crate::pyobject::{PyContext, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let ellipsis_type = context.ellipsis_type.as_object(); - extend_class!(context, ellipsis_type, { + extend_class!(context, &context.ellipsis_type, { "__new__" => context.new_rustfunc(ellipsis_new), "__repr__" => context.new_rustfunc(ellipsis_repr) }); diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 7f53fb5ae..cfe44dea3 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -20,7 +20,7 @@ pub fn new_memory_view( bytes_object: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - vm.ctx.set_attr(&cls, "obj", bytes_object.clone()); + vm.set_attr(cls.as_object(), "obj", bytes_object.clone())?; PyMemoryView { obj: bytes_object }.into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 11a8309ab..86a56de5d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -77,7 +77,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.set_item(&vm.ctx, &attr_name.value, value); + dict.set_item(attr_name, value, vm); Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -208,7 +208,7 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } - if let Some(obj_attr) = object_getattr(&obj, &name) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm) { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { vm.call_get_descriptor(attr, obj) @@ -219,9 +219,9 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } -fn object_getattr(obj: &PyObjectRef, attr_name: &str) -> Option { +fn object_getattr(obj: &PyObjectRef, attr_name: &str, vm: &VirtualMachine) -> Option { if let Some(ref dict) = obj.dict { - dict.get_item(attr_name) + dict.get_item(attr_name, vm) } else { None } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 592a4950f..62cc923b1 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -762,6 +762,12 @@ impl IntoPyObject for &str { } } +impl IntoPyObject for &String { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.clone())) + } +} + pub fn init(ctx: &PyContext) { PyStringRef::extend_class(ctx, &ctx.str_type); } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 24424d265..19c7c2ea8 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -124,7 +124,7 @@ fn super_new( } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg) { + match vm.get_locals().get_item(first_arg, vm) { Some(obj) => obj.clone(), _ => { return Err(vm diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 16cfbaa5e..b1b50dfbd 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -162,6 +162,13 @@ impl PyClassRef { Ok(()) } + // This is used for class initialisation where the vm is not yet available. + pub fn set_str_attr>(&self, attr_name: &str, value: V) { + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value.into()); + } + fn subclasses(self, _vm: &VirtualMachine) -> PyList { let mut subclasses = self.subclasses.borrow_mut(); subclasses.retain(|x| x.upgrade().is_some()); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 12d6fa1ca..0c7d4955b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -684,24 +684,6 @@ impl PyContext { .into_ref() } - pub fn set_attr<'a, T: Into<&'a PyObjectRef>, V: Into>( - &'a self, - obj: T, - attr_name: &str, - value: V, - ) { - let obj = obj.into(); - if let Some(PyClass { ref attributes, .. }) = obj.payload::() { - attributes - .borrow_mut() - .insert(attr_name.to_string(), value.into()); - } else if let Some(ref dict) = obj.dict { - dict.set_item(self, attr_name, value.into()); - } else { - unimplemented!("set_attr unimplemented for: {:?}", obj); - }; - } - pub fn unwrap_constant(&self, value: &bytecode::Constant) -> PyObjectRef { match *value { bytecode::Constant::Integer { ref value } => self.new_int(value.clone()), @@ -931,16 +913,14 @@ impl TypeProtocol for PyRef { } pub trait DictProtocol { - fn contains_key(&self, k: &str) -> bool; - fn get_item(&self, k: &str) -> Option; + fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; + fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef); + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: &str); } pub trait ItemProtocol { - // // Move: doesn't really belong in this protocol. - // fn get_key_value_pairs(&self, vm: &VirtualMachine) -> PyResult; fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( &self, @@ -1280,7 +1260,7 @@ pub trait PyClassImpl: PyClassDef { fn extend_class(ctx: &PyContext, class: &PyClassRef) { Self::impl_extend_class(ctx, class); if let Some(doc) = Self::DOC { - ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); + class.set_str_attr("__doc__", ctx.new_str(doc.into())); } } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 9daaf95d2..c6dbbc161 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -99,7 +99,7 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); - vm.ctx.set_attr(buffered, "raw", raw.clone()); + vm.set_attr(buffered, "raw", raw.clone())?; Ok(vm.get_none()) } @@ -148,10 +148,10 @@ fn file_io_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let args = vec![name.clone(), vm.ctx.new_int(os_mode)]; let file_no = os::os_open(vm, PyFuncArgs::new(args, vec![]))?; - vm.ctx.set_attr(file_io, "name", name.clone()); - vm.ctx.set_attr(file_io, "fileno", file_no); - vm.ctx.set_attr(file_io, "closefd", vm.new_bool(false)); - vm.ctx.set_attr(file_io, "closed", vm.new_bool(false)); + vm.set_attr(file_io, "name", name.clone())?; + vm.set_attr(file_io, "fileno", file_no)?; + vm.set_attr(file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(file_io, "closed", vm.new_bool(false))?; Ok(vm.get_none()) } @@ -215,7 +215,7 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }; let updated = os::raw_file_number(f.into_inner()); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; Ok(vm.get_none()) } @@ -241,7 +241,7 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(len) => { //reset raw fd on the FileIO object let updated = os::raw_file_number(handle); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; //return number of bytes written Ok(vm.ctx.new_int(len)) @@ -273,7 +273,7 @@ fn text_io_wrapper_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(text_io_wrapper, None), (buffer, None)] ); - vm.ctx.set_attr(text_io_wrapper, "buffer", buffer.clone()); + vm.set_attr(text_io_wrapper, "buffer", buffer.clone())?; Ok(vm.get_none()) } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 80f0f45b6..4695b7acb 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -176,7 +176,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { Some(PyString { ref value }) => value.clone(), _ => unimplemented!("map keys must be strings"), }; - dict.set_item(&self.vm.ctx, &key, value); + dict.set_item(&key, value, self.vm); } Ok(dict.into_object()) } @@ -208,8 +208,10 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { .unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); - vm.ctx.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())); - vm.ctx.set_attr(&exc, "colno", vm.ctx.new_int(err.column())); + vm.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())) + .unwrap(); + vm.set_attr(&exc, "colno", vm.ctx.new_int(err.column())) + .unwrap(); exc }) } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 42b4160ca..31b3ee644 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -114,24 +114,22 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "os", { + let os_name = if cfg!(windows) { + "nt".to_string() + } else { + "posix".to_string() + }; + + py_module!(ctx, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), "O_NONBLOCK" => ctx.new_int(4), "O_APPEND" => ctx.new_int(8), "O_CREAT" => ctx.new_int(512) - }); - - if cfg!(windows) { - ctx.set_attr(&py_mod, "name", ctx.new_str("nt".to_string())); - } else { - // Assume we're on a POSIX system - ctx.set_attr(&py_mod, "name", ctx.new_str("posix".to_string())); - } - - py_mod + }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a4a37b350..c803daaad 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -124,8 +124,7 @@ setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; let modules = ctx.new_dict(); - let sys_name = "sys"; - let sys_mod = py_module!(ctx, sys_name, { + let sys_mod = py_module!(ctx, "sys", { "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), @@ -135,11 +134,11 @@ settrace() -- set the global debug tracing function "ps2" => ctx.new_str("..... ".to_string()), "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), + "modules" => modules.clone(), }); - modules.set_item(&ctx, sys_name, sys_mod.clone()); - modules.set_item(&ctx, "builtins", builtins); - ctx.set_attr(&sys_mod, "modules", modules); + modules.unsafe_str_insert("sys", sys_mod.clone(), &ctx); + modules.unsafe_str_insert("builtins", builtins, &ctx); sys_mod } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 1649d4efc..6c35983e7 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -419,7 +419,7 @@ impl VirtualMachine { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &args.args[i]; - locals.set_item(&self.ctx, arg_name, arg.clone()); + locals.set_item(arg_name, arg.clone(), self); } // Pack other positional arguments in to *args: @@ -432,7 +432,7 @@ impl VirtualMachine { } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(&self.ctx, vararg_name, vararg_value); + locals.set_item(vararg_name, vararg_value, self); } bytecode::Varargs::Unnamed => { // just ignore the rest of the args @@ -452,7 +452,7 @@ impl VirtualMachine { let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { let d = self.ctx.new_dict(); - locals.set_item(&self.ctx, kwargs_name, d.as_object().clone()); + locals.set_item(kwargs_name, d.as_object().clone(), self); Some(d) } bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), @@ -464,15 +464,15 @@ impl VirtualMachine { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { - if locals.contains_key(&name) { + if locals.contains_key(&name, self) { return Err( self.new_type_error(format!("Got multiple values for argument '{}'", name)) ); } - locals.set_item(&self.ctx, &name, value); + locals.set_item(&name, value, self); } else if let Some(d) = &kwargs { - d.set_item(&self.ctx, &name, value); + d.set_item(&name, value, self); } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -499,7 +499,7 @@ impl VirtualMachine { let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; - if !locals.contains_key(variable_name) { + if !locals.contains_key(variable_name, self) { missing.push(variable_name) } } @@ -515,12 +515,8 @@ impl VirtualMachine { // the default if we don't already have a value for (default_index, i) in (required_args..nexpected_args).enumerate() { let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name) { - locals.set_item( - &self.ctx, - arg_name, - available_defaults[default_index].clone(), - ); + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, available_defaults[default_index].clone(), self); } } }; @@ -528,7 +524,7 @@ impl VirtualMachine { // Check if kw only arguments are all present: let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { - if !locals.contains_key(arg_name) { + if !locals.contains_key(arg_name, self) { if kwdefs.contains_key(arg_name) { // If not yet specified, take the default value unimplemented!(); diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index af6ad029e..717c9a973 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -192,7 +192,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&vm.ctx, &String::from(js_sys::JsString::from(key)), py_val); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm); } dict.into_object() } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c7e637a9f..6e4ceaee2 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -243,8 +243,7 @@ impl WASMVirtualMachine { } else { return Err(error()); }; - let rustfunc = vm.ctx.new_rustfunc(print_fn); - vm.ctx.set_attr(&vm.builtins, "print", rustfunc); + vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn))?; Ok(()) })? } From 7438b0685c6d923921f0a615021898cae6693d62 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 15:03:56 +0100 Subject: [PATCH 111/658] Changes suggested by code review. --- vm/src/obj/objdict.rs | 5 +---- vm/src/pyobject.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e110e71f8..6062e07e6 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -251,10 +251,7 @@ impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - match self.entries.borrow().get(key_str) { - Some(v) => Some(v.1.clone()), - None => None, - } + self.entries.borrow().get(key_str).map(|v| v.1.clone()) } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 0c7d4955b..139b2a1e1 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -921,18 +921,18 @@ pub trait DictProtocol { } pub trait ItemProtocol { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( &self, key: T, value: PyObjectRef, vm: &VirtualMachine, - ) -> PyResult; - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; } impl ItemProtocol for PyObjectRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } @@ -941,11 +941,11 @@ impl ItemProtocol for PyObjectRef { key: T, value: PyObjectRef, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } From 9ed051e3b7d7cbe7140d628f06cae4c80c90dafc Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 17:56:39 +0100 Subject: [PATCH 112/658] Module Initialisation takes VirtualMachine rather than PyContext. --- vm/src/builtins.rs | 18 ++++++++++-------- vm/src/dictdatatype.rs | 15 +++++++++------ vm/src/import.rs | 2 +- vm/src/macros.rs | 19 +++++++++++++------ vm/src/obj/objdict.rs | 17 ----------------- vm/src/stdlib/ast.rs | 8 +++++--- vm/src/stdlib/dis.rs | 8 +++++--- vm/src/stdlib/io.rs | 10 +++++----- vm/src/stdlib/json.rs | 9 +++++---- vm/src/stdlib/keyword.rs | 8 +++++--- vm/src/stdlib/math.rs | 8 +++++--- vm/src/stdlib/mod.rs | 6 ++++-- vm/src/stdlib/os.rs | 8 +++++--- vm/src/stdlib/platform.rs | 7 ++++--- vm/src/stdlib/pystruct.rs | 8 +++++--- vm/src/stdlib/random.rs | 8 +++++--- vm/src/stdlib/re.rs | 8 +++++--- vm/src/stdlib/socket.rs | 10 +++++----- vm/src/stdlib/string.rs | 9 ++++++--- vm/src/stdlib/time_module.rs | 8 +++++--- vm/src/stdlib/tokenize.rs | 8 +++++--- vm/src/stdlib/types.rs | 8 +++++--- vm/src/stdlib/weakref.rs | 9 ++++++--- vm/src/sysmodule.rs | 14 +++++++------- vm/src/vm.rs | 21 +++++++++++++-------- wasm/lib/src/browser_module.rs | 10 +++++----- wasm/lib/src/vm_class.rs | 10 +++++----- 27 files changed, 153 insertions(+), 121 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 7264c9c6e..5aa5ee226 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -20,7 +20,7 @@ use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + DictProtocol, IdProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -668,13 +668,15 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_vars -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - #[cfg(target_arch = "wasm32")] - let open = ctx.none(); - #[cfg(not(target_arch = "wasm32"))] - let open = ctx.new_rustfunc(io_open); +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { + let ctx = &vm.ctx; - py_module!(ctx, "__builtins__", { + #[cfg(target_arch = "wasm32")] + let open = vm.ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = vm.ctx.new_rustfunc(io_open); + + extend_module!(vm, module, { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -765,7 +767,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), - }) + }); } pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 436923ea9..92a93a4e9 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,6 +9,7 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; +#[derive(Default)] pub struct Dict { size: usize, indices: HashMap, @@ -88,6 +89,12 @@ impl Dict { } } + pub fn clear(&mut self) { + self.entries.clear(); + self.indices.clear(); + self.size = 0 + } + /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { @@ -175,12 +182,8 @@ fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { } /// Invoke __eq__ on two keys -fn do_eq( - vm: &VirtualMachine, - key1: &PyObjectRef, - key2: &PyObjectRef, -) -> Result { - let result = vm._eq(key1, key2.clone())?; +fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { + let result = vm._eq(key1.clone(), key2.clone())?; Ok(objbool::get_value(&result)) } diff --git a/vm/src/import.rs b/vm/src/import.rs index 28b3d02ef..713393deb 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -15,7 +15,7 @@ use crate::vm::VirtualMachine; fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &str) -> PyResult { // Check for Rust-native modules if let Some(module) = vm.stdlib_inits.borrow().get(module) { - return Ok(module(&vm.ctx).clone()); + return Ok(module(vm).clone()); } let notfound_error = vm.context().exceptions.module_not_found_error.clone(); diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 0c37e0dfc..27d164718 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -114,17 +114,24 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ - let mut attributes = $crate::pyobject::PyAttributes::new(); + ( $vm:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let module = $vm.ctx.new_module($module_name, $vm.ctx.new_dict()); $( - let value: PyObjectRef = $value.into(); - attributes.insert($name.to_string(), value); + $vm.set_attr(&module, $name, $value).unwrap(); )* - let module_dict = $crate::obj::objdict::PyDictRef::from_attributes($ctx, attributes); - $ctx.new_module($module_name, module_dict) + module }}; } +#[macro_export] +macro_rules! extend_module { + ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)* }) => { + $( + $vm.set_attr(&$module, $name, $value).unwrap(); + )* + } +} + #[macro_export] macro_rules! py_class { ( $ctx:expr, $class_name:expr, $class_base:expr, { $($name:expr => $value:expr),* $(,)* }) => { diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6062e07e6..6ebe5aa2e 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -215,23 +215,6 @@ impl PyDictRef { } } - // Used during module initialisation when vm isn't available. - pub fn from_attributes(ctx: &PyContext, attributes: PyAttributes) -> PyDictRef { - let dict = ctx.new_dict(); - for (key_str, value) in attributes.into_iter() { - dict.unsafe_str_insert(&key_str, value, ctx); - } - dict - } - - // Pub needed for some nasty edge cases. - // It will be unsafe if there are entries in the dictionary that compare equal. - pub fn unsafe_str_insert(&self, key: &str, value: PyObjectRef, ctx: &PyContext) { - self.entries - .borrow_mut() - .insert(key.to_string(), (ctx.new_str(key.to_string()), value)); - } - /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index f6aa93aad..e7ac5308c 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -12,7 +12,7 @@ use rustpython_parser::{ast, parser}; use crate::function::PyFuncArgs; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -656,8 +656,10 @@ fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(ast_node) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "ast", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index 222f23383..4c35f45e9 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -1,5 +1,5 @@ use crate::obj::objcode::PyCodeRef; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TryFromObject}; +use crate::pyobject::{PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -17,8 +17,10 @@ fn dis_disassemble(co: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "dis", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "dis", { "dis" => ctx.new_rustfunc(dis_dis), "disassemble" => ctx.new_rustfunc(dis_disassemble) }) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index c6dbbc161..bd33eac75 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,9 +20,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{ - BufferProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; fn compute_c_flag(mode: &str) -> u16 { @@ -365,7 +363,9 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), @@ -421,7 +421,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "getvalue" => ctx.new_rustfunc(bytes_io_getvalue) }); - py_module!(ctx, "io", { + py_module!(vm, "io", { "open" => ctx.new_rustfunc(io_open), "IOBase" => io_base, "RawIOBase" => raw_io_base, diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 4695b7acb..631f8257f 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -12,8 +12,7 @@ use crate::obj::{ objtype, }; use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, - TypeProtocol, + create_type, DictProtocol, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol, }; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -232,7 +231,9 @@ fn json_loads(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { de_pyobject(vm, &objstr::get_value(&string)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + // TODO: Make this a proper type with a constructor let json_decode_error = create_type( "JSONDecodeError", @@ -240,7 +241,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { &ctx.exceptions.exception_type, ); - py_module!(ctx, "json", { + py_module!(vm, "json", { "dumps" => ctx.new_rustfunc(json_dumps), "loads" => ctx.new_rustfunc(json_loads), "JSONDecodeError" => json_decode_error diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index fe27399c5..3143af21a 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,7 +18,9 @@ fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let keyword_kwlist = ctx.new_list( lexer::get_keywords() .keys() @@ -26,7 +28,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { .collect(), ); - py_module!(ctx, "keyword", { + py_module!(vm, "keyword", { "iskeyword" => ctx.new_rustfunc(keyword_iskeyword), "kwlist" => keyword_kwlist }) diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 5af599cb7..dc7981ab6 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; // Helper macro: @@ -172,8 +172,10 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "math", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "math", { // Number theory functions: "fabs" => ctx.new_rustfunc(math_fabs), "isfinite" => ctx.new_rustfunc(math_isfinite), diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index b0b61119c..a69a80d11 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -15,14 +15,16 @@ mod types; mod weakref; use std::collections::HashMap; +use crate::vm::VirtualMachine; + #[cfg(not(target_arch = "wasm32"))] pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; -pub type StdlibInitFunc = Box PyObjectRef>; +pub type StdlibInitFunc = Box PyObjectRef>; pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 31b3ee644..3f0f557f1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,7 +7,7 @@ use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::objint; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -113,14 +113,16 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let os_name = if cfg!(windows) { "nt".to_string() } else { "posix".to_string() }; - py_module!(ctx, "os", { + py_module!(vm, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), diff --git a/vm/src/stdlib/platform.rs b/vm/src/stdlib/platform.rs index a87383a72..6d4751a35 100644 --- a/vm/src/stdlib/platform.rs +++ b/vm/src/stdlib/platform.rs @@ -1,9 +1,10 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "platform", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "platform", { "python_compiler" => ctx.new_rustfunc(platform_python_compiler), "python_implementation" => ctx.new_rustfunc(platform_python_implementation), "python_version" => ctx.new_rustfunc(platform_python_version), diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 9dadc520a..2ff61d630 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -15,7 +15,7 @@ use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::VirtualMachine; #[derive(Debug)] @@ -297,8 +297,10 @@ fn struct_unpack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_tuple(items)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "struct", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "struct", { "pack" => ctx.new_rustfunc(struct_pack), "unpack" => ctx.new_rustfunc(struct_unpack) }) diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index 82c1f410c..c6cc44a36 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,11 +4,13 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "random", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "random", { "guass" => ctx.new_rustfunc(random_gauss), "normalvariate" => ctx.new_rustfunc(random_normalvariate), "random" => ctx.new_rustfunc(random_random), diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index fee100bfe..b9f921502 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -8,7 +8,7 @@ use regex::{Match, Regex}; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; impl PyValue for Regex { @@ -98,7 +98,9 @@ impl PyMatchRef { } /// Create the python `re` module with all its members. -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let match_type = py_class!(ctx, "Match", ctx.object(), { "start" => ctx.new_rustfunc(PyMatchRef::start), "end" => ctx.new_rustfunc(PyMatchRef::end) @@ -109,7 +111,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "search" => ctx.new_rustfunc(PyRegexRef::search) }); - py_module!(ctx, "re", { + py_module!(vm, "re", { "compile" => ctx.new_rustfunc(re_compile), "Match" => match_type, "match" => ctx.new_rustfunc(re_match), diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index fe0ce29a0..ab5072f7c 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,9 +10,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objsequence::get_elements; use crate::obj::objstr; -use crate::pyobject::{ - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use crate::obj::objtype::PyClassRef; @@ -411,7 +409,9 @@ fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { Ok(vm.ctx.new_tuple(vec![ip, port])) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let socket = py_class!(ctx, "socket", ctx.object(), { "__new__" => ctx.new_rustfunc(socket_new), "connect" => ctx.new_rustfunc(socket_connect), @@ -426,7 +426,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "recvfrom" => ctx.new_rustfunc(socket_recvfrom), }); - py_module!(ctx, "socket", { + py_module!(vm, "socket", { "AF_INET" => ctx.new_int(AddressFamily::Inet as i32), "SOCK_STREAM" => ctx.new_int(SocketKind::Stream as i32), "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 157dae035..c255ffc93 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -3,9 +3,12 @@ * */ -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { let ascii_lowercase = "abcdefghijklmnopqrstuvwxyz".to_string(); let ascii_uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_string(); let ascii_letters = format!("{}{}", ascii_lowercase, ascii_uppercase); @@ -19,7 +22,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { */ // Constants: - py_module!(ctx, "string", { + py_module!(vm, "string", { "ascii_letters" => ctx.new_str(ascii_letters), "ascii_lowercase" => ctx.new_str(ascii_lowercase), "ascii_uppercase" => ctx.new_str(ascii_uppercase), diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index dfe3ab8e3..93beabf8a 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -32,8 +32,10 @@ fn time_time(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "time", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "time", { "sleep" => ctx.new_rustfunc(time_sleep), "time" => ctx.new_rustfunc(time_time) }) diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 0aa5df3d2..88a26881c 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -25,8 +25,10 @@ fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: create main function when called with -m -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "tokenize", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "tokenize", { "tokenize" => ctx.new_rustfunc(tokenize_tokenize) }) } diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs index 8a5ab2939..ad76a80a4 100644 --- a/vm/src/stdlib/types.rs +++ b/vm/src/stdlib/types.rs @@ -7,7 +7,7 @@ use crate::obj::objdict::PyDict; use crate::obj::objstr::PyStringRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; +use crate::pyobject::{PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; use crate::VirtualMachine; fn types_new_class( @@ -25,8 +25,10 @@ fn types_new_class( objtype::type_new_class(vm, vm.ctx.type_type(), name, bases, dict) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "types", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "types", { "new_class" => ctx.new_rustfunc(types_new_class), "FunctionType" => ctx.function_type(), "LambdaType" => ctx.function_type(), diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 5616c3ddf..b5f7b756a 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -5,10 +5,13 @@ //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -use super::super::pyobject::{PyContext, PyObjectRef}; +use super::super::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "_weakref", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "_weakref", { "ref" => ctx.weakref_type() }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index c803daaad..772dfe198 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -39,7 +39,9 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } -pub fn make_module(ctx: &PyContext, builtins: PyObjectRef) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { + let ctx = &vm.ctx; + let path_list = match env::var_os("PYTHONPATH") { Some(paths) => env::split_paths(&paths) .map(|path| { @@ -124,7 +126,7 @@ setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; let modules = ctx.new_dict(); - let sys_mod = py_module!(ctx, "sys", { + extend_module!(vm, module, { "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), @@ -137,8 +139,6 @@ settrace() -- set the global debug tracing function "modules" => modules.clone(), }); - modules.unsafe_str_insert("sys", sys_mod.clone(), &ctx); - modules.unsafe_str_insert("builtins", builtins, &ctx); - - sys_mod + modules.set_item("sys", module.clone(), vm); + modules.set_item("builtins", builtins.clone(), vm); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 6c35983e7..0bd255574 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -58,18 +58,22 @@ impl VirtualMachine { let ctx = PyContext::new(); // Hard-core modules: - let builtins = builtins::make_module(&ctx); - let sysmod = sysmodule::make_module(&ctx, builtins.clone()); + let builtins = ctx.new_module("builtins", ctx.new_dict()); + let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); - VirtualMachine { - builtins, - sys_module: sysmod, + let vm = VirtualMachine { + builtins: builtins.clone(), + sys_module: sysmod.clone(), stdlib_inits, ctx, frames: RefCell::new(vec![]), wasm_id: None, - } + }; + + builtins::make_module(&vm, builtins.clone()); + sysmodule::make_module(&vm, sysmod, builtins); + vm } pub fn run_code_obj(&self, code: PyCodeRef, scope: Scope) -> PyResult { @@ -564,15 +568,16 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: PyObjectRef) -> PyResult + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: V) -> PyResult where K: TryIntoRef, + V: Into, { let attr_name = attr_name.try_into_ref(self)?; self.call_method( obj, "__setattr__", - vec![attr_name.into_object(), attr_value], + vec![attr_name.into_object(), attr_value.into()], ) } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 0e13aeb3a..cb00d175c 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -12,9 +12,7 @@ use rustpython_vm::obj::{ objstr::{self, PyStringRef}, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{ - PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use rustpython_vm::VirtualMachine; use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; @@ -348,7 +346,9 @@ fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(result) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let promise = py_class!(ctx, "Promise", ctx.object(), { "then" => ctx.new_rustfunc(PyPromise::then), "catch" => ctx.new_rustfunc(PyPromise::catch), @@ -371,7 +371,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "set_attr" => ctx.new_rustfunc(Element::set_attr), }); - py_module!(ctx, "browser", { + py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), "request_animation_frame" => ctx.new_rustfunc(browser_request_animation_frame), "cancel_animation_frame" => ctx.new_rustfunc(browser_cancel_animation_frame), diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 6e4ceaee2..dbf148f2e 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -8,7 +8,7 @@ use wasm_bindgen::prelude::*; use rustpython_vm::compile; use rustpython_vm::frame::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyResult}; +use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult}; use rustpython_vm::VirtualMachine; use crate::browser_module::setup_browser_module; @@ -260,12 +260,12 @@ impl WASMVirtualMachine { let mod_name = name.clone(); - let stdlib_init_fn = move |ctx: &PyContext| { - let py_mod = ctx.new_module(&name, ctx.new_dict()); + let stdlib_init_fn = move |vm: &VirtualMachine| { + let module = vm.ctx.new_module(&name, vm.ctx.new_dict()); for (key, value) in module_items.clone() { - ctx.set_attr(&py_mod, &key, value); + vm.set_attr(&module, key, value).unwrap(); } - py_mod + module }; vm.stdlib_inits From b943f4a4bb465d88af212461dfc7406259247b83 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 17:58:14 +0100 Subject: [PATCH 113/658] Historic real proper dictionary support. --- tests/snippets/dict.py | 4 ++ vm/src/frame.rs | 4 +- vm/src/lib.rs | 1 + vm/src/obj/objdict.rs | 90 ++++++++++++++++------------------------ vm/src/obj/objobject.rs | 2 +- vm/src/pyobject.rs | 2 +- wasm/lib/src/vm_class.rs | 3 +- 7 files changed, 47 insertions(+), 59 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 170a6492d..a80b5b9e2 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -34,3 +34,7 @@ res = set() for key in a.keys(): res.add(key) assert res == set(['a','b']) + +x = {} +x[1] = 1 +assert x[1] == 1 diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 1a0cc4de7..0b735df87 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -157,8 +157,8 @@ impl NameProtocol for Scope { self.get_locals().set_item(key, value, vm) } - fn delete_name(&self, _vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key) + fn delete_name(&self, vm: &VirtualMachine, key: &str) { + self.get_locals().del_item(key, vm) } } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 4a300dcef..89b4dd872 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -44,6 +44,7 @@ pub mod macros; mod builtins; pub mod bytecode; pub mod compile; +mod dictdatatype; pub mod error; pub mod eval; mod exceptions; diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6ebe5aa2e..22e4fb518 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,20 +1,22 @@ use std::cell::{Cell, RefCell}; -use std::collections::HashMap; use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, IntoPyObject, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, + PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; +use crate::dictdatatype; + use super::objiter; use super::objlist::PyListIterator; -use super::objstr::{self, PyStringRef}; +use super::objstr; use super::objtype; use crate::obj::objtype::PyClassRef; -pub type DictContentType = HashMap; +pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { @@ -37,13 +39,11 @@ impl PyValue for PyDict { } pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = dict.payload::().unwrap().entries.borrow(); - let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_elements.iter() { - let (key, obj) = pair; - pairs.push((key.clone(), obj.clone())); - } - pairs + dict.payload::() + .unwrap() + .entries + .borrow() + .get_items() } // Python dict methods: @@ -112,18 +112,12 @@ impl PyDictRef { Ok(vm.new_str(s)) } - fn contains(self, key: PyStringRef, _vm: &VirtualMachine) -> bool { - self.entries.borrow().contains_key(&key.value) + fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let key = &key.value; - // Delete the item: - let mut elements = self.entries.borrow_mut(); - match elements.remove(key) { - Some(_) => Ok(()), - None => Err(vm.new_value_error(format!("Key not found: {}", key))), - } + fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().delete(vm, &key) } fn clear(self, _vm: &VirtualMachine) { @@ -136,7 +130,8 @@ impl PyDictRef { let keys = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(k, _v)| k.clone()) .collect(); let key_list = vm.ctx.new_list(keys); @@ -152,7 +147,8 @@ impl PyDictRef { let values = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(_k, v)| v.clone()) .collect(); let values_list = vm.ctx.new_list(values); @@ -168,7 +164,8 @@ impl PyDictRef { let items = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) .collect(); let items_list = vm.ctx.new_list(items); @@ -183,35 +180,22 @@ impl PyDictRef { self.set_item(needle, value, vm) } - fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { - let key = &key.value; - - // What we are looking for: - let elements = self.entries.borrow(); - if elements.contains_key(key) { - Ok(elements[key].1.clone()) - } else { - Err(vm.new_value_error(format!("Key not found: {}", key))) - } + fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().get(vm, &key) } fn get( self, - key: PyStringRef, + key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - // What we are looking for: - let key = &key.value; - - let elements = self.entries.borrow(); - if elements.contains_key(key) { - elements[key].1.clone() - } else { - match default { + match self.into_object().get_item(key, vm) { + Ok(value) => value, + Err(_) => match default { OptionalArg::Present(value) => value, OptionalArg::Missing => vm.ctx.none(), - } + }, } } @@ -228,13 +212,13 @@ impl PyDictRef { impl DictProtocol for PyDictRef { fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { - let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - self.entries.borrow().get(key_str).is_some() + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().contains(vm, &key).unwrap() } fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { - let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - self.entries.borrow().get(key_str).map(|v| v.1.clone()) + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().get(vm, &key).ok() } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -244,14 +228,12 @@ impl DictProtocol for PyDictRef { // Item set/get: fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { let key = key.into_pyobject(vm).unwrap(); - let key_str = &objstr::get_value(&key); - let elements = &mut self.entries.borrow_mut(); - elements.insert(key_str.to_string(), (key.clone(), value)); + self.entries.borrow_mut().insert(vm, &key, value).unwrap() } - fn del_item(&self, key: &str) { - let elements = &mut self.entries.borrow_mut(); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) { + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow_mut().delete(vm, &key).unwrap(); } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 86a56de5d..1336e5246 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -98,7 +98,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.del_item(&attr_name.value); + dict.del_item(attr_name, vm); Ok(()) } else { Err(vm.new_attribute_error(format!( diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 139b2a1e1..de4d29628 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -917,7 +917,7 @@ pub trait DictProtocol { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); - fn del_item(&self, key: &str); + fn del_item(&self, key: T, vm: &VirtualMachine); } pub trait ItemProtocol { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index dbf148f2e..d6f9ede0e 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -243,7 +243,8 @@ impl WASMVirtualMachine { } else { return Err(error()); }; - vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn))?; + vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn)) + .unwrap(); Ok(()) })? } From 43347e2d8bb67395aacfde800dd68ddc9a6d2f7a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 18:00:14 +0100 Subject: [PATCH 114/658] Dictionary: KeyError -> ValueError. --- vm/src/dictdatatype.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 92a93a4e9..ac1df994a 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -85,7 +85,7 @@ impl Dict { } } else { let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } @@ -103,7 +103,7 @@ impl Dict { Ok(()) } else { let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } From 77d5f57df1dd1f63e5393d279c711d9eac8605ce Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 14:36:19 +0100 Subject: [PATCH 115/658] Add some additional dictionary tests. --- tests/snippets/dict.py | 50 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index a80b5b9e2..5c5ef62b1 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + def dict_eq(d1, d2): return (all(k in d2 and d1[k] == d2[k] for k in d1) and all(k in d1 and d1[k] == d2[k] for k in d2)) @@ -21,20 +23,60 @@ assert len(a) == 0 a = {'a': 5, 'b': 6} res = set() for value in a.values(): - res.add(value) + res.add(value) assert res == set([5,6]) count = 0 for (key, value) in a.items(): - assert a[key] == value - count += 1 + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) + res.add(key) assert res == set(['a','b']) x = {} x[1] = 1 assert x[1] == 1 + + +x[7] = 7 +x[2] = 2 +x[(5, 6)] = 5 + +with assertRaises(KeyError): + x["not here"] + +with assertRaises(TypeError): + x[[]] # Unhashable type. + +x["here"] = "here" +assert x.get("not here", "default") == "default" +assert x.get("here", "default") == "here" +assert x.get("not here") == None + +# An object that hashes to the same value always, and compares equal if any its values match. +class Hashable(object): + def __init__(self, *args): + self.values = args + def __hash__(self): + return 1 + def __eq__(self, other): + for x in self.values: + for y in other.values: + if x == y: + return True + return False + +x = {} +x[Hashable(1,2)] = 8 + +assert x[Hashable(1,2)] == 8 +assert x[Hashable(3,1)] == 8 + +x[Hashable(8)] = 19 +x[Hashable(19,8)] = 1 +assert x[Hashable(8)] == 1 +assert len(x) == 2 From ec94168a15dedc80297659934623a1730a99ce6a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:00:10 +0100 Subject: [PATCH 116/658] Remove objdict::get_key_value_pairs. --- vm/src/frame.rs | 11 +++++++---- vm/src/obj/objdict.rs | 25 ++++++++----------------- vm/src/obj/objmodule.rs | 2 +- vm/src/pyobject.rs | 1 - vm/src/stdlib/json.rs | 7 +++++-- wasm/lib/src/browser_module.rs | 4 +++- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 0b735df87..f6fc8e96e 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,7 +12,7 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict::{self, PyDictRef}; +use crate::obj::objdict::PyDictRef; use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; @@ -390,7 +390,9 @@ impl Frame { let obj = self.pop_value(); if *unpack { // Take all key-value pairs from the dict: - let dict_elements = objdict::get_key_value_pairs(&obj); + let dict: PyDictRef = + obj.downcast().expect("Need a dictionary to build a map."); + let dict_elements = dict.get_key_value_pairs(); for (key, value) in dict_elements.iter() { map_obj.set_item(key.clone(), value.clone(), vm); } @@ -612,8 +614,9 @@ impl Frame { } bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { - let kw_dict = self.pop_value(); - let dict_elements = objdict::get_key_value_pairs(&kw_dict).clone(); + let kw_dict: PyDictRef = + self.pop_value().downcast().expect("Kwargs must be a dict."); + let dict_elements = kw_dict.get_key_value_pairs(); dict_elements .into_iter() .map(|elem| (objstr::get_value(&elem.0), elem.1)) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4ac1431b3..b2bf45304 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -13,7 +13,6 @@ use crate::dictdatatype; use super::objiter; use super::objlist::PyListIterator; use super::objstr; -use super::objtype; use crate::obj::objtype::PyClassRef; pub type DictContentType = dictdatatype::Dict; @@ -38,14 +37,6 @@ impl PyValue for PyDict { } } -pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - dict.payload::() - .unwrap() - .entries - .borrow() - .get_items() -} - // Python dict methods: impl PyDictRef { fn new( @@ -56,8 +47,9 @@ impl PyDictRef { ) -> PyResult { let dict = vm.ctx.new_dict(); if let OptionalArg::Present(dict_obj) = dict_obj { - if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { - for (needle, value) in get_key_value_pairs(&dict_obj) { + let dicted: PyResult = dict_obj.clone().downcast(); + if let Ok(dict_obj) = dicted { + for (needle, value) in dict_obj.get_key_value_pairs() { dict.set_item(needle, value, vm); } } else { @@ -97,9 +89,8 @@ impl PyDictRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { - let elements = get_key_value_pairs(self.as_object()); let mut str_parts = vec![]; - for (key, value) in elements { + for (key, value) in self.get_key_value_pairs() { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -176,6 +167,10 @@ impl PyDictRef { } } + pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + self.entries.borrow().get_items() + } + fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { self.set_item(needle, value, vm) } @@ -225,10 +220,6 @@ impl DictProtocol for PyDictRef { self.entries.borrow().get(vm, &key).ok() } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - get_key_value_pairs(self.as_object()) - } - // Item set/get: fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { let key = key.into_pyobject(vm).unwrap(); diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index b641f0426..75749d557 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -1,5 +1,5 @@ use crate::obj::objtype::PyClassRef; -use crate::pyobject::{DictProtocol, PyContext, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index de4d29628..4df7e57fe 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -915,7 +915,6 @@ impl TypeProtocol for PyRef { pub trait DictProtocol { fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: T, vm: &VirtualMachine); } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 631f8257f..b7cb704b1 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -7,7 +7,9 @@ use serde_json; use crate::function::PyFuncArgs; use crate::obj::{ - objbool, objdict, objfloat, objint, objsequence, + objbool, + objdict::PyDictRef, + objfloat, objint, objsequence, objstr::{self, PyString}, objtype, }; @@ -63,7 +65,8 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { let elements = objsequence::get_elements(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_key_value_pairs(self.pyobject); + let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); + let pairs = dict.get_key_value_pairs(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index cb00d175c..6f7c31964 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,6 +7,7 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ + objdict::PyDictRef, objfunction::PyFunction, objint, objstr::{self, PyStringRef}, @@ -72,7 +73,8 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(headers) = headers { let h = request.headers(); - for (key, value) in rustpython_vm::obj::objdict::get_key_value_pairs(&headers) { + let headers: PyDictRef = headers.downcast().unwrap(); + for (key, value) in headers.get_key_value_pairs() { let key = &vm.to_str(&key)?.value; let value = &vm.to_str(&value)?.value; h.set(key, value) From 4212594c5b108afcd076fa56930ec861c559bf8a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:01:44 +0100 Subject: [PATCH 117/658] Prefer key to needle. --- vm/src/obj/objdict.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index b2bf45304..449c60726 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -49,8 +49,8 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - for (needle, value) in dict_obj.get_key_value_pairs() { - dict.set_item(needle, value, vm); + for (key, value) in dict_obj.get_key_value_pairs() { + dict.set_item(key, value, vm); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -63,18 +63,17 @@ impl PyDictRef { None => break, }; let elem_iter = objiter::get_iter(vm, &element)?; - let needle = - objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + let key = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(needle, value, vm); + dict.set_item(key, value, vm); } } } - for (needle, value) in kwargs.into_iter() { - dict.set_item(vm.new_str(needle), value, vm); + for (key, value) in kwargs.into_iter() { + dict.set_item(vm.new_str(key), value, vm); } Ok(dict) } @@ -171,8 +170,8 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(needle, value, vm) + fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { + self.set_item(key, value, vm) } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { From 0dce9bb96aac9e446f0b87111e8b27c0135f2489 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:10:52 +0100 Subject: [PATCH 118/658] Use item protocol when executing instructions. --- vm/src/frame.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index f6fc8e96e..858b33a08 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -21,8 +21,8 @@ use crate::obj::objstr; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, + DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -990,22 +990,18 @@ impl Frame { } } - fn subscript(&self, vm: &VirtualMachine, a: PyObjectRef, b: PyObjectRef) -> PyResult { - vm.call_method(&a, "__getitem__", vec![b]) - } - fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); let value = self.pop_value(); - vm.call_method(&obj, "__setitem__", vec![idx, value])?; + obj.set_item(idx, value, vm)?; Ok(None) } fn execute_delete_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); - vm.call_method(&obj, "__delitem__", vec![idx])?; + obj.del_item(idx, vm)?; Ok(None) } @@ -1040,7 +1036,7 @@ impl Frame { bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), // TODO: Subscript should probably have its own op bytecode::BinaryOperator::Subscript if inplace => unreachable!(), - bytecode::BinaryOperator::Subscript => self.subscript(vm, a_ref, b_ref), + bytecode::BinaryOperator::Subscript => a_ref.get_item(b_ref, vm), bytecode::BinaryOperator::Modulo if inplace => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::Modulo => vm._mod(a_ref, b_ref), bytecode::BinaryOperator::Lshift if inplace => vm._ilshift(a_ref, b_ref), From f354f4ce028cd86b63f33641dc0aa4c53d253274 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 16:14:02 +0100 Subject: [PATCH 119/658] Move contains_key to PyDictRef. --- vm/src/obj/objdict.rs | 6 +++--- vm/src/pyobject.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 449c60726..541645d5f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -206,14 +206,14 @@ impl PyDictRef { fn hash(self, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } -} -impl DictProtocol for PyDictRef { - fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + pub fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().contains(vm, &key).unwrap() } +} +impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().get(vm, &key).ok() diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4df7e57fe..bf5cf3c86 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -913,7 +913,6 @@ impl TypeProtocol for PyRef { } pub trait DictProtocol { - fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: T, vm: &VirtualMachine); From 37e7972dcd438dfd0e0305236e4a0bbbc66a99eb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 17:51:03 +0300 Subject: [PATCH 120/658] Add os.read --- tests/snippets/stdlib_os.py | 9 ++++++++- vm/src/stdlib/os.rs | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ed9e91c70..d70050495 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -2,8 +2,15 @@ import os from testutils import assert_raises -assert os.open('README.md', 0) > 0 +fd = os.open('README.md', 0) +assert fd > 0 +assert len(os.read(fd, 10)) == 10 +assert len(os.read(fd, 5)) == 5 + +assert_raises(OSError, lambda: os.read(fd + 1, 10)) +os.close(fd) +assert_raises(OSError, lambda: os.read(fd, 10)) assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 3f0f557f1..0f5762014 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,6 +1,6 @@ use std::fs::File; use std::fs::OpenOptions; -use std::io::ErrorKind; +use std::io::{ErrorKind, Read}; use num_traits::cast::ToPrimitive; @@ -113,6 +113,25 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } +fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(fd, Some(vm.ctx.int_type())), (n, Some(vm.ctx.int_type()))] + ); + + let mut buffer = vec![0u8; objint::get_value(n).to_usize().unwrap()]; + let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); + match file.read_exact(&mut buffer) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_bytes(buffer)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -126,6 +145,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "read" => ctx.new_rustfunc(os_read), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 28c3ef1ae3c8a867c52f4306bd81e7587b484bff Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:17:36 +0300 Subject: [PATCH 121/658] Add os.write --- vm/src/stdlib/os.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0f5762014..b7e4280e5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,10 +1,11 @@ use std::fs::File; use std::fs::OpenOptions; -use std::io::{ErrorKind, Read}; +use std::io::{ErrorKind, Read, Write}; use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; +use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; @@ -132,6 +133,27 @@ fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(buffer)) } +fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [ + (fd, Some(vm.ctx.int_type())), + (data, Some(vm.ctx.bytes_type())) + ] + ); + + let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); + let written = match file.write(&objbytes::get_value(&data)) { + Ok(written) => written, + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_int(written)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -146,6 +168,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), "read" => ctx.new_rustfunc(os_read), + "write" => ctx.new_rustfunc(os_write), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 1f19951ece36d1e229f0054abacdcfa4a2cf0907 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:43:46 +0300 Subject: [PATCH 122/658] Add os.{remove,unlink} --- vm/src/stdlib/os.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b7e4280e5..cd4c0709b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,3 +1,4 @@ +use std::fs; use std::fs::File; use std::fs::OpenOptions; use std::io::{ErrorKind, Read, Write}; @@ -154,6 +155,18 @@ fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(written)) } +fn os_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(path, Some(vm.ctx.str_type()))]); + let fname = objstr::get_value(&path); + + match fs::remove_file(fname) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -169,6 +182,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "error" => ctx.new_rustfunc(os_error), "read" => ctx.new_rustfunc(os_read), "write" => ctx.new_rustfunc(os_write), + "remove" => ctx.new_rustfunc(os_remove), + "unlink" => ctx.new_rustfunc(os_remove), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From cdd39bc50e7054b6ffbb9e12245b2e5140e59d83 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:44:09 +0300 Subject: [PATCH 123/658] Add tests for os.{read, write} --- tests/snippets/stdlib_os.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index d70050495..29c8005dc 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -5,13 +5,36 @@ from testutils import assert_raises fd = os.open('README.md', 0) assert fd > 0 -assert len(os.read(fd, 10)) == 10 -assert len(os.read(fd, 5)) == 5 - assert_raises(OSError, lambda: os.read(fd + 1, 10)) os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) +FNAME = "test_file_that_no_one_will_have_on_disk" +CONTENT = b"testing" +CONTENT2 = b"rustpython" +CONTENT3 = b"BOYA" + +class TestWithFile(): + def __enter__(self): + open(FNAME, "wb") + return FNAME + + def __exit__(self, exc_type, exc_val, exc_tb): + os.remove(FNAME) + + +with TestWithFile() as fname: + fd = os.open(fname, 1) + assert os.write(fd, CONTENT2) == len(CONTENT2) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) + + fd = os.open(fname, 0) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) + + assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) From ed8c1e1a955725093c1ab04ea6b4e2c9c52e94ac Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 29 Mar 2019 16:39:52 +0300 Subject: [PATCH 124/658] Implement set and frozenset with PySetInner --- tests/snippets/set.py | 7 + vm/src/obj/objset.rs | 1110 +++++++++++++++++++++++------------------ 2 files changed, 622 insertions(+), 495 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 9e49dff89..011716335 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -205,6 +205,13 @@ assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) assert_raises(TypeError, lambda: frozenset([[]])) +a = frozenset([1,2,3]) +b = set() +for e in a: + assert e == 1 or e == 2 or e == 3 + b.add(e) +assert a == b + # set and frozen set assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 34d487fbb..eaa0cd09c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -7,7 +7,7 @@ use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; use std::hash::{Hash, Hasher}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::OptionalArg; use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -20,13 +20,13 @@ use super::objtype::PyClassRef; #[derive(Default)] pub struct PySet { - elements: RefCell>, + inner: RefCell, } pub type PySetRef = PyRef; #[derive(Default)] pub struct PyFrozenSet { - elements: HashMap, + inner: PySetInner, } pub type PyFrozenSetRef = PyRef; @@ -56,13 +56,564 @@ impl PyValue for PyFrozenSet { } } -pub fn get_elements(obj: &PyObjectRef) -> HashMap { +#[derive(Default, Clone)] +struct PySetInner { + elements: HashMap, +} + +impl PySetInner { + fn len(&self) -> usize { + self.elements.len() + } + fn copy(&self) -> PySetInner { + PySetInner { + elements: self.elements.clone(), + } + } + + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + for element in self.elements.iter() { + let value = vm._eq(needle.clone(), element.1.clone())?; + if objbool::get_value(&value) { + return Ok(vm.new_bool(true)); + } + } + Ok(vm.new_bool(false)) + } + + fn _compare_inner( + &self, + other: &PySetInner, + size_func: &Fn(usize, usize) -> bool, + swap: bool, + vm: &VirtualMachine, + ) -> PyResult { + let get_zelf = |swap: bool| -> &PySetInner { + if swap { + other + } else { + self + } + }; + let get_other = |swap: bool| -> &PySetInner { + if swap { + self + } else { + other + } + }; + + if size_func(get_zelf(swap).len(), get_other(swap).len()) { + return Ok(vm.new_bool(false)); + } + for element in get_other(swap).elements.iter() { + let value = get_zelf(swap).contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + return Ok(vm.new_bool(false)); + } + } + Ok(vm.new_bool(true)) + } + + fn eq(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf != other }, + false, + vm, + ) + } + + fn ge(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + false, + vm, + ) + } + + fn gt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + false, + vm, + ) + } + + fn le(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + true, + vm, + ) + } + + fn lt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + true, + vm, + ) + } + + fn union(&self, other: &PySetInner, _vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + elements.extend(other.elements.clone()); + + Ok(PySetInner { elements }) + } + + fn intersection(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._combine_inner(other, vm, SetCombineOperation::Intersection) + } + + fn difference(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._combine_inner(other, vm, SetCombineOperation::Difference) + } + + fn _combine_inner( + &self, + other: &PySetInner, + vm: &VirtualMachine, + op: SetCombineOperation, + ) -> PyResult { + let mut elements = HashMap::new(); + + for element in self.elements.iter() { + let value = other.contains(element.1.clone(), vm)?; + let should_add = match op { + SetCombineOperation::Intersection => objbool::get_value(&value), + SetCombineOperation::Difference => !objbool::get_value(&value), + }; + if should_add { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + Ok(PySetInner { elements }) + } + + fn symmetric_difference( + &self, + other: &PySetInner, + vm: &VirtualMachine, + ) -> PyResult { + let mut elements = HashMap::new(); + + for element in self.elements.iter() { + let value = other.contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + for element in other.elements.iter() { + let value = self.contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + Ok(PySetInner { elements }) + } + + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + let items = self.elements.values().cloned().collect(); + let set_list = vm.ctx.new_list(items); + PyListIterator { + position: Cell::new(0), + list: set_list.downcast().unwrap(), + } + } + + fn repr(&self, vm: &VirtualMachine) -> PyResult { + let mut str_parts = vec![]; + for elem in self.elements.values() { + let part = vm.to_repr(elem)?; + str_parts.push(part.value.clone()); + } + + Ok(format!("{{{}}}", str_parts.join(", "))) + } + + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + insert_into_set(vm, &mut self.elements, &item) + } + + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn remove( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + value: &PyObjectRef, + ) -> PyResult { + match elements.remove(&key) { + None => { + let item_str = format!("{:?}", value); + Err(vm.new_key_error(item_str)) + } + Some(_) => Ok(vm.get_none()), + } + } + perform_action_with_hash(vm, &mut self.elements, &item, &remove) + } + + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn discard( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + _value: &PyObjectRef, + ) -> PyResult { + elements.remove(&key); + Ok(vm.get_none()) + } + perform_action_with_hash(vm, &mut self.elements, &item, &discard) + } + + fn clear(&mut self, vm: &VirtualMachine) -> PyResult { + self.elements.clear(); + Ok(vm.get_none()) + } + + fn pop(&mut self, vm: &VirtualMachine) -> PyResult { + let elements = &mut self.elements; + match elements.clone().keys().next() { + Some(key) => Ok(elements.remove(key).unwrap()), + None => Err(vm.new_key_error("pop from an empty set".to_string())), + } + } + + fn update(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut self.elements, &v)?; + } + Ok(vm.get_none()) + } + + fn combine_update_inner( + &mut self, + iterable: &PyObjectRef, + vm: &VirtualMachine, + op: SetCombineOperation, + ) -> PyResult { + let elements = &mut self.elements; + for element in elements.clone().iter() { + let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; + let should_remove = match op { + SetCombineOperation::Intersection => !objbool::get_value(&value), + SetCombineOperation::Difference => objbool::get_value(&value), + }; + if should_remove { + elements.remove(&element.0.clone()); + } + } + Ok(vm.get_none()) + } + + fn ixor(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let elements_original = self.elements.clone(); + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut self.elements, &v)?; + } + for element in elements_original.iter() { + let value = vm.call_method(&iterable, "__contains__", vec![element.1.clone()])?; + if objbool::get_value(&value) { + self.elements.remove(&element.0.clone()); + } + } + + Ok(vm.get_none()) + } +} + +impl PySetRef { + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.borrow().len() + } + fn copy(self, _vm: &VirtualMachine) -> PySet { + PySet { + inner: RefCell::new(self.inner.borrow().copy()), + } + } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().eq(&get_inner(vm, &other)?, vm) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().ge(&get_inner(vm, &other)?, vm) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().gt(&get_inner(vm, &other)?, vm) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().le(&get_inner(vm, &other)?, vm) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().lt(&get_inner(vm, &other)?, vm) + } + + fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new(self.inner.borrow().union(&get_inner(vm, &other)?, vm)?), + }, + PySet::class(vm), + None, + )) + } + + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .intersection(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .difference(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .symmetric_difference(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.borrow().iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = self.inner.borrow(); + let s = if inner.len() == 0 { + format!("set()") + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + inner.repr(vm)? + } else { + format!("set(...)") + }; + Ok(vm.new_str(s)) + } + + fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().add(&item, vm) + } + + fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().remove(&item, vm) + } + + fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().discard(&item, vm) + } + + fn clear(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().clear(vm) + } + + fn pop(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().pop(vm) + } + + fn ior(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(self.as_object().clone()) + } + + fn update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn intersection_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Intersection, + )?; + Ok(vm.get_none()) + } + + fn iand(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Intersection, + )?; + Ok(self.as_object().clone()) + } + + fn difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Difference, + )?; + Ok(vm.get_none()) + } + + fn isub(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Difference, + )?; + Ok(self.as_object().clone()) + } + + fn symmetric_difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().ixor(iterable, vm)?; + Ok(vm.get_none()) + } + + fn ixor(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().ixor(iterable, vm)?; + Ok(self.as_object().clone()) + } +} + +impl PyFrozenSetRef { + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } + fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { + PyFrozenSet { + inner: self.inner.copy(), + } + } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.eq(&get_inner(vm, &other)?, vm) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.ge(&get_inner(vm, &other)?, vm) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.gt(&get_inner(vm, &other)?, vm) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.le(&get_inner(vm, &other)?, vm) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.lt(&get_inner(vm, &other)?, vm) + } + + fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.union(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.intersection(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.difference(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self + .inner + .symmetric_difference(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = &self.inner; + let s = if inner.len() == 0 { + format!("frozenset()") + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + format!("frozenset({})", inner.repr(vm)?) + } else { + format!("frozenset(...)") + }; + Ok(vm.new_str(s)) + } +} + +fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { if let Some(set) = obj.payload::() { - return set.elements.borrow().clone(); + Ok(set.inner.borrow().clone()) } else if let Some(frozenset) = obj.payload::() { - return frozenset.elements.clone(); + Ok(frozenset.inner.clone()) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) } - panic!("Not frozenset or set"); } fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { @@ -82,14 +633,16 @@ fn create_set( if objtype::issubclass(&cls, &vm.ctx.set_type()) { Ok(PyObject::new( PySet { - elements: RefCell::new(elements), + inner: RefCell::new(PySetInner { elements: elements }), }, PySet::class(vm), None, )) } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { Ok(PyObject::new( - PyFrozenSet { elements: elements }, + PyFrozenSet { + inner: PySetInner { elements: elements }, + }, PyFrozenSet::class(vm), None, )) @@ -130,84 +683,6 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } -fn set_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.add called with: {:?}", args); - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (item, None)] - ); - match zelf.payload::() { - Some(set) => insert_into_set(vm, &mut set.elements.borrow_mut(), item), - _ => Err(vm.new_type_error("set.add is called with no item".to_string())), - } -} - -fn set_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.remove called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &remove) - } - _ => Err(vm.new_type_error("set.remove is called with no item".to_string())), - } -} - -fn set_discard(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.discard called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &discard) - } - None => Err(vm.new_type_error("set.discard is called with no item".to_string())), - } -} - -fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.clear called"); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - match s.payload::() { - Some(set) => { - set.elements.borrow_mut().clear(); - Ok(vm.get_none()) - } - None => Err(vm.new_type_error("".to_string())), - } -} - /* Create a new object of sub-type of set */ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { validate_set_or_frozenset(vm, cls.clone())?; @@ -227,367 +702,11 @@ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMach create_set(vm, elements, cls.clone()) } -fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.len called with: {:?}", args); - arg_check!(vm, args, required = [(s, None)]); - validate_set_or_frozenset(vm, s.class())?; - let elements = get_elements(s); - Ok(vm.context().new_int(elements.len())) -} - -fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - trace!("set.copy called with: {:?}", obj); - validate_set_or_frozenset(vm, obj.class())?; - let elements = get_elements(&obj).clone(); - create_set(vm, elements, obj.class()) -} - -fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.set_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(o) { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("{{{}}}", str_parts.join(", ")) - } else { - "set(...)".to_string() - }; - Ok(vm.new_str(s)) -} - -pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(set, None), (needle, None)]); - validate_set_or_frozenset(vm, set.class())?; - for element in get_elements(set).iter() { - match vm._eq(needle.clone(), element.1.clone()) { - Ok(value) => { - if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), - } - } - - Ok(vm.new_bool(false)) -} - -fn set_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf != other }, - false, - ) -} - -fn set_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - false, - ) -} - -fn set_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - false, - ) -} - -fn set_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - true, - ) -} - -fn set_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - true, - ) -} - -fn set_compare_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - size_func: &Fn(usize, usize) -> bool, - swap: bool, -) -> PyResult { - arg_check!(vm, args, required = [(zelf, None), (other, None)]); - - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - - let get_zelf = |swap: bool| -> &PyObjectRef { - if swap { - other - } else { - zelf - } - }; - let get_other = |swap: bool| -> &PyObjectRef { - if swap { - zelf - } else { - other - } - }; - - let zelf_elements = get_elements(get_zelf(swap)); - let other_elements = get_elements(get_other(swap)); - if size_func(zelf_elements.len(), other_elements.len()) { - return Ok(vm.new_bool(false)); - } - for element in other_elements.iter() { - match vm.call_method(get_zelf(swap), "__contains__", vec![element.1.clone()]) { - Ok(value) => { - if !objbool::get_value(&value) { - return Ok(vm.new_bool(false)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), - } - } - Ok(vm.new_bool(true)) -} - -fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - - let mut elements = get_elements(&zelf).clone(); - elements.extend(get_elements(&other).clone()); - - create_set(vm, elements, zelf.class()) -} - -fn set_intersection(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Intersection) -} - -fn set_difference(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Difference) -} - -fn set_symmetric_difference( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - for element in get_elements(&other).iter() { - let value = vm.call_method(&zelf, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - enum SetCombineOperation { Intersection, Difference, } -fn set_combine_inner( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, - op: SetCombineOperation, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), - }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - -fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - - match s.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), - } - } - _ => Err(vm.new_type_error("".to_string())), - } -} - -fn set_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ior(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ior(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - } - _ => return Err(vm.new_type_error("set.update is called with no other".to_string())), - } - Ok(zelf.clone()) -} - -fn set_intersection_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection)?; - Ok(vm.get_none()) -} - -fn set_iand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection) -} - -fn set_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference)?; - Ok(vm.get_none()) -} - -fn set_isub(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference) -} - -fn set_combine_update_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - op: SetCombineOperation, -) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - Ok(zelf.clone()) -} - -fn set_symmetric_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ixor(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let elements_original = set.elements.borrow().clone(); - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - set.elements.borrow_mut().remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - - Ok(zelf.clone()) -} - -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type - let items = zelf.elements.borrow().values().cloned().collect(); - let set_list = vm.ctx.new_list(items); - PyListIterator { - position: Cell::new(0), - list: set_list.downcast().unwrap(), - } -} - -fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.frozenset_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "frozenset()".to_string() - } else { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("frozenset({{{}}})", str_parts.join(", ")) - }; - Ok(vm.new_str(s)) -} - fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } @@ -600,42 +719,42 @@ pub fn init(context: &PyContext) { Build an unordered collection of unique elements."; extend_class!(context, set_type, { - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), + "__contains__" => context.new_rustfunc(PySetRef::contains), + "__len__" => context.new_rustfunc(PySetRef::len), "__new__" => context.new_rustfunc(set_new), - "__repr__" => context.new_rustfunc(set_repr), "__hash__" => context.new_rustfunc(set_hash), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), + "__repr__" => context.new_rustfunc(PySetRef::repr), + "__eq__" => context.new_rustfunc(PySetRef::eq), + "__ge__" => context.new_rustfunc(PySetRef::ge), + "__gt__" => context.new_rustfunc(PySetRef::gt), + "__le__" => context.new_rustfunc(PySetRef::le), + "__lt__" => context.new_rustfunc(PySetRef::lt), + "issubset" => context.new_rustfunc(PySetRef::le), + "issuperset" => context.new_rustfunc(PySetRef::ge), + "union" => context.new_rustfunc(PySetRef::union), + "__or__" => context.new_rustfunc(PySetRef::union), + "intersection" => context.new_rustfunc(PySetRef::intersection), + "__and__" => context.new_rustfunc(PySetRef::intersection), + "difference" => context.new_rustfunc(PySetRef::difference), + "__sub__" => context.new_rustfunc(PySetRef::difference), + "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), "__doc__" => context.new_str(set_doc.to_string()), - "add" => context.new_rustfunc(set_add), - "remove" => context.new_rustfunc(set_remove), - "discard" => context.new_rustfunc(set_discard), - "clear" => context.new_rustfunc(set_clear), - "copy" => context.new_rustfunc(set_copy), - "pop" => context.new_rustfunc(set_pop), - "update" => context.new_rustfunc(set_update), - "__ior__" => context.new_rustfunc(set_ior), - "intersection_update" => context.new_rustfunc(set_intersection_update), - "__iand__" => context.new_rustfunc(set_iand), - "difference_update" => context.new_rustfunc(set_difference_update), - "__isub__" => context.new_rustfunc(set_isub), - "symmetric_difference_update" => context.new_rustfunc(set_symmetric_difference_update), - "__ixor__" => context.new_rustfunc(set_ixor), - "__iter__" => context.new_rustfunc(set_iter) + "add" => context.new_rustfunc(PySetRef::add), + "remove" => context.new_rustfunc(PySetRef::remove), + "discard" => context.new_rustfunc(PySetRef::discard), + "clear" => context.new_rustfunc(PySetRef::clear), + "copy" => context.new_rustfunc(PySetRef::copy), + "pop" => context.new_rustfunc(PySetRef::pop), + "update" => context.new_rustfunc(PySetRef::update), + "__ior__" => context.new_rustfunc(PySetRef::ior), + "intersection_update" => context.new_rustfunc(PySetRef::intersection_update), + "__iand__" => context.new_rustfunc(PySetRef::iand), + "difference_update" => context.new_rustfunc(PySetRef::difference_update), + "__isub__" => context.new_rustfunc(PySetRef::isub), + "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), + "__ixor__" => context.new_rustfunc(PySetRef::ixor), + "__iter__" => context.new_rustfunc(PySetRef::iter) }); let frozenset_type = &context.frozenset_type; @@ -646,25 +765,26 @@ pub fn init(context: &PyContext) { extend_class!(context, frozenset_type, { "__new__" => context.new_rustfunc(set_new), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), + "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), + "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), + "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), + "__le__" => context.new_rustfunc(PyFrozenSetRef::le), + "__lt__" => context.new_rustfunc(PyFrozenSetRef::lt), + "issubset" => context.new_rustfunc(PyFrozenSetRef::le), + "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), + "union" => context.new_rustfunc(PyFrozenSetRef::union), + "__or__" => context.new_rustfunc(PyFrozenSetRef::union), + "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), + "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), + "difference" => context.new_rustfunc(PyFrozenSetRef::difference), + "__sub__" => context.new_rustfunc(PyFrozenSetRef::difference), + "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), + "__len__" => context.new_rustfunc(PyFrozenSetRef::len), "__doc__" => context.new_str(frozenset_doc.to_string()), - "__repr__" => context.new_rustfunc(frozenset_repr), - "copy" => context.new_rustfunc(set_copy) + "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), + "copy" => context.new_rustfunc(PyFrozenSetRef::copy), + "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter) }); } From a685216f29ecc75e2bc3f9637a42e5fe878c5b37 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 4 Apr 2019 20:47:29 +0300 Subject: [PATCH 125/658] Use new arg style for set and frozenset new --- vm/src/obj/objset.rs | 103 +++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 58 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index eaa0cd09c..fd31044aa 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -15,7 +15,6 @@ use super::objbool; use super::objint; use super::objiter; use super::objlist::PyListIterator; -use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] @@ -62,6 +61,22 @@ struct PySetInner { } impl PySetInner { + fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + let elements: HashMap = match iterable { + OptionalArg::Missing => HashMap::new(), + OptionalArg::Present(iterable) => { + let mut elements = HashMap::new(); + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut elements, &v)?; + } + elements + } + }; + + Ok(PySetInner { elements }) + } + fn len(&self) -> usize { self.elements.len() } @@ -334,14 +349,27 @@ impl PySetInner { } impl PySetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PySet { + inner: RefCell::new(PySetInner::new(iterable, vm)?), + } + .into_ref_with_type(vm, cls) + } + fn len(self, _vm: &VirtualMachine) -> usize { self.inner.borrow().len() } + fn copy(self, _vm: &VirtualMachine) -> PySet { PySet { inner: RefCell::new(self.inner.borrow().copy()), } } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow().contains(needle, vm) } @@ -512,14 +540,27 @@ impl PySetRef { } impl PyFrozenSetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyFrozenSet { + inner: PySetInner::new(iterable, vm)?, + } + .into_ref_with_type(vm, cls) + } + fn len(self, _vm: &VirtualMachine) -> usize { self.inner.len() } + fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { PyFrozenSet { inner: self.inner.copy(), } } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.contains(needle, vm) } @@ -616,41 +657,6 @@ fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { } } -fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { - if !(objtype::issubclass(&cls, &vm.ctx.set_type()) - || objtype::issubclass(&cls, &vm.ctx.frozenset_type())) - { - return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))); - } - Ok(()) -} - -fn create_set( - vm: &VirtualMachine, - elements: HashMap, - cls: PyClassRef, -) -> PyResult { - if objtype::issubclass(&cls, &vm.ctx.set_type()) { - Ok(PyObject::new( - PySet { - inner: RefCell::new(PySetInner { elements: elements }), - }, - PySet::class(vm), - None, - )) - } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { - Ok(PyObject::new( - PyFrozenSet { - inner: PySetInner { elements: elements }, - }, - PyFrozenSet::class(vm), - None, - )) - } else { - Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) - } -} - fn perform_action_with_hash( vm: &VirtualMachine, elements: &mut HashMap, @@ -683,25 +689,6 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } -/* Create a new object of sub-type of set */ -fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, cls.clone())?; - - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; - } - elements - } - }; - - create_set(vm, elements, cls.clone()) -} - enum SetCombineOperation { Intersection, Difference, @@ -719,10 +706,10 @@ pub fn init(context: &PyContext) { Build an unordered collection of unique elements."; extend_class!(context, set_type, { + "__hash__" => context.new_rustfunc(set_hash), "__contains__" => context.new_rustfunc(PySetRef::contains), "__len__" => context.new_rustfunc(PySetRef::len), - "__new__" => context.new_rustfunc(set_new), - "__hash__" => context.new_rustfunc(set_hash), + "__new__" => context.new_rustfunc(PySetRef::new), "__repr__" => context.new_rustfunc(PySetRef::repr), "__eq__" => context.new_rustfunc(PySetRef::eq), "__ge__" => context.new_rustfunc(PySetRef::ge), @@ -764,7 +751,7 @@ pub fn init(context: &PyContext) { Build an immutable unordered collection of unique elements."; extend_class!(context, frozenset_type, { - "__new__" => context.new_rustfunc(set_new), + "__new__" => context.new_rustfunc(PyFrozenSetRef::new), "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), From 2a3a88c2f2cf0f13f0ed89843f19680f43b8d403 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 4 Apr 2019 21:23:00 +0300 Subject: [PATCH 126/658] Use match_class macro to avoid unwanted clones --- vm/src/obj/objset.rs | 135 ++++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 45 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index fd31044aa..6c67312fb 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -375,29 +375,53 @@ impl PySetRef { } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().eq(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().ge(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().gt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().le(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().lt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(self.inner.borrow().union(&get_inner(vm, &other)?, vm)?), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().union(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().union(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -407,11 +431,11 @@ impl PySetRef { fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .intersection(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().intersection(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().intersection(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -421,11 +445,11 @@ impl PySetRef { fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .difference(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -435,11 +459,11 @@ impl PySetRef { fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .symmetric_difference(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().symmetric_difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().symmetric_difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -566,29 +590,53 @@ impl PyFrozenSetRef { } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.eq(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.ge(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.gt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.le(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.lt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.union(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.union(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.union(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -598,7 +646,11 @@ impl PyFrozenSetRef { fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.intersection(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.intersection(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.intersection(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -608,7 +660,11 @@ impl PyFrozenSetRef { fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.difference(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -618,9 +674,11 @@ impl PyFrozenSetRef { fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self - .inner - .symmetric_difference(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.symmetric_difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.symmetric_difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -644,19 +702,6 @@ impl PyFrozenSetRef { } } -fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - if let Some(set) = obj.payload::() { - Ok(set.inner.borrow().clone()) - } else if let Some(frozenset) = obj.payload::() { - Ok(frozenset.inner.clone()) - } else { - Err(vm.new_type_error(format!( - "{} is not a subtype of set or frozenset", - obj.class() - ))) - } -} - fn perform_action_with_hash( vm: &VirtualMachine, elements: &mut HashMap, From ab3d004c4227d79ada563c72a65cb2ac569fd47c Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Fri, 5 Apr 2019 21:59:20 +0200 Subject: [PATCH 127/658] Fix syntax for float literals, statements seperated by semicolons and starargs after keyword arguments. --- parser/src/ast.rs | 2 +- parser/src/lexer.rs | 22 ++++++++++-------- parser/src/parser.rs | 18 +++++++-------- parser/src/python.lalrpop | 38 ++++++++++++++++++++------------ tests/snippets/comprehensions.py | 5 +++++ tests/snippets/floats.py | 4 ++++ tests/snippets/function_args.py | 9 ++++++++ tests/snippets/statements.py | 9 ++++++++ vm/src/compile.rs | 16 ++++++++------ 9 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 tests/snippets/statements.py diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 21c8dd736..9fbe8153e 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -16,7 +16,7 @@ pub struct Node { #[derive(Debug, PartialEq)] pub enum Top { Program(Program), - Statement(LocatedStatement), + Statement(Vec), Expression(Expression), } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index edf21c973..ebdb9555d 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1044,16 +1044,20 @@ where return Some(Ok((tok_start, Tok::Comma, tok_end))); } '.' => { - let tok_start = self.get_pos(); - self.next_char(); - if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { - self.next_char(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + if let Some('0'..='9') = self.chr1 { + return Some(self.lex_number()); } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Dot, tok_end))); + let tok_start = self.get_pos(); + self.next_char(); + if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { + self.next_char(); + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Dot, tok_end))); + } } } '\n' => { diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 4c37f2842..dd58bd11d 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result { do_lalr_parsing!(source, Program, StartProgram) } -pub fn parse_statement(source: &str) -> Result { +pub fn parse_statement(source: &str) -> Result, ParseError> { do_lalr_parsing!(source, Statement, StartStatement) } @@ -180,7 +180,7 @@ mod tests { let parse_ast = parse_statement(&source).unwrap(); assert_eq!( parse_ast, - ast::LocatedStatement { + vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::If { test: ast::Expression::Number { @@ -229,7 +229,7 @@ mod tests { } },]), } - } + }] ); } @@ -239,7 +239,7 @@ mod tests { let parse_ast = parse_statement(&source); assert_eq!( parse_ast, - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Expression { expression: ast::Expression::Lambda { @@ -271,7 +271,7 @@ mod tests { }) } } - }) + }]) ) } @@ -281,7 +281,7 @@ mod tests { assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Assign { targets: vec![ast::Expression::Tuple { @@ -309,7 +309,7 @@ mod tests { ] } } - }) + }]) ) } @@ -320,7 +320,7 @@ mod tests { ); assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::ClassDef { name: String::from("Foo"), @@ -393,7 +393,7 @@ mod tests { ], decorator_list: vec![], } - }) + }]) ) } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 305f97c10..a9663943f 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -24,29 +24,32 @@ pub Top: ast::Top = { Program: ast::Program = { => ast::Program { - statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) + statements: Vec::from_iter(lines.into_iter().flatten()) }, }; // A file line either has a declaration, or an empty newline: -FileLine: Option = { - => Some(s), - "\n" => None, +FileLine: Vec = { + Statement, + "\n" => vec![], }; Suite: Vec = { - => vec![s], - "\n" indent dedent => s, -}; - -Statement: ast::LocatedStatement = { SimpleStatement, - CompoundStatement, + "\n" indent dedent => s.into_iter().flatten().collect(), }; -SimpleStatement: ast::LocatedStatement = { - "\n" => s, - ";" => s, +Statement: Vec = { + SimpleStatement, + => vec![s], +}; + +SimpleStatement: Vec = { + ";"? "\n" => { + let mut statements = vec![s1]; + statements.extend(s2.into_iter().map(|e| e.1)); + statements + } }; SmallStatement: ast::LocatedStatement = { @@ -899,7 +902,14 @@ ArgumentList: (Vec, Vec) = { keywords.push(ast::Keyword { name: n, value: value }); }, None => { - if keywords.len() > 0 { + // Allow starred args after keyword arguments. + let is_starred = if let ast::Expression::Starred { .. } = &value { + true + } else { + false + }; + + if keywords.len() > 0 && !is_starred { panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); diff --git a/tests/snippets/comprehensions.py b/tests/snippets/comprehensions.py index 14c545de3..132f3cdcb 100644 --- a/tests/snippets/comprehensions.py +++ b/tests/snippets/comprehensions.py @@ -22,3 +22,8 @@ assert u['1'] == -1 y = [a+2 for a in x if a % 2] print(y) assert y == [3, 5] + +z = [(9,), (10,)] +w = [x for x, in z] +assert w == [9, 10] + diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 6675615c5..08e7d4b2b 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -106,3 +106,7 @@ assert_raises(OverflowError, float('inf').as_integer_ratio) assert_raises(OverflowError, float('-inf').as_integer_ratio) assert_raises(ValueError, float('nan').as_integer_ratio) +# Test special case for lexer, float starts with a dot: +a = .5 +assert a == 0.5 + diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 745648f9d..884cb47df 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -42,3 +42,12 @@ va2(5, *x[1:]) x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) + +# star arg after keyword args: +def fubar(x, y, obj=None): + assert x == 4 + assert y == 5 + assert obj == 6 + +rest = [4, 5] +fubar(obj=6, *rest) diff --git a/tests/snippets/statements.py b/tests/snippets/statements.py new file mode 100644 index 000000000..1921363f5 --- /dev/null +++ b/tests/snippets/statements.py @@ -0,0 +1,9 @@ + +# Test several statement types + +# small ones, seperated by ';': + +if True: + 5;4; + b=4 + b; diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 88818f52f..5802d9e30 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -129,15 +129,17 @@ impl Compiler { // Compile statement in eval mode: fn compile_statement_eval( &mut self, - statement: &ast::LocatedStatement, + statements: &[ast::LocatedStatement], ) -> Result<(), CompileError> { - if let ast::Statement::Expression { ref expression } = statement.node { - self.compile_expression(expression)?; - self.emit(Instruction::ReturnValue); - Ok(()) - } else { - Err(CompileError::ExpectExpr) + for statement in statements { + if let ast::Statement::Expression { ref expression } = statement.node { + self.compile_expression(expression)?; + } else { + return Err(CompileError::ExpectExpr); + } } + self.emit(Instruction::ReturnValue); + Ok(()) } fn compile_statements( From dda0d561ac8f23da8f16951bc33891630e0be955 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 10:43:39 +1300 Subject: [PATCH 128/658] Fix order of dict literal --- Cargo.lock | 1 + tests/snippets/dict.py | 6 ++++++ vm/Cargo.toml | 1 + vm/src/frame.rs | 14 ++++++++------ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36e313e26..864e90ce5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,6 +809,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 5c5ef62b1..2d369465c 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -80,3 +80,9 @@ x[Hashable(8)] = 19 x[Hashable(19,8)] = 1 assert x[Hashable(8)] == 1 assert len(x) == 2 + +assert list({'a': 2, 'b': 10}) == ['a', 'b'] +x = {} +x['a'] = 2 +x['b'] = 10 +assert list(x) == ['a', 'b'] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e1b1cc9ff..ef1a9cbc3 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -26,3 +26,4 @@ caseless = "0.2.1" unicode-segmentation = "1.2.1" lazy_static = "^1.0.1" lexical = "2.0.0" +itertools = "^0.8.0" diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 858b33a08..2f525afdd 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -25,6 +25,7 @@ use crate::pyobject::{ TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use itertools::Itertools; /* * So a scope is a linked list of scopes. @@ -386,9 +387,8 @@ impl Frame { } bytecode::Instruction::BuildMap { size, unpack } => { let map_obj = vm.ctx.new_dict(); - for _x in 0..*size { - let obj = self.pop_value(); - if *unpack { + if *unpack { + for obj in self.pop_multiple(*size) { // Take all key-value pairs from the dict: let dict: PyDictRef = obj.downcast().expect("Need a dictionary to build a map."); @@ -396,11 +396,13 @@ impl Frame { for (key, value) in dict_elements.iter() { map_obj.set_item(key.clone(), value.clone(), vm); } - } else { - let key = self.pop_value(); - map_obj.set_item(key, obj, vm) + } + } else { + for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { + map_obj.set_item(key, value, vm) } } + self.push_value(map_obj.into_object()); Ok(None) } From a7d5b0e9ac1ecd8153d5ceeaa796340755ccb692 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 6 Apr 2019 01:44:00 +0200 Subject: [PATCH 129/658] start --- vm/src/obj/mod.rs | 1 + vm/src/obj/objbyteinner.rs | 9 +++ vm/src/obj/objbytes.rs | 151 +++++++++++++++++++++---------------- 3 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 vm/src/obj/objbyteinner.rs diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index 0419d827f..c824e726d 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -3,6 +3,7 @@ pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; +pub mod objbyteinner; pub mod objbytes; pub mod objclassmethod; pub mod objcode; diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs new file mode 100644 index 000000000..0607621f1 --- /dev/null +++ b/vm/src/obj/objbyteinner.rs @@ -0,0 +1,9 @@ +#[derive(Debug, Default, Clone)] +pub struct PyByteInner { + pub elements: Vec, +} +impl PyByteInner { + pub fn new(data: Vec) -> Self { + PyByteInner { elements: data } + } +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f24c9273c..8f43cd05f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,22 +6,37 @@ use std::ops::Deref; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; use crate::vm::VirtualMachine; +use super::objbyteinner::PyByteInner; use super::objint; use super::objiter; use super::objtype::PyClassRef; +/// "bytes(iterable_of_ints) -> bytes\n\ +/// bytes(string, encoding[, errors]) -> bytes\n\ +/// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ +/// bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ +/// bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytes", __inside_vm)] #[derive(Debug)] pub struct PyBytes { - value: Vec, + inner: PyByteInner, } type PyBytesRef = PyRef; impl PyBytes { - pub fn new(data: Vec) -> Self { - PyBytes { value: data } + pub fn new(elements: Vec) -> Self { + PyBytes { + inner: PyByteInner { elements }, + } } } @@ -29,7 +44,7 @@ impl Deref for PyBytes { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.value + &self.inner.elements } } @@ -42,62 +57,74 @@ impl PyValue for PyBytes { // Binary data support // Fill bytes class methods: -pub fn init(context: &PyContext) { - let bytes_doc = - "bytes(iterable_of_ints) -> bytes\n\ - bytes(string, encoding[, errors]) -> bytes\n\ - bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ - bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ - bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - any object implementing the buffer API.\n \ - - an integer"; - extend_class!(context, &context.bytes_type, { - "__new__" => context.new_rustfunc(bytes_new), - "__eq__" => context.new_rustfunc(PyBytesRef::eq), - "__lt__" => context.new_rustfunc(PyBytesRef::lt), - "__le__" => context.new_rustfunc(PyBytesRef::le), - "__gt__" => context.new_rustfunc(PyBytesRef::gt), - "__ge__" => context.new_rustfunc(PyBytesRef::ge), - "__hash__" => context.new_rustfunc(PyBytesRef::hash), - "__repr__" => context.new_rustfunc(PyBytesRef::repr), - "__len__" => context.new_rustfunc(PyBytesRef::len), - "__iter__" => context.new_rustfunc(PyBytesRef::iter), - "__doc__" => context.new_str(bytes_doc.to_string()) - }); - - let bytesiterator_type = &context.bytesiterator_type; - extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), - }); +pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { + &obj.payload::().unwrap().inner.elements } -fn bytes_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); - } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; - - PyBytes::new(value).into_ref_with_type(vm, cls) +// pub fn init(context: &PyContext) { +// let bytes_doc = +pub fn init(ctx: &PyContext) { + PyBytesRef::extend_class(ctx, &ctx.bytes_type); } +//extend_class!(context, &context.bytes_type, { +//"__new__" => context.new_rustfunc(bytes_new), +/* "__eq__" => context.new_rustfunc(PyBytesRef::eq), +"__lt__" => context.new_rustfunc(PyBytesRef::lt), +"__le__" => context.new_rustfunc(PyBytesRef::le), +"__gt__" => context.new_rustfunc(PyBytesRef::gt), +"__ge__" => context.new_rustfunc(PyBytesRef::ge), +"__hash__" => context.new_rustfunc(PyBytesRef::hash), +"__repr__" => context.new_rustfunc(PyBytesRef::repr), +"__len__" => context.new_rustfunc(PyBytesRef::len), +"__iter__" => context.new_rustfunc(PyBytesRef::iter), +"__doc__" => context.new_str(bytes_doc.to_string())*/ +// }); +/* let bytesiterator_type = &context.bytesiterator_type; +extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), +});*/ +//} +#[pyimpl(__inside_vm)] impl PyBytesRef { + #[pymethod(name = "__new__")] + fn bytes_new( + cls: PyClassRef, + val_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // Create bytes data: + let value = if let OptionalArg::Present(ival) = val_option { + let elements = vm.extract_elements(&ival)?; + let mut data_bytes = vec![]; + for elem in elements.iter() { + let v = objint::to_int(vm, elem, 10)?; + data_bytes.push(v.to_u8().unwrap()); + } + data_bytes + // return Err(vm.new_type_error("Cannot construct bytes".to_string())); + } else { + vec![] + }; + + PyBytes::new(value).into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__repr__")] + fn repr(self, _vm: &VirtualMachine) -> String { + // TODO: don't just unwrap + let data = self.inner.elements.clone(); + format!("b'{:?}'", data) + } + + #[pymethod(name = "__len__")] + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.elements.len() + } +} +/* fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if let Ok(other) = other.downcast::() { vm.ctx.new_bool(self.value == other.value) @@ -138,9 +165,6 @@ impl PyBytesRef { } } - fn len(self, _vm: &VirtualMachine) -> usize { - self.value.len() - } fn hash(self, _vm: &VirtualMachine) -> u64 { let mut hasher = DefaultHasher::new(); @@ -148,11 +172,6 @@ impl PyBytesRef { hasher.finish() } - fn repr(self, _vm: &VirtualMachine) -> String { - // TODO: don't just unwrap - let data = String::from_utf8(self.value.clone()).unwrap(); - format!("b'{}'", data) - } fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { PyBytesIterator { @@ -162,9 +181,7 @@ impl PyBytesRef { } } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - &obj.payload::().unwrap().value -} + #[derive(Debug)] pub struct PyBytesIterator { @@ -194,4 +211,4 @@ impl PyBytesIteratorRef { fn iter(self, _vm: &VirtualMachine) -> Self { self } -} +}*/ From 7fffc572559d5d2b889af747f083f5dad16645f9 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 12:46:48 +1300 Subject: [PATCH 130/658] Fix decorator on functions with defaults --- tests/snippets/decorators.py | 10 ++++++++++ vm/src/compile.rs | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/snippets/decorators.py b/tests/snippets/decorators.py index fcc196f93..2633308f1 100644 --- a/tests/snippets/decorators.py +++ b/tests/snippets/decorators.py @@ -15,6 +15,16 @@ c = add(10, 3) assert c == 14 +@logged +def add3(a, b, c=2): + return a + b + c + + +d = add3(12, 5) + +assert d == 20 + + def f(func): return lambda: 42 class A: pass a = A() diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 88818f52f..d7f774b83 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -585,6 +585,9 @@ impl Compiler { let was_in_function_def = self.in_function_def; self.in_loop = false; self.in_function_def = true; + + self.prepare_decorators(decorator_list)?; + let mut flags = self.enter_function(name, args)?; let (new_body, doc_str) = get_doc(body); @@ -598,8 +601,6 @@ impl Compiler { self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - self.prepare_decorators(decorator_list)?; - // Prepare type annotations: let mut num_annotations = 0; From 909d94ec233749d9bfc42fdab830e08eb3cfa4fd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Fri, 5 Apr 2019 20:25:53 -0600 Subject: [PATCH 131/658] impl and use SequenceIndex for list.__delitem__ - removal of del_item - all del function impl on PyListRef - stepped deletion now loops only over the range insted of the whole list --- tests/snippets/list.py | 4 + vm/src/obj/objlist.rs | 176 ++++++++++++++++++++++++++++++++++++-- vm/src/obj/objsequence.rs | 152 +++----------------------------- 3 files changed, 188 insertions(+), 144 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6efa52cc0..3d9743c0b 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -190,6 +190,10 @@ x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] del x[-5:] assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] +x = list(range(12)) +del x[10:2:-2] +assert x == [0,1,2,3,5,7,9,11] + def bad_del_1(): del ['a', 'b']['a'] assert_raises(TypeError, bad_del_1) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index f3e812615..60c14b107 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,9 +1,10 @@ use std::cell::{Cell, RefCell}; use std::fmt; -use std::ops::DerefMut; +use std::ops::Range; -use num_traits::ToPrimitive; +use num_bigint::BigInt; +use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; @@ -13,9 +14,10 @@ use super::objbool; use super::objint; use super::objiter; use super::objsequence::{ - del_item, get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, - seq_mul, PySliceableSequence, + get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, + PySliceableSequence, SequenceIndex, }; +use super::objslice::PySliceRef; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -46,6 +48,54 @@ impl PyValue for PyList { } } +impl PyList { + pub fn get_len(&self) -> usize { + self.elements.borrow().len() + } + + pub fn get_pos(&self, p: i32) -> Option { + // convert a (potentially negative) positon into a real index + if p < 0 { + if -p as usize > self.get_len() { + None + } else { + Some(self.get_len() - ((-p) as usize)) + } + } else if p as usize >= self.get_len() { + None + } else { + Some(p as usize) + } + } + + pub fn get_slice_pos(&self, slice_pos: &BigInt) -> usize { + if let Some(pos) = slice_pos.to_i32() { + if let Some(index) = self.get_pos(pos) { + // within bounds + return index; + } + } + + if slice_pos.is_negative() { + // slice past start bound, round to start + 0 + } else { + // slice past end bound, round to end + self.get_len() + } + } + + pub fn get_slice_range(&self, start: &Option, stop: &Option) -> Range { + let start = start.as_ref().map(|x| self.get_slice_pos(x)).unwrap_or(0); + let stop = stop + .as_ref() + .map(|x| self.get_slice_pos(x)) + .unwrap_or_else(|| self.get_len()); + + start..stop + } +} + pub type PyListRef = PyRef; impl PyListRef { @@ -313,9 +363,123 @@ impl PyListRef { } } - fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn delitem(self, subscript: SequenceIndex, vm: &VirtualMachine) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.delindex(index, vm), + SequenceIndex::Slice(slice) => self.delslice(slice, vm), + } + } + + fn delindex(self, index: i32, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut().remove(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + + fn delslice(self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { + let start = &slice.start; + let stop = &slice.stop; + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&start, &stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice_reverse(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } + } + + fn _del_slice(self, range: Range) { + self.elements.borrow_mut().drain(range); + } + + fn _del_stepped_slice(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; let mut elements = self.elements.borrow_mut(); - del_item(vm, self.as_object(), elements.deref_mut(), key) + let mut indexes = range.clone().step_by(step).peekable(); + + for i in range.clone() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + elements.swap(i - deleted, i); + } + } + // then drain (the values to delete should now be contiguous at the end of the range) + elements.drain((range.end - deleted)..range.end); + } + + fn _del_stepped_slice_reverse(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().rev().step_by(step).peekable(); + + for i in range.clone().rev() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards back + elements.swap(i + deleted, i); + } + } + // then drain (the values to delete should now be contiguous at teh start of the range) + elements.drain(range.start..(range.start + deleted)); } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 9593e57fe..14f9c5c93 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut, Range}; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; -use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use super::objbool; @@ -137,104 +137,21 @@ impl PySliceableSequence for Vec { } } -pub trait PySliceableSequenceMut: PySliceableSequence { - //fn set_slice(&self, range: Range, seq: T) -> Self; - //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; - //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; - //fn set_stepped_slice_reverse(&self, range: Range, step: usize, seq: T) -> Self; - - fn del_slice(&mut self, range: Range); - fn del_stepped_slice(&mut self, range: Range, step: usize); - fn del_index(&mut self, index: usize); - - fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PySliceRef) -> PyResult { - - let start = &slice.start; - let stop = &slice.stop; - - let step = slice.step.clone().unwrap_or_else(BigInt::one); - - if step.is_zero() { - Err(vm.new_value_error("slice step cannot be zero".to_string())) - } else if step.is_positive() { - let range = self.get_slice_range(start, stop); - if range.start < range.end { - #[allow(clippy::range_plus_one)] - match step.to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.start..range.start + 1); - Ok(vm.get_none()) - } - } - } else { - // no del to do - Ok(vm.get_none()) - } - } else { - // calculate the range for the reverse slice, first the bounds needs to be made - // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); - let range = self.get_slice_range(&stop, &start); - if range.start < range.end { - match (-step).to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.end - 1..range.end); - Ok(vm.get_none()) - } - } - } else { - // no del to do - Ok(vm.get_none()) - } - } - - } +pub enum SequenceIndex { + Int(i32), + Slice(PySliceRef), } -impl PySliceableSequenceMut for Vec { - fn del_slice(&mut self, range: Range) { - self.drain(range); - } - - fn del_stepped_slice(&mut self, range: Range, step: usize) { - // no easy way to delete steped indexes so heres what we'll do - let len = self.len(); - let mut deleted = 0; - let mut indexes = range.step_by(step).peekable(); - for i in 0..len { - // is this an index to delete? - if indexes.peek() == Some(&i) { - // record and move on - indexes.next(); - deleted += 1; - } else { - // swap towards front - self.swap(i - deleted, i); - } - } - // then truncate the vec (the values to delete should now be the last elements and thus choped off) - self.truncate(len - deleted); - } - - fn del_index(&mut self, index: usize) { - self.remove(index); +impl TryFromObject for SequenceIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(SequenceIndex::Int(i32::try_from_object(vm, i.into_object())?)), + s @ PySlice => Ok(SequenceIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) } } @@ -284,47 +201,6 @@ pub fn get_item( } } -pub fn del_item( - vm: &VirtualMachine, - sequence: &PyObjectRef, - elements: &mut T, - subscript: PyObjectRef, -) -> PyResult { - if let Some(i) = subscript.payload::() { - return match i.as_bigint().to_i32() { - Some(value) => { - if let Some(pos_index) = elements.get_pos(value) { - elements.del_index(pos_index); - Ok(vm.get_none()) - } else { - Err(vm.new_index_error("Index out of bounds!".to_string())) - } - } - None => { - Err(vm.new_index_error("cannot fit 'int' into an index-sized integer".to_string())) - } - }; - } - - if subscript.payload::().is_some() { - if sequence.payload::().is_some() { - if let Ok(slice) = subscript.downcast::() { - elements.del_slice_items(vm, &slice) - } else { - panic!("PySlice is not a slice? this should not be") - } - - } else { - panic!("sequence del_item called for non-mutable-sequence") - } - } else { - Err(vm.new_type_error(format!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - sequence, subscript - ))) - } -} - pub fn seq_equal( vm: &VirtualMachine, zelf: &[PyObjectRef], From 157204c61ad6f80503c413245019e1a4ffd88054 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 5 Apr 2019 23:24:26 -0500 Subject: [PATCH 132/658] Clean up the wasm crate --- derive/src/from_args.rs | 23 +++--- wasm/lib/src/browser_module.rs | 123 +++++++++++++++------------------ wasm/lib/src/convert.rs | 8 +-- wasm/lib/src/vm_class.rs | 58 +++++----------- 4 files changed, 89 insertions(+), 123 deletions(-) diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs index 7845b2880..040ebf36c 100644 --- a/derive/src/from_args.rs +++ b/derive/src/from_args.rs @@ -109,7 +109,7 @@ impl ArgAttribute { } } -fn generate_field(field: &Field) -> TokenStream2 { +fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { let mut pyarg_attrs = field .attrs .iter() @@ -132,7 +132,7 @@ fn generate_field(field: &Field) -> TokenStream2 { let name = &field.ident; let middle = quote! { - .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + .map(|x| #rp_path::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? }; let ending = if let Some(default) = attr.default { quote! { @@ -140,16 +140,16 @@ fn generate_field(field: &Field) -> TokenStream2 { } } else if attr.optional { quote! { - .map(crate::function::OptionalArg::Present) - .unwrap_or(crate::function::OptionalArg::Missing) + .map(#rp_path::function::OptionalArg::Present) + .unwrap_or(#rp_path::function::OptionalArg::Missing) } } else { let err = match attr.kind { ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - crate::function::ArgumentError::TooFewArgs + #rp_path::function::ArgumentError::TooFewArgs }, ParameterKind::KeywordOnly => quote! { - crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + #rp_path::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) }, }; quote! { @@ -181,7 +181,10 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { let fields = match input.data { Data::Struct(ref data) => { match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(generate_field), + Fields::Named(ref fields) => fields + .named + .iter() + .map(|field| generate_field(field, &rp_path)), Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message } } @@ -192,9 +195,9 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { quote! { impl #rp_path::function::FromArgs for #name { fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { + vm: &#rp_path::VirtualMachine, + args: &mut #rp_path::function::PyFuncArgs + ) -> Result { Ok(#name { #(#fields)* }) } } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 6f7c31964..607cfc886 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,16 +7,13 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ - objdict::PyDictRef, - objfunction::PyFunction, - objint, - objstr::{self, PyStringRef}, + objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; -use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; +use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; enum FetchResponseFormat { Json, @@ -42,25 +39,38 @@ impl FetchResponseFormat { } } -fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); +#[derive(FromArgs)] +struct FetchArgs { + #[pyarg(keyword_only, default = "None")] + response_format: Option, + #[pyarg(keyword_only, default = "None")] + method: Option, + #[pyarg(keyword_only, default = "None")] + headers: Option, + #[pyarg(keyword_only, default = "None")] + body: Option, + #[pyarg(keyword_only, default = "None")] + content_type: Option, +} - let response_format = - args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; - let method = args.get_optional_kwarg_with_type("method", vm.ctx.str_type(), vm)?; - let headers = args.get_optional_kwarg_with_type("headers", vm.ctx.dict_type(), vm)?; - let body = args.get_optional_kwarg("body"); - let content_type = args.get_optional_kwarg_with_type("content_type", vm.ctx.str_type(), vm)?; +fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyResult { + let FetchArgs { + response_format, + method, + headers, + body, + content_type, + } = args; let response_format = match response_format { - Some(s) => FetchResponseFormat::from_str(vm, &objstr::get_value(&s))?, + Some(s) => FetchResponseFormat::from_str(vm, s.as_str())?, None => FetchResponseFormat::Text, }; let mut opts = web_sys::RequestInit::new(); match method { - Some(s) => opts.method(&objstr::get_value(&s)), + Some(s) => opts.method(s.as_str()), None => opts.method("GET"), }; @@ -68,16 +78,15 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { opts.body(Some(&convert::py_to_js(vm, body))); } - let request = web_sys::Request::new_with_str_and_init(&objstr::get_value(url), &opts) + let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; if let Some(headers) = headers { let h = request.headers(); - let headers: PyDictRef = headers.downcast().unwrap(); for (key, value) in headers.get_key_value_pairs() { - let key = &vm.to_str(&key)?.value; - let value = &vm.to_str(&value)?.value; - h.set(key, value) + let key = vm.to_str(&key)?; + let value = vm.to_str(&value)?; + h.set(key.as_str(), value.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } } @@ -85,7 +94,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(content_type) = content_type { request .headers() - .set("Content-Type", &objstr::get_value(&content_type)) + .set("Content-Type", content_type.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } @@ -104,9 +113,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(func, Some(vm.ctx.function_type()))]); - +fn browser_request_animation_frame(func: PyFunctionRef, vm: &VirtualMachine) -> PyResult { use std::{cell::RefCell, rc::Rc}; // this basic setup for request_animation_frame taken from: @@ -115,18 +122,16 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR let f = Rc::new(RefCell::new(None)); let g = f.clone(); - let func = func.clone(); - - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); *g.borrow_mut() = Some(Closure::wrap(Box::new(move |time: f64| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid from inside of request_animation_frame"); let vm = &stored_vm.vm; let func = func.clone(); let args = vec![vm.ctx.new_float(time)]; - let _ = vm.invoke(func, args); + let _ = vm.invoke(func.into_object(), args); let closure = f.borrow_mut().take(); drop(closure); @@ -141,10 +146,8 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR Ok(vm.ctx.new_int(id)) } -fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(id, Some(vm.ctx.int_type()))]); - - let id = objint::get_value(id).to_i32().ok_or_else(|| { +fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult { + let id = id.as_bigint().to_i32().ok_or_else(|| { vm.new_exception( vm.ctx.exceptions.value_error.clone(), "Integer too large to convert to i32 for animationFrame id".into(), @@ -186,14 +189,14 @@ impl PyPromise { fn then( zelf: PyPromiseRef, - on_fulfill: PyRef, - on_reject: OptionalArg>, + on_fulfill: PyFunctionRef, + on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { - let stored_vm = &acc_vm + let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -217,16 +220,12 @@ impl PyPromise { PyPromise::from_future(ret_future).into_ref(vm) } - fn catch( - zelf: PyPromiseRef, - on_reject: PyRef, - vm: &VirtualMachine, - ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + fn catch(zelf: PyPromiseRef, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + let weak_vm = weak_vm(vm); let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { res.or_else(|err| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -303,41 +302,31 @@ impl Element { } } -fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_alert(message: PyStringRef, vm: &VirtualMachine) -> PyResult { window() - .alert_with_message(&objstr::get_value(message)) + .alert_with_message(message.as_str()) .expect("alert() not to fail"); Ok(vm.get_none()) } -fn browser_confirm(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_confirm(message: PyStringRef, vm: &VirtualMachine) -> PyResult { let result = window() - .confirm_with_message(&objstr::get_value(message)) + .confirm_with_message(message.as_str()) .expect("confirm() not to fail"); Ok(vm.new_bool(result)) } -fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(message, Some(vm.ctx.str_type()))], - optional = [(default, Some(vm.ctx.str_type()))] - ); - - let result = if let Some(default) = default { - window().prompt_with_message_and_default( - &objstr::get_value(message), - &objstr::get_value(default), - ) +fn browser_prompt( + message: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let result = if let OptionalArg::Present(default) = default { + window().prompt_with_message_and_default(message.as_str(), default.as_str()) } else { - window().prompt_with_message(&objstr::get_value(message)) + window().prompt_with_message(message.as_str()) }; let result = match result.expect("prompt() not to fail") { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 717c9a973..6fb527d76 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -8,7 +8,7 @@ use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; -use crate::vm_class::{AccessibleVM, WASMVirtualMachine}; +use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { macro_rules! map_exceptions { @@ -81,11 +81,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { return Err(err); } }; - let acc_vm = AccessibleVM::from(wasm_vm.clone()); - let stored_vm = acc_vm - .upgrade() - .expect("acc. VM to be invalid when WASM vm is valid"); - let vm = &stored_vm.vm; + let vm = &stored_vm_from_wasm(&wasm_vm).vm; let mut py_func_args = PyFuncArgs::default(); if let Some(ref args) = args { for arg in args.values() { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index d6f9ede0e..3f7bdef8a 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -43,8 +43,24 @@ impl StoredVirtualMachine { // probably gets compiled down to a normal-ish static varible, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { - static STORED_VMS: RefCell>> = - RefCell::default(); + static STORED_VMS: RefCell>> = RefCell::default(); +} + +pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { + STORED_VMS.with(|cell| { + cell.borrow() + .get(&wasm_vm.id) + .expect("VirtualMachine is not valid") + .clone() + }) +} +pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak { + let id = vm + .wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set"); + STORED_VMS + .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } #[wasm_bindgen(js_name = vmStore)] @@ -104,44 +120,6 @@ impl VMStore { } } -#[derive(Clone)] -pub(crate) struct AccessibleVM { - weak: Weak, - id: String, -} - -impl AccessibleVM { - pub fn from_id(id: String) -> AccessibleVM { - let weak = STORED_VMS - .with(|cell| Rc::downgrade(cell.borrow().get(&id).expect("WASM VM to be valid"))); - AccessibleVM { weak, id } - } - - pub fn upgrade(&self) -> Option> { - self.weak.upgrade() - } -} - -impl From for AccessibleVM { - fn from(vm: WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id) - } -} -impl From<&WASMVirtualMachine> for AccessibleVM { - fn from(vm: &WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id.clone()) - } -} -impl From<&VirtualMachine> for AccessibleVM { - fn from(vm: &VirtualMachine) -> AccessibleVM { - AccessibleVM::from_id( - vm.wasm_id - .clone() - .expect("VM passed to from::() to have wasm_id be Some()"), - ) - } -} - #[wasm_bindgen(js_name = VirtualMachine)] #[derive(Clone)] pub struct WASMVirtualMachine { From 3e42edd2613791cd91cdc262d7573640bac33970 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 08:51:44 +0100 Subject: [PATCH 133/658] dict.__new__ - support for dict subtypes. --- tests/snippets/dict.py | 7 +++++++ vm/src/obj/objdict.rs | 16 ++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 2d369465c..d2aa9ed5b 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -57,6 +57,13 @@ assert x.get("not here", "default") == "default" assert x.get("here", "default") == "here" assert x.get("not here") == None +class LengthDict(dict): + pass + +x = LengthDict() +assert type(x) == LengthDict + + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 541645d5f..6388aa9ab 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -40,17 +40,18 @@ impl PyValue for PyDict { // Python dict methods: impl PyDictRef { fn new( - _class: PyClassRef, // TODO Support subclasses of int. + class: PyClassRef, dict_obj: OptionalArg, kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult { - let dict = vm.ctx.new_dict(); + let mut dict = DictContentType::default(); + if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { for (key, value) in dict_obj.get_key_value_pairs() { - dict.set_item(key, value, vm); + dict.insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -68,14 +69,17 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(key, value, vm); + dict.insert(vm, &key, value)?; } } } for (key, value) in kwargs.into_iter() { - dict.set_item(vm.new_str(key), value, vm); + dict.insert(vm, &vm.new_str(key), value)?; } - Ok(dict) + PyDict { + entries: RefCell::new(dict), + } + .into_ref_with_type(vm, class) } fn bool(self, _vm: &VirtualMachine) -> bool { From ee9066a713e0b81802e718b79a485eeee1bf06b3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:02:49 +0100 Subject: [PATCH 134/658] dict.get shouldn't call into __getitem__ --- tests/snippets/dict.py | 6 ++++-- vm/src/dictdatatype.rs | 7 +++---- vm/src/obj/objdict.rs | 22 ++++++++++++++-------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index d2aa9ed5b..e28fd9da4 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -58,11 +58,13 @@ assert x.get("here", "default") == "here" assert x.get("not here") == None class LengthDict(dict): - pass + def __getitem__(self, k): + return len(k) x = LengthDict() assert type(x) == LengthDict - +assert x['word'] == 4 +assert x.get('word') is None # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index ac1df994a..65ad55110 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -76,16 +76,15 @@ impl Dict { } /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.entries[index] { - Ok(entry.value.clone()) + Ok(Some(entry.value.clone())) } else { panic!("Lookup returned invalid index into entries!"); } } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + Ok(None) } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6388aa9ab..0e1a2aedd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -179,7 +179,13 @@ impl PyDictRef { } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.entries.borrow().get(vm, &key) + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => { + let key_repr = vm.to_pystr(&key)?; + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + } + } } fn get( @@ -187,12 +193,12 @@ impl PyDictRef { key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine, - ) -> PyObjectRef { - match self.into_object().get_item(key, vm) { - Ok(value) => value, - Err(_) => match default { - OptionalArg::Present(value) => value, - OptionalArg::Missing => vm.ctx.none(), + ) -> PyResult { + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => match default { + OptionalArg::Present(value) => Ok(value), + OptionalArg::Missing => Ok(vm.ctx.none()), }, } } @@ -220,7 +226,7 @@ impl PyDictRef { impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow().get(vm, &key).ok() + self.entries.borrow().get(vm, &key).unwrap() } // Item set/get: From 7b2d92f495e037fd85c19398ee2ba37b42a79cae Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:41:46 +0100 Subject: [PATCH 135/658] Delete DictProtocol. impl ItemProtocol for PyDictRef. --- tests/snippets/dict.py | 2 ++ vm/src/builtins.rs | 4 ++-- vm/src/frame.rs | 18 +++++++++--------- vm/src/import.rs | 4 ++-- vm/src/obj/objdict.rs | 34 +++++++++++++++++++--------------- vm/src/obj/objobject.rs | 18 +++++++++++------- vm/src/obj/objsuper.rs | 4 ++-- vm/src/pyobject.rs | 22 ++++++++++++++++------ vm/src/stdlib/json.rs | 6 ++---- vm/src/sysmodule.rs | 6 +++--- vm/src/vm.rs | 14 +++++++------- wasm/lib/src/convert.rs | 5 +++-- 12 files changed, 78 insertions(+), 59 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index e28fd9da4..c04165f16 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -66,6 +66,8 @@ assert type(x) == LengthDict assert x['word'] == 4 assert x.get('word') is None +assert 5 == eval("a + word", LengthDict()) + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 5aa5ee226..210d12b49 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -20,7 +20,7 @@ use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -804,6 +804,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu "__call__", vec![name_arg, bases, namespace.into_object()], )?; - cells.set_item("__class__", class.clone(), vm); + cells.set_item("__class__", class.clone(), vm)?; Ok(class) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2f525afdd..ca3e54d07 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -21,8 +21,8 @@ use crate::obj::objstr; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; use itertools::Itertools; @@ -133,12 +133,12 @@ pub trait NameProtocol { impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name, vm) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } - if let Some(value) = self.globals.get_item(name, vm) { + if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { return Some(value); } @@ -147,7 +147,7 @@ impl NameProtocol for Scope { fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name, vm) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } @@ -155,11 +155,11 @@ impl NameProtocol for Scope { } fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(key, value, vm) + self.get_locals().set_item(key, value, vm).unwrap(); } fn delete_name(&self, vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key, vm) + self.get_locals().del_item(key, vm).unwrap(); } } @@ -394,12 +394,12 @@ impl Frame { obj.downcast().expect("Need a dictionary to build a map."); let dict_elements = dict.get_key_value_pairs(); for (key, value) in dict_elements.iter() { - map_obj.set_item(key.clone(), value.clone(), vm); + map_obj.set_item(key.clone(), value.clone(), vm).unwrap(); } } } else { for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { - map_obj.set_item(key, value, vm) + map_obj.set_item(key, value, vm).unwrap(); } } diff --git a/vm/src/import.rs b/vm/src/import.rs index 713393deb..14c0a7566 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use crate::compile; use crate::frame::Scope; use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, ItemProtocol, PyResult}; +use crate::pyobject::{ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; @@ -39,7 +39,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item("__name__", vm.new_str(module.to_string()), vm); + attrs.set_item("__name__", vm.new_str(module.to_string()), vm)?; vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 0e1a2aedd..02f8443b5 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,8 +3,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, - PyResult, PyValue, + IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -174,8 +173,8 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(key, value, vm) + fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().insert(vm, &key, value) } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -223,21 +222,26 @@ impl PyDictRef { } } -impl DictProtocol for PyDictRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow().get(vm, &key).unwrap() +impl ItemProtocol for PyDictRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.as_object(), "__getitem__", key.into_pyobject(vm)?) } - // Item set/get: - fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow_mut().insert(vm, &key, value).unwrap() + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method( + self.as_object(), + "__setitem__", + vec![key.into_pyobject(vm)?, value], + ) } - fn del_item(&self, key: T, vm: &VirtualMachine) { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow_mut().delete(vm, &key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.as_object(), "__delitem__", key.into_pyobject(vm)?) } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c03ff31a1..a3bc41103 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -6,7 +6,7 @@ use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, + IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -77,7 +77,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.set_item(attr_name, value, vm); + dict.set_item(attr_name, value, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -98,7 +98,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.del_item(attr_name, vm); + dict.del_item(attr_name, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -208,7 +208,7 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } - if let Some(obj_attr) = object_getattr(&obj, &name, &vm) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm)? { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { vm.call_get_descriptor(attr, obj) @@ -219,11 +219,15 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } -fn object_getattr(obj: &PyObjectRef, attr_name: &str, vm: &VirtualMachine) -> Option { +fn object_getattr( + obj: &PyObjectRef, + attr_name: &str, + vm: &VirtualMachine, +) -> PyResult> { if let Some(ref dict) = obj.dict { - dict.get_item(attr_name, vm) + dict.get_item_option(attr_name, vm) } else { - None + Ok(None) } } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 19c7c2ea8..94fe0fc7b 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -12,7 +12,7 @@ use crate::obj::objfunction::PyMethod; use crate::obj::objstr; use crate::obj::objtype::{PyClass, PyClassRef}; use crate::pyobject::{ - DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -124,7 +124,7 @@ fn super_new( } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg, vm) { + match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { return Err(vm diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index bf5cf3c86..ee765f987 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -912,12 +912,6 @@ impl TypeProtocol for PyRef { } } -pub trait DictProtocol { - fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; - fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); - fn del_item(&self, key: T, vm: &VirtualMachine); -} - pub trait ItemProtocol { fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( @@ -927,6 +921,22 @@ pub trait ItemProtocol { vm: &VirtualMachine, ) -> PyResult; fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item_option( + &self, + key: T, + vm: &VirtualMachine, + ) -> PyResult> { + match self.get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) => { + if objtype::isinstance(&exc, &vm.ctx.exceptions.key_error) { + Ok(None) + } else { + Err(exc) + } + } + } + } } impl ItemProtocol for PyObjectRef { diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index b7cb704b1..f4061bc2a 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -13,9 +13,7 @@ use crate::obj::{ objstr::{self, PyString}, objtype, }; -use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol, -}; +use crate::pyobject::{create_type, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -178,7 +176,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { Some(PyString { ref value }) => value.clone(), _ => unimplemented!("map keys must be strings"), }; - dict.set_item(&key, value, self.vm); + dict.set_item(&key, value, self.vm).unwrap(); } Ok(dict.into_object()) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 772dfe198..2f8690448 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -139,6 +139,6 @@ settrace() -- set the global debug tracing function "modules" => modules.clone(), }); - modules.set_item("sys", module.clone(), vm); - modules.set_item("builtins", builtins.clone(), vm); + modules.set_item("sys", module.clone(), vm).unwrap(); + modules.set_item("builtins", builtins.clone(), vm).unwrap(); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 0bd255574..90daaa668 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -30,7 +30,7 @@ use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::stdlib; @@ -423,7 +423,7 @@ impl VirtualMachine { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &args.args[i]; - locals.set_item(arg_name, arg.clone(), self); + locals.set_item(arg_name, arg.clone(), self)?; } // Pack other positional arguments in to *args: @@ -436,7 +436,7 @@ impl VirtualMachine { } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(vararg_name, vararg_value, self); + locals.set_item(vararg_name, vararg_value, self)?; } bytecode::Varargs::Unnamed => { // just ignore the rest of the args @@ -456,7 +456,7 @@ impl VirtualMachine { let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { let d = self.ctx.new_dict(); - locals.set_item(kwargs_name, d.as_object().clone(), self); + locals.set_item(kwargs_name, d.as_object().clone(), self)?; Some(d) } bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), @@ -474,9 +474,9 @@ impl VirtualMachine { ); } - locals.set_item(&name, value, self); + locals.set_item(&name, value, self)?; } else if let Some(d) = &kwargs { - d.set_item(&name, value, self); + d.set_item(&name, value, self)?; } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -520,7 +520,7 @@ impl VirtualMachine { for (default_index, i) in (required_args..nexpected_args).enumerate() { let arg_name = &code_object.arg_names[i]; if !locals.contains_key(arg_name, self) { - locals.set_item(arg_name, available_defaults[default_index].clone(), self); + locals.set_item(arg_name, available_defaults[default_index].clone(), self)?; } } }; diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 717c9a973..25024734c 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -4,7 +4,7 @@ use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbytes, objint, objsequence, objtype}; -use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; @@ -192,7 +192,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm) + .unwrap(); } dict.into_object() } From 0fe3f7748cf5a8a01097a70ee2db9292fd033a18 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:46:17 +0100 Subject: [PATCH 136/658] Simplify PyDictRef.__repr__. --- vm/src/obj/objdict.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 02f8443b5..7cefce9af 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -89,7 +89,7 @@ impl PyDictRef { self.entries.borrow().len() } - fn repr(self, vm: &VirtualMachine) -> PyResult { + fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; for (key, value) in self.get_key_value_pairs() { @@ -102,7 +102,7 @@ impl PyDictRef { } else { "{...}".to_string() }; - Ok(vm.new_str(s)) + Ok(s) } fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { From e2a4f22be87cdd43c00eed9f134379c9a3d3f21e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:49:32 +0100 Subject: [PATCH 137/658] Avoid name similarity between ItemProtocol and inner methods. --- vm/src/obj/objdict.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7cefce9af..17910db23 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -109,7 +109,7 @@ impl PyDictRef { self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn inner_delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.entries.borrow_mut().delete(vm, &key) } @@ -173,11 +173,16 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn inner_setitem( + self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { self.entries.borrow_mut().insert(vm, &key, value) } - fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self.entries.borrow().get(vm, &key)? { Some(value) => Ok(value), None => { @@ -250,12 +255,12 @@ pub fn init(context: &PyContext) { "__bool__" => context.new_rustfunc(PyDictRef::bool), "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), - "__delitem__" => context.new_rustfunc(PyDictRef::delitem), - "__getitem__" => context.new_rustfunc(PyDictRef::getitem), + "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), - "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__setitem__" => context.new_rustfunc(PyDictRef::inner_setitem), "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), From b1dd5836af08bd550cf85005d0c4eaccc1e0cdcd Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:52:18 +0100 Subject: [PATCH 138/658] Simpler implementation of ItemProtocol for PyDictRef. --- vm/src/obj/objdict.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 17910db23..3b7a33666 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -229,7 +229,7 @@ impl PyDictRef { impl ItemProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.as_object(), "__getitem__", key.into_pyobject(vm)?) + self.as_object().get_item(key, vm) } fn set_item( @@ -238,15 +238,11 @@ impl ItemProtocol for PyDictRef { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - vm.call_method( - self.as_object(), - "__setitem__", - vec![key.into_pyobject(vm)?, value], - ) + self.as_object().set_item(key, value, vm) } fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.as_object(), "__delitem__", key.into_pyobject(vm)?) + self.as_object().del_item(key, vm) } } From 584b707356ba6aa614ab3d7d2de02feb280c7f79 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 10:04:18 +0100 Subject: [PATCH 139/658] Add dict.__missing__ support. --- tests/snippets/dict.py | 10 ++++++++++ vm/src/obj/objdict.rs | 14 ++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index c04165f16..69040d624 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -68,6 +68,16 @@ assert x.get('word') is None assert 5 == eval("a + word", LengthDict()) + +class Squares(dict): + def __missing__(self, k): + v = k * k + self[k] = v + return v + +x = Squares() +assert x[-5] == 25 + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 3b7a33666..d54f62bb7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -183,13 +183,15 @@ impl PyDictRef { } fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match self.entries.borrow().get(vm, &key)? { - Some(value) => Ok(value), - None => { - let key_repr = vm.to_pystr(&key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) - } + if let Some(value) = self.entries.borrow().get(vm, &key)? { + return Ok(value); } + + if let Ok(method) = vm.get_method(self.clone().into_object(), "__missing__") { + return vm.invoke(method, vec![key]); + } + + Err(vm.new_key_error(format!("Key not found: {}", vm.to_pystr(&key)?))) } fn get( From afb28bfccba4e4b07f5c43b6100d330b18fdc430 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sat, 6 Apr 2019 11:53:59 +0200 Subject: [PATCH 140/658] Fix moment when docstring is set. --- tests/snippets/function.py | 14 ++++++++++++++ vm/src/compile.rs | 31 +++++++++++++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index aafc9b53c..e4d0ab544 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -41,3 +41,17 @@ def f4(): pass assert f4.__doc__ == "test4" + + +def revdocstr(f): + d = f.__doc__ + d = d + 'w00t' + f.__doc__ = d + return f + +@revdocstr +def f5(): + """abc""" + +assert f5.__doc__ == 'abcw00t', f5.__doc__ + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index d7f774b83..d0ea96d32 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -650,25 +650,13 @@ impl Compiler { // Turn code object into function object: self.emit(Instruction::MakeFunction { flags }); + self.store_docstring(doc_str); self.apply_decorators(decorator_list); self.emit(Instruction::StoreName { name: name.to_string(), }); - if let Some(doc_string) = doc_str { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: doc_string.to_string(), - }, - }); - self.emit(Instruction::LoadName { - name: name.to_string(), - }); - self.emit(Instruction::StoreAttr { - name: "__doc__".to_string(), - }); - } self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -761,26 +749,33 @@ impl Compiler { }); } + self.store_docstring(doc_str); self.apply_decorators(decorator_list); self.emit(Instruction::StoreName { name: name.to_string(), }); + self.in_loop = was_in_loop; + Ok(()) + } + + fn store_docstring(&mut self, doc_str: Option) { if let Some(doc_string) = doc_str { + // Duplicate top of stack (the function or class object) + self.emit(Instruction::Duplicate); + + // Doc string value: self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: doc_string.to_string(), }, }); - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + + self.emit(Instruction::Rotate { amount: 2 }); self.emit(Instruction::StoreAttr { name: "__doc__".to_string(), }); } - self.in_loop = was_in_loop; - Ok(()) } fn compile_for( From 1f0fd39722e63468884ee4b89947b97c89d40657 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 17:29:40 +0300 Subject: [PATCH 141/658] Change os new funcs to new arg style --- vm/src/obj/objbytes.rs | 2 +- vm/src/stdlib/os.rs | 38 +++++++++++--------------------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f24c9273c..4a8b1ad5e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -17,7 +17,7 @@ use super::objtype::PyClassRef; pub struct PyBytes { value: Vec, } -type PyBytesRef = PyRef; +pub type PyBytesRef = PyRef; impl PyBytes { pub fn new(data: Vec) -> Self { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cd4c0709b..ae487598a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -6,9 +6,11 @@ use std::io::{ErrorKind, Read, Write}; use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; -use crate::obj::objbytes; +use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; +use crate::obj::objint::PyIntRef; use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -115,15 +117,9 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(fd, Some(vm.ctx.int_type())), (n, Some(vm.ctx.int_type()))] - ); - - let mut buffer = vec![0u8; objint::get_value(n).to_usize().unwrap()]; - let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); +fn os_read(fd: PyIntRef, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; n.as_bigint().to_usize().unwrap()]; + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); match file.read_exact(&mut buffer) { Ok(_) => (), Err(s) => return Err(vm.new_os_error(s.to_string())), @@ -134,18 +130,9 @@ fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(buffer)) } -fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (fd, Some(vm.ctx.int_type())), - (data, Some(vm.ctx.bytes_type())) - ] - ); - - let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); - let written = match file.write(&objbytes::get_value(&data)) { +fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + let written = match file.write(&data) { Ok(written) => written, Err(s) => return Err(vm.new_os_error(s.to_string())), }; @@ -155,11 +142,8 @@ fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(path, Some(vm.ctx.str_type()))]); - let fname = objstr::get_value(&path); - - match fs::remove_file(fname) { +fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::remove_file(&path.value) { Ok(_) => (), Err(s) => return Err(vm.new_os_error(s.to_string())), } From 04acfe266523139f793e31e9cf148ab9374d27fe Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 17:29:52 +0300 Subject: [PATCH 142/658] Remove failing line in windows --- tests/snippets/stdlib_os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 29c8005dc..ba6033ff8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -5,7 +5,6 @@ from testutils import assert_raises fd = os.open('README.md', 0) assert fd > 0 -assert_raises(OSError, lambda: os.read(fd + 1, 10)) os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) From e67d5f3c627f8701f07dc55e5136f6c4389d7c9e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 15:27:46 +0100 Subject: [PATCH 143/658] Parameterise dictdatatype::Dict by value type. --- vm/src/dictdatatype.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 65ad55110..6641a1884 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,20 +9,29 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; -#[derive(Default)] -pub struct Dict { +pub struct Dict { size: usize, indices: HashMap, - entries: Vec>, + entries: Vec>>, } -struct DictEntry { +impl Default for Dict { + fn default() -> Self { + Dict { + size: 0, + indices: HashMap::new(), + entries: Vec::new(), + } + } +} + +struct DictEntry { hash: usize, key: PyObjectRef, - value: PyObjectRef, + value: T, } -impl Dict { +impl Dict { pub fn new() -> Self { Dict { size: 0, @@ -32,12 +41,7 @@ impl Dict { } /// Store a key - pub fn insert( - &mut self, - vm: &VirtualMachine, - key: &PyObjectRef, - value: PyObjectRef, - ) -> PyResult<()> { + pub fn insert(&mut self, vm: &VirtualMachine, key: &PyObjectRef, value: T) -> PyResult<()> { match self.lookup(vm, key)? { LookupResult::Existing(index) => { // Update existing key @@ -76,7 +80,7 @@ impl Dict { } /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.entries[index] { Ok(Some(entry.value.clone())) @@ -114,7 +118,7 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { self.entries .iter() .filter(|e| e.is_some()) From eb9378fe900db7a859f0d71ab99bf8ce9071e932 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:20:18 +0300 Subject: [PATCH 144/658] Add os.mkdir --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index ae487598a..312b79549 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -151,6 +151,15 @@ fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::create_dir(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -168,6 +177,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "write" => ctx.new_rustfunc(os_write), "remove" => ctx.new_rustfunc(os_remove), "unlink" => ctx.new_rustfunc(os_remove), + "mkdir" => ctx.new_rustfunc(os_mkdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 2ec6afeb11a4a400584c000b6606dbd0640f09a0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:24:06 +0300 Subject: [PATCH 145/658] Add os.mkdirs --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 312b79549..fc7651027 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -160,6 +160,15 @@ fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::create_dir_all(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -178,6 +187,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "remove" => ctx.new_rustfunc(os_remove), "unlink" => ctx.new_rustfunc(os_remove), "mkdir" => ctx.new_rustfunc(os_mkdir), + "mkdirs" => ctx.new_rustfunc(os_mkdirs), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From bf6b12266a9b3996badd01f2ef36aac1b35ca4b7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:30:40 +0300 Subject: [PATCH 146/658] Add os.rmdir --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index fc7651027..6ffb36ca5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -169,6 +169,15 @@ fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::remove_dir(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -188,6 +197,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "unlink" => ctx.new_rustfunc(os_remove), "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), + "rmdir" => ctx.new_rustfunc(os_rmdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From a21aa6eac9ca8016e13c54689c83120eb57b482c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 19:14:23 +0300 Subject: [PATCH 147/658] Add os.listdir --- vm/src/stdlib/os.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6ffb36ca5..f0f5c3dd5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -178,6 +178,21 @@ fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => { + let res: PyResult> = iter + .map(|entry| match entry { + Ok(path) => Ok(vm.ctx.new_str(path.file_name().into_string().unwrap())), + Err(s) => Err(vm.new_os_error(s.to_string())), + }) + .collect(); + Ok(vm.ctx.new_list(res?)) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -198,6 +213,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), + "listdir" => ctx.new_rustfunc(os_listdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 401ca08b659d137200a80caf3bc87f08c3dc2642 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 21:11:58 +0300 Subject: [PATCH 148/658] Use map_err --- vm/src/stdlib/os.rs | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f0f5c3dd5..25937cd5c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -142,40 +142,20 @@ fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::remove_file(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_file(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::create_dir(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::create_dir_all(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir_all(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::remove_dir(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { From 5fd3cf2bcda2858acb5e2ca01163cf7a7a66dee1 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 20:29:25 +1300 Subject: [PATCH 149/658] Implemented keyword only defaults --- tests/snippets/function_args.py | 36 +++++++++++---- vm/src/bytecode.rs | 1 + vm/src/compile.rs | 28 ++++++++++-- vm/src/frame.rs | 26 +++++++++-- vm/src/obj/objfunction.rs | 13 +++++- vm/src/pyobject.rs | 7 +-- vm/src/vm.rs | 79 +++++++++++++++------------------ 7 files changed, 127 insertions(+), 63 deletions(-) diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 884cb47df..fd164d113 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -1,21 +1,28 @@ +from testutils import assertRaises + + def sum(x, y): return x+y -# def total(a, b, c, d): -# return sum(sum(a,b), sum(c,d)) -# -# assert total(1,1,1,1) == 4 -# assert total(1,2,3,4) == 10 +def total(a, b, c, d): + return sum(sum(a,b), sum(c,d)) + + +assert total(1,1,1,1) == 4 +assert total(1,2,3,4) == 10 assert sum(1, 1) == 2 assert sum(1, 3) == 4 + def sum2y(x, y): return x+y*2 + assert sum2y(1, 1) == 3 assert sum2y(1, 3) == 7 + def va(a, b=2, *c, d, **e): assert a == 1 assert b == 22 @@ -23,21 +30,34 @@ def va(a, b=2, *c, d, **e): assert d == 1337 assert e['f'] == 42 + va(1, 22, 3, 4, d=1337, f=42) + def va2(*args, **kwargs): assert args == (5, 4) assert len(kwargs) == 0 + va2(5, 4) x = (5, 4) va2(*x) va2(5, *x[1:]) -# def va3(x, *, b=2): -# pass -# va3(1, 2, 3, b=10) + +def va3(x, *, a, b=2, c=9): + return x + b + c + + +assert va3(1, a=1, b=10) == 20 + +with assertRaises(TypeError): + va3(1, 2, 3, a=1, b=10) + +with assertRaises(TypeError): + va3(1, b=10) + x = {'f': 42, 'e': 1337} y = {'d': 1337} diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 2fe485b63..b24801861 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -31,6 +31,7 @@ pub struct CodeObject { bitflags! { pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; + const HAS_KW_ONLY_DEFAULTS = 0x02; const HAS_ANNOTATIONS = 0x04; } } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 3780595c2..dd045c3bf 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -422,8 +422,8 @@ impl Compiler { name: &str, args: &ast::Parameters, ) -> Result { - let have_kwargs = !args.defaults.is_empty(); - if have_kwargs { + let have_defaults = !args.defaults.is_empty(); + if have_defaults { // Construct a tuple: let size = args.defaults.len(); for element in &args.defaults { @@ -435,6 +435,25 @@ impl Compiler { }); } + let mut num_kw_only_defaults = 0; + for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) { + if let Some(default) = default { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: kw.arg.clone(), + }, + }); + self.compile_expression(default)?; + num_kw_only_defaults += 1; + } + } + if num_kw_only_defaults > 0 { + self.emit(Instruction::BuildMap { + size: num_kw_only_defaults, + unpack: false, + }); + } + let line_number = self.get_source_line_number(); self.code_object_stack.push(CodeObject::new( args.args.iter().map(|a| a.arg.clone()).collect(), @@ -447,9 +466,12 @@ impl Compiler { )); let mut flags = bytecode::FunctionOpArg::empty(); - if have_kwargs { + if have_defaults { flags |= bytecode::FunctionOpArg::HAS_DEFAULTS; } + if num_kw_only_defaults > 0 { + flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS; + } Ok(flags) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ca3e54d07..2570c0934 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,12 +12,13 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict::PyDictRef; +use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ @@ -578,16 +579,33 @@ impl Frame { vm.ctx.new_dict().into_object() }; + let kw_only_defaults = + if flags.contains(bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS) { + Some( + self.pop_value().downcast::().expect( + "Stack value for keyword only defaults expected to be a dict", + ), + ) + } else { + None + }; + let defaults = if flags.contains(bytecode::FunctionOpArg::HAS_DEFAULTS) { - self.pop_value() + Some( + self.pop_value() + .downcast::() + .expect("Stack value for defaults expected to be a tuple"), + ) } else { - vm.get_none() + None }; // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm.ctx.new_function(code_obj, scope, defaults); + let obj = vm + .ctx + .new_function(code_obj, scope, defaults, kw_only_defaults); vm.set_attr(&obj, "__annotations__", annotations)?; diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 410794696..e8c1d7df8 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,5 +1,7 @@ use crate::frame::Scope; use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; @@ -11,15 +13,22 @@ pub struct PyFunction { // TODO: these shouldn't be public pub code: PyCodeRef, pub scope: Scope, - pub defaults: PyObjectRef, + pub defaults: Option, + pub kw_only_defaults: Option, } impl PyFunction { - pub fn new(code: PyCodeRef, scope: Scope, defaults: PyObjectRef) -> Self { + pub fn new( + code: PyCodeRef, + scope: Scope, + defaults: Option, + kw_only_defaults: Option, + ) -> Self { PyFunction { code, scope, defaults, + kw_only_defaults, } } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index ee765f987..4d2c8353b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -48,7 +48,7 @@ use crate::obj::objslice; use crate::obj::objstaticmethod; use crate::obj::objstr; use crate::obj::objsuper; -use crate::obj::objtuple::{self, PyTuple}; +use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; use crate::obj::objweakref; use crate::obj::objzip; @@ -658,10 +658,11 @@ impl PyContext { &self, code_obj: PyCodeRef, scope: Scope, - defaults: PyObjectRef, + defaults: Option, + kw_only_defaults: Option, ) -> PyObjectRef { PyObject::new( - PyFunction::new(code_obj, scope, defaults), + PyFunction::new(code_obj, scope, defaults, kw_only_defaults), self.function_type(), Some(self.new_dict()), ) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 90daaa668..d43ee7bea 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -23,10 +23,9 @@ use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGeneratorRef; use crate::obj::objiter; -use crate::obj::objlist::PyList; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; -use crate::obj::objtuple::PyTuple; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ @@ -332,9 +331,10 @@ impl VirtualMachine { ref code, ref scope, ref defaults, + ref kw_only_defaults, }) = func_ref.payload() { - return self.invoke_python_function(code, scope, defaults, args); + return self.invoke_python_function(code, scope, defaults, kw_only_defaults, args); } if let Some(PyMethod { ref function, @@ -356,11 +356,18 @@ impl VirtualMachine { &self, code: &PyCodeRef, scope: &Scope, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, args: PyFuncArgs, ) -> PyResult { let scope = scope.child_scope(&self.ctx); - self.fill_locals_from_args(&code.code, &scope.get_locals(), args, defaults)?; + self.fill_locals_from_args( + &code.code, + &scope.get_locals(), + args, + defaults, + kw_only_defaults, + )?; // Construct frame: let frame = Frame::new(code.clone(), scope).into_ref(self); @@ -379,12 +386,7 @@ impl VirtualMachine { cells: PyDictRef, locals: PyDictRef, ) -> PyResult { - if let Some(PyFunction { - code, - scope, - defaults: _, - }) = &function.payload() - { + if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope .child_scope_with_locals(cells) .child_scope_with_locals(locals); @@ -402,7 +404,8 @@ impl VirtualMachine { code_object: &bytecode::CodeObject, locals: &PyDictRef, args: PyFuncArgs, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, ) -> PyResult<()> { let nargs = args.args.len(); let nexpected_args = code_object.arg_names.len(); @@ -438,10 +441,7 @@ impl VirtualMachine { locals.set_item(vararg_name, vararg_value, self)?; } - bytecode::Varargs::Unnamed => { - // just ignore the rest of the args - } - bytecode::Varargs::None => { + bytecode::Varargs::Unnamed | bytecode::Varargs::None => { // Check the number of positional arguments if nargs > nexpected_args { return Err(self.new_type_error(format!( @@ -487,19 +487,11 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let available_defaults = if defaults.is(&self.get_none()) { - vec![] - } else if let Some(list) = defaults.payload::() { - list.elements.borrow().clone() - } else if let Some(tuple) = defaults.payload::() { - tuple.elements.borrow().clone() - } else { - panic!("function defaults not tuple or None"); - }; + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.borrow().len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception - let required_args = nexpected_args - available_defaults.len(); + let required_args = nexpected_args - num_defaults_available; let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; @@ -514,31 +506,32 @@ impl VirtualMachine { missing ))); } - - // We have sufficient defaults, so iterate over the corresponding names and use - // the default if we don't already have a value - for (default_index, i) in (required_args..nexpected_args).enumerate() { - let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name, self) { - locals.set_item(arg_name, available_defaults[default_index].clone(), self)?; + if let Some(defaults) = defaults { + let defaults = defaults.elements.borrow(); + // We have sufficient defaults, so iterate over the corresponding names and use + // the default if we don't already have a value + for (default_index, i) in (required_args..nexpected_args).enumerate() { + let arg_name = &code_object.arg_names[i]; + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, defaults[default_index].clone(), self)?; + } } } }; // Check if kw only arguments are all present: - let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { if !locals.contains_key(arg_name, self) { - if kwdefs.contains_key(arg_name) { - // If not yet specified, take the default value - unimplemented!(); - } else { - // No default value and not specified. - return Err(self.new_type_error(format!( - "Missing required kw only argument: '{}'", - arg_name - ))); + if let Some(kw_only_defaults) = kw_only_defaults { + if let Some(default) = kw_only_defaults.get_item(arg_name, self) { + locals.set_item(arg_name, default, self); + continue; + } } + + // No default value and not specified. + return Err(self + .new_type_error(format!("Missing required kw only argument: '{}'", arg_name))); } } From d5a9d96d62126f0b2080cf2ce3a33bd822b2bf15 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 07:36:07 +1200 Subject: [PATCH 150/658] Expose function.__defaults__ and function.__kwdefaults__(only getters) --- tests/snippets/function_args.py | 10 ++++++++++ vm/src/obj/objfunction.rs | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index fd164d113..b9c5a1626 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -33,6 +33,9 @@ def va(a, b=2, *c, d, **e): va(1, 22, 3, 4, d=1337, f=42) +assert va.__defaults__ == (2,) +assert va.__kwdefaults__ is None + def va2(*args, **kwargs): assert args == (5, 4) @@ -59,6 +62,13 @@ with assertRaises(TypeError): va3(1, b=10) +assert va3.__defaults__ is None +kw_defaults = va3.__kwdefaults__ +# assert va3.__kwdefaults__ == {'b': 2, 'c': 9} +assert set(kw_defaults) == {'b', 'c'} +assert kw_defaults['b'] == 2 +assert kw_defaults['c'] == 9 + x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index e8c1d7df8..01e4b586f 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -43,6 +43,14 @@ impl PyFunctionRef { fn code(self, _vm: &VirtualMachine) -> PyCodeRef { self.code.clone() } + + fn defaults(self, _vm: &VirtualMachine) -> Option { + self.defaults.clone() + } + + fn kwdefaults(self, _vm: &VirtualMachine) -> Option { + self.kw_only_defaults.clone() + } } #[derive(Debug)] @@ -68,7 +76,9 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), - "__code__" => context.new_property(PyFunctionRef::code) + "__code__" => context.new_property(PyFunctionRef::code), + "__defaults__" => context.new_property(PyFunctionRef::defaults), + "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), }); let builtin_function_or_method_type = &context.builtin_function_or_method_type; From 51732feaed7e4f7196ecaac9ab1d28b3cdc23181 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 07:43:29 +1200 Subject: [PATCH 151/658] Use get_item_option from ItemProtocol instead of get_item --- vm/src/vm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index d43ee7bea..7be05023b 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -523,8 +523,8 @@ impl VirtualMachine { for arg_name in &code_object.kwonlyarg_names { if !locals.contains_key(arg_name, self) { if let Some(kw_only_defaults) = kw_only_defaults { - if let Some(default) = kw_only_defaults.get_item(arg_name, self) { - locals.set_item(arg_name, default, self); + if let Some(default) = kw_only_defaults.get_item_option(arg_name, self)? { + locals.set_item(arg_name, default, self)?; continue; } } From 10cbf2ae740c0fe7de2151a2e3daf88576c7a394 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 01:49:16 +0200 Subject: [PATCH 152/658] bytes._new stuff --- vm/src/obj/objbytes.rs | 83 +++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8f43cd05f..26de25010 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,3 +1,9 @@ +use crate::obj::objint::PyInt; +use crate::obj::objlist; +use crate::obj::objlist::PyList; +use crate::obj::objstr::PyString; +use crate::obj::objtuple::PyTuple; +use crate::obj::objtype; use std::cell::Cell; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -7,7 +13,8 @@ use num_traits::ToPrimitive; use crate::function::OptionalArg; use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -87,29 +94,79 @@ extend_class!(context, bytesiterator_type, { "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), });*/ //} + +fn load_byte( + elements: PyResult>, + vm: &VirtualMachine, +) -> Result, PyObjectRef> { + if let Ok(value) = elements { + let mut data_bytes = vec![]; + for elem in value.iter() { + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); + } + } + Ok(data_bytes) + } else { + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } +} + #[pyimpl(__inside_vm)] impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( cls: PyClassRef, val_option: OptionalArg, + enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); + if let OptionalArg::Present(enc) = enc_option { + if let OptionalArg::Present(eval) = val_option { + if objtype::isinstance(&eval, &vm.ctx.str_type()) + && objtype::isinstance(&enc, &vm.ctx.str_type()) + { + //return Ok(PyBytes::new(vec![1]).into_ref_with_type(vm, cls.clone())); + return Err(vm.new_type_error("Ok les 2 sont sstr".to_string())); + } else { + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; + } - PyBytes::new(value).into_ref_with_type(vm, cls) + let value = if let OptionalArg::Present(ival) = val_option { + println!("{:?}", ival); + match_class!(ival.clone(), + _i @ PyInt => { + let size = objint::get_value(&ival).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _l @ PyString=> load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _obj => {return Err(vm.new_type_error(format!( + "int() argument must be a string or a number, not " + )));} + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), + Err(err) => Err(err), + } } #[pymethod(name = "__repr__")] From be55562457fbd361e3478035cdaeccef6151b8dd Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 6 Apr 2019 12:56:09 -0700 Subject: [PATCH 153/658] Allow first argument to be a shared ref to a python value --- vm/src/function.rs | 27 ++++++++++++++++++++++++--- vm/src/obj/objfloat.rs | 14 ++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index aae4eefe8..305c4de5e 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use std::ops::RangeInclusive; use crate::obj::objtype::{isinstance, PyClassRef}; -use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use crate::pyobject::{ + IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use self::OptionalArg::*; @@ -403,6 +405,7 @@ tuple_from_py_func_args!(A, B); tuple_from_py_func_args!(A, B, C); tuple_from_py_func_args!(A, B, C, D); tuple_from_py_func_args!(A, B, C, D, E); +tuple_from_py_func_args!(A, B, C, D, E, F); /// A built-in Python function. pub type PyNativeFunc = Box PyResult + 'static>; @@ -440,17 +443,19 @@ impl IntoPyNativeFunc for PyNativeFunc { } } +pub struct OwnedParam(std::marker::PhantomData); +pub struct RefParam(std::marker::PhantomData); + // This is the "magic" that allows rust functions of varying signatures to // generate native python functions. // // Note that this could be done without a macro - it is simply to avoid repetition. macro_rules! into_py_native_func_tuple { ($(($n:tt, $T:ident)),*) => { - impl IntoPyNativeFunc<($($T,)*), R> for F + impl IntoPyNativeFunc<($(OwnedParam<$T>,)*), R> for F where F: Fn($($T,)* &VirtualMachine) -> R + 'static, $($T: FromArgs,)* - ($($T,)*): FromArgs, R: IntoPyObject, { fn into_func(self) -> PyNativeFunc { @@ -461,6 +466,22 @@ macro_rules! into_py_native_func_tuple { }) } } + + impl IntoPyNativeFunc<(RefParam, $(OwnedParam<$T>,)*), R> for F + where + F: Fn(&S, $($T,)* &VirtualMachine) -> R + 'static, + S: PyValue, + $($T: FromArgs,)* + R: IntoPyObject, + { + fn into_func(self) -> PyNativeFunc { + Box::new(move |vm, args| { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)* vm).into_pyobject(vm) + }) + } + } }; } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index ca6bad9d2..de7c465d3 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -34,6 +34,13 @@ impl From for PyFloat { } } +impl PyFloat { + fn is_integer(&self, _vm: &VirtualMachine) -> bool { + let v = self.value; + (v - v.round()).abs() < std::f64::EPSILON + } +} + pub type PyFloatRef = PyRef; impl PyFloatRef { @@ -311,11 +318,6 @@ impl PyFloatRef { } } - fn is_integer(self, _vm: &VirtualMachine) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON - } - fn real(self, _vm: &VirtualMachine) -> Self { self } @@ -385,7 +387,7 @@ pub fn init(context: &PyContext) { "__mul__" => context.new_rustfunc(PyFloatRef::mul), "__rmul__" => context.new_rustfunc(PyFloatRef::mul), "real" => context.new_property(PyFloatRef::real), - "is_integer" => context.new_rustfunc(PyFloatRef::is_integer), + "is_integer" => context.new_rustfunc(PyFloat::is_integer), "as_integer_ratio" => context.new_rustfunc(PyFloatRef::as_integer_ratio) }); } From c2716040d2419311837d9ef025e476b68f1310e0 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 6 Apr 2019 19:05:18 -0700 Subject: [PATCH 154/658] float: take self by shared ref --- vm/src/obj/objfloat.rs | 112 ++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index de7c465d3..796da71ae 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -35,16 +35,7 @@ impl From for PyFloat { } impl PyFloat { - fn is_integer(&self, _vm: &VirtualMachine) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON - } -} - -pub type PyFloatRef = PyRef; - -impl PyFloatRef { - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let value = self.value; let result = if objtype::isinstance(&other, &vm.ctx.float_type()) { let other = get_value(&other); @@ -63,7 +54,7 @@ impl PyFloatRef { vm.ctx.new_bool(result) } - fn lt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn lt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 < get_value(&i2)) @@ -75,7 +66,7 @@ impl PyFloatRef { } } - fn le(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn le(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 <= get_value(&i2)) @@ -87,7 +78,7 @@ impl PyFloatRef { } } - fn gt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn gt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 > get_value(&i2)) @@ -99,7 +90,7 @@ impl PyFloatRef { } } - fn ge(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ge(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 >= get_value(&i2)) @@ -111,11 +102,11 @@ impl PyFloatRef { } } - fn abs(self, _vm: &VirtualMachine) -> f64 { + fn abs(&self, _vm: &VirtualMachine) -> f64 { self.value.abs() } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1 + get_value(&other)) @@ -127,23 +118,23 @@ impl PyFloatRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { self.value != 0.0 } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.float_type()) || objtype::isinstance(&other, &vm.ctx.int_type()) { - let r1 = PyFloatRef::floordiv(self.clone(), other.clone(), vm)?; - let r2 = PyFloatRef::mod_(self, other, vm)?; + let r1 = self.floordiv(other.clone(), vm)?; + let r2 = self.mod_(other, vm)?; Ok(vm.ctx.new_tuple(vec![r1, r2])) } else { Ok(vm.ctx.not_implemented()) } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -202,7 +193,7 @@ impl PyFloatRef { PyFloat { value }.into_ref_with_type(vm, cls) } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -221,11 +212,11 @@ impl PyFloatRef { } } - fn neg(self, _vm: &VirtualMachine) -> f64 { + fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1.powf(get_value(&other))) @@ -237,7 +228,7 @@ impl PyFloatRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(v1 - get_value(&other))) @@ -250,7 +241,7 @@ impl PyFloatRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(get_value(&other) - v1)) @@ -263,11 +254,11 @@ impl PyFloatRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -286,7 +277,7 @@ impl PyFloatRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -305,7 +296,7 @@ impl PyFloatRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type) { Ok(vm.ctx.new_float(v1 * get_value(&other))) @@ -318,11 +309,16 @@ impl PyFloatRef { } } - fn real(self, _vm: &VirtualMachine) -> Self { - self + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf } - fn as_integer_ratio(self, vm: &VirtualMachine) -> PyResult { + fn is_integer(&self, _vm: &VirtualMachine) -> bool { + let v = self.value; + (v - v.round()).abs() < std::f64::EPSILON + } + + fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult { let value = self.value; if value.is_infinite() { return Err( @@ -340,6 +336,8 @@ impl PyFloatRef { } } +pub type PyFloatRef = PyRef; + // Retrieve inner float value: pub fn get_value(obj: &PyObjectRef) -> f64 { obj.payload::().unwrap().value @@ -363,31 +361,31 @@ pub fn init(context: &PyContext) { let float_doc = "Convert a string or number to a floating point number, if possible."; extend_class!(context, float_type, { - "__eq__" => context.new_rustfunc(PyFloatRef::eq), - "__lt__" => context.new_rustfunc(PyFloatRef::lt), - "__le__" => context.new_rustfunc(PyFloatRef::le), - "__gt__" => context.new_rustfunc(PyFloatRef::gt), - "__ge__" => context.new_rustfunc(PyFloatRef::ge), - "__abs__" => context.new_rustfunc(PyFloatRef::abs), - "__add__" => context.new_rustfunc(PyFloatRef::add), - "__radd__" => context.new_rustfunc(PyFloatRef::add), - "__bool__" => context.new_rustfunc(PyFloatRef::bool), - "__divmod__" => context.new_rustfunc(PyFloatRef::divmod), - "__floordiv__" => context.new_rustfunc(PyFloatRef::floordiv), - "__new__" => context.new_rustfunc(PyFloatRef::new_float), - "__mod__" => context.new_rustfunc(PyFloatRef::mod_), - "__neg__" => context.new_rustfunc(PyFloatRef::neg), - "__pow__" => context.new_rustfunc(PyFloatRef::pow), - "__sub__" => context.new_rustfunc(PyFloatRef::sub), - "__rsub__" => context.new_rustfunc(PyFloatRef::rsub), - "__repr__" => context.new_rustfunc(PyFloatRef::repr), + "__eq__" => context.new_rustfunc(PyFloat::eq), + "__lt__" => context.new_rustfunc(PyFloat::lt), + "__le__" => context.new_rustfunc(PyFloat::le), + "__gt__" => context.new_rustfunc(PyFloat::gt), + "__ge__" => context.new_rustfunc(PyFloat::ge), + "__abs__" => context.new_rustfunc(PyFloat::abs), + "__add__" => context.new_rustfunc(PyFloat::add), + "__radd__" => context.new_rustfunc(PyFloat::add), + "__bool__" => context.new_rustfunc(PyFloat::bool), + "__divmod__" => context.new_rustfunc(PyFloat::divmod), + "__floordiv__" => context.new_rustfunc(PyFloat::floordiv), + "__new__" => context.new_rustfunc(PyFloat::new_float), + "__mod__" => context.new_rustfunc(PyFloat::mod_), + "__neg__" => context.new_rustfunc(PyFloat::neg), + "__pow__" => context.new_rustfunc(PyFloat::pow), + "__sub__" => context.new_rustfunc(PyFloat::sub), + "__rsub__" => context.new_rustfunc(PyFloat::rsub), + "__repr__" => context.new_rustfunc(PyFloat::repr), "__doc__" => context.new_str(float_doc.to_string()), - "__truediv__" => context.new_rustfunc(PyFloatRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyFloatRef::rtruediv), - "__mul__" => context.new_rustfunc(PyFloatRef::mul), - "__rmul__" => context.new_rustfunc(PyFloatRef::mul), - "real" => context.new_property(PyFloatRef::real), + "__truediv__" => context.new_rustfunc(PyFloat::truediv), + "__rtruediv__" => context.new_rustfunc(PyFloat::rtruediv), + "__mul__" => context.new_rustfunc(PyFloat::mul), + "__rmul__" => context.new_rustfunc(PyFloat::mul), + "real" => context.new_property(PyFloat::real), "is_integer" => context.new_rustfunc(PyFloat::is_integer), - "as_integer_ratio" => context.new_rustfunc(PyFloatRef::as_integer_ratio) + "as_integer_ratio" => context.new_rustfunc(PyFloat::as_integer_ratio) }); } From 5afc5585648b8faf8661269a8bc7442a2adf0245 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 6 Apr 2019 23:47:01 -0500 Subject: [PATCH 155/658] Add id() method --- wasm/lib/src/vm_class.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 3f7bdef8a..c159f60e1 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -37,6 +37,13 @@ impl StoredVirtualMachine { held_objects: RefCell::new(Vec::new()), } } + + pub fn id(&self) -> &str { + self.vm + .wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") + } } // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! From 7ca3b77fcdeaf0349cab7071cee3ea1ca3f8783d Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 6 Apr 2019 23:55:30 -0500 Subject: [PATCH 156/658] Just a fn --- wasm/lib/src/vm_class.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c159f60e1..5efa7c849 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -37,13 +37,6 @@ impl StoredVirtualMachine { held_objects: RefCell::new(Vec::new()), } } - - pub fn id(&self) -> &str { - self.vm - .wasm_id - .as_ref() - .expect("VirtualMachine inside of WASM crate should have wasm_id set") - } } // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! @@ -53,6 +46,11 @@ thread_local! { static STORED_VMS: RefCell>> = RefCell::default(); } +pub fn get_vm_id(vm: &VirtualMachine) -> &str { + vm.wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") +} pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { STORED_VMS.with(|cell| { cell.borrow() @@ -62,10 +60,7 @@ pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc Weak { - let id = vm - .wasm_id - .as_ref() - .expect("VirtualMachine inside of WASM crate should have wasm_id set"); + let id = get_vm_id(vm); STORED_VMS .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } From 02a21cd07262ceadcefaa5be814aee5a57bdab03 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 09:46:45 +1200 Subject: [PATCH 157/658] Add builtin warning classes --- vm/src/builtins.rs | 13 +++++++++++++ vm/src/exceptions.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 210d12b49..27e024aaf 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -767,6 +767,19 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), + + // Warnings + "Warning" => ctx.exceptions.warning.clone(), + "BytesWarning" => ctx.exceptions.bytes_warning.clone(), + "UnicodeWarning" => ctx.exceptions.unicode_warning.clone(), + "DeprecationWarning" => ctx.exceptions.deprecation_warning.clone(), + "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.clone(), + "FutureWarning" => ctx.exceptions.future_warning.clone(), + "ImportWarning" => ctx.exceptions.import_warning.clone(), + "SyntaxWarning" => ctx.exceptions.syntax_warning.clone(), + "ResourceWarning" => ctx.exceptions.resource_warning.clone(), + "RuntimeWarning" => ctx.exceptions.runtime_warning.clone(), + "UserWarning" => ctx.exceptions.user_warning.clone(), }); } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index cec4b3125..d81764e25 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -103,6 +103,18 @@ pub struct ExceptionZoo { pub type_error: PyClassRef, pub value_error: PyClassRef, pub zero_division_error: PyClassRef, + + pub warning: PyClassRef, + pub bytes_warning: PyClassRef, + pub unicode_warning: PyClassRef, + pub deprecation_warning: PyClassRef, + pub pending_deprecation_warning: PyClassRef, + pub future_warning: PyClassRef, + pub import_warning: PyClassRef, + pub syntax_warning: PyClassRef, + pub resource_warning: PyClassRef, + pub runtime_warning: PyClassRef, + pub user_warning: PyClassRef, } impl ExceptionZoo { @@ -130,6 +142,19 @@ impl ExceptionZoo { let file_not_found_error = create_type("FileNotFoundError", &type_type, &os_error); let permission_error = create_type("PermissionError", &type_type, &os_error); + let warning = create_type("Warning", &type_type, &exception_type); + let bytes_warning = create_type("BytesWarning", &type_type, &warning); + let unicode_warning = create_type("UnicodeWarning", &type_type, &warning); + let deprecation_warning = create_type("DeprecationWarning", &type_type, &warning); + let pending_deprecation_warning = + create_type("PendingDeprecationWarning", &type_type, &warning); + let future_warning = create_type("FutureWarning", &type_type, &warning); + let import_warning = create_type("ImportWarning", &type_type, &warning); + let syntax_warning = create_type("SyntaxWarning", &type_type, &warning); + let resource_warning = create_type("ResourceWarning", &type_type, &warning); + let runtime_warning = create_type("RuntimeWarning", &type_type, &warning); + let user_warning = create_type("UserWarning", &type_type, &warning); + ExceptionZoo { arithmetic_error, assertion_error, @@ -152,6 +177,17 @@ impl ExceptionZoo { type_error, value_error, zero_division_error, + warning, + bytes_warning, + unicode_warning, + deprecation_warning, + pending_deprecation_warning, + future_warning, + import_warning, + syntax_warning, + resource_warning, + runtime_warning, + user_warning, } } } From 32051c9d961dce2488c79541ba43b5ddd7e61598 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 19:19:39 +1200 Subject: [PATCH 158/658] Implement copy and update dict methods --- tests/snippets/dict.py | 16 ++++++++++++ vm/src/dictdatatype.rs | 9 +++++-- vm/src/obj/objdict.rs | 58 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 69040d624..f924bb0e7 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -107,3 +107,19 @@ x = {} x['a'] = 2 x['b'] = 10 assert list(x) == ['a', 'b'] + +y = x.copy() +x['c'] = 12 +assert list(y) == ['a', 'b'] +assert y['a'] == 2 +assert y['b'] == 10 + +y.update({'c': 19, "d": -1, 'b': 12}) +assert list(y) == ['a', 'b', 'c', 'd'] +assert y['a'] == 2 +assert y['b'] == 12 +assert y['c'] == 19 +assert y['d'] == -1 + +y.update(y) +assert list(y) == ['a', 'b', 'c', 'd'] diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6641a1884..8a6a8df1b 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,6 +9,7 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; +#[derive(Clone)] pub struct Dict { size: usize, indices: HashMap, @@ -25,6 +26,7 @@ impl Default for Dict { } } +#[derive(Clone)] struct DictEntry { hash: usize, key: PyObjectRef, @@ -118,13 +120,16 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { + pub fn iter_items(&self) -> impl Iterator + '_ { self.entries .iter() .filter(|e| e.is_some()) .map(|e| e.as_ref().unwrap()) .map(|e| (e.key.clone(), e.value.clone())) - .collect() + } + + pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { + self.iter_items().collect() } /// Lookup the index for the given key. diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index d54f62bb7..e84f7b178 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,7 +3,8 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, + PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -44,13 +45,28 @@ impl PyDictRef { kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult { - let mut dict = DictContentType::default(); + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + // it's unfortunate that we can't abstract over RefCall, as we should be able to use dict + // directly here, but that would require generic associated types + PyDictRef::merge(&entries, dict_obj, kwargs, vm)?; + + PyDict { entries }.into_ref_with_type(vm, class) + } + + fn merge( + dict: &RefCell, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - for (key, value) in dict_obj.get_key_value_pairs() { - dict.insert(vm, &key, value)?; + let mut dict_borrowed = dict.borrow_mut(); + for (key, value) in dict_obj.entries.borrow().iter_items() { + dict_borrowed.insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -68,17 +84,15 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.insert(vm, &key, value)?; + dict.borrow_mut().insert(vm, &key, value)?; } } } + let mut dict_borrowed = dict.borrow_mut(); for (key, value) in kwargs.into_iter() { - dict.insert(vm, &vm.new_str(key), value)?; + dict_borrowed.insert(vm, &vm.new_str(key), value)?; } - PyDict { - entries: RefCell::new(dict), - } - .into_ref_with_type(vm, class) + Ok(()) } fn bool(self, _vm: &VirtualMachine) -> bool { @@ -209,6 +223,28 @@ impl PyDictRef { } } + fn copy(self, _vm: &VirtualMachine) -> PyDict { + PyDict { + entries: self.entries.clone(), + } + } + + fn update( + self, + mut dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let OptionalArg::Present(ref other) = dict_obj { + if self.is(other) { + // Updating yourself is a noop, and this avoids a borrow error + dict_obj = OptionalArg::Missing; + } + } + + PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -266,5 +302,7 @@ pub fn init(context: &PyContext) { // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::iter), "get" => context.new_rustfunc(PyDictRef::get), + "copy" => context.new_rustfunc(PyDictRef::copy), + "update" => context.new_rustfunc(PyDictRef::update), }); } From 7e965c7cc8b331c11a339537161aee841b5a6ad0 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 10:12:57 +0200 Subject: [PATCH 159/658] check encoding in new --- vm/src/obj/objbytes.rs | 71 +++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 26de25010..1b06a9641 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -21,7 +21,9 @@ use crate::vm::VirtualMachine; use super::objbyteinner::PyByteInner; use super::objint; use super::objiter; +use super::objstr; use super::objtype::PyClassRef; +use std::clone::Clone; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ @@ -125,13 +127,24 @@ impl PyBytesRef { vm: &VirtualMachine, ) -> PyResult { // Create bytes data: + if let OptionalArg::Present(enc) = enc_option { if let OptionalArg::Present(eval) = val_option { - if objtype::isinstance(&eval, &vm.ctx.str_type()) - && objtype::isinstance(&enc, &vm.ctx.str_type()) - { - //return Ok(PyBytes::new(vec![1]).into_ref_with_type(vm, cls.clone())); - return Err(vm.new_type_error("Ok les 2 sont sstr".to_string())); + if let Ok(input) = eval.downcast::() { + if let Ok(encoding) = enc.downcast::() { + if encoding.value.to_lowercase() == "utf8".to_string() + || encoding.value.to_lowercase() == "utf-8".to_string() + { + return PyBytes::new(input.value.as_bytes().to_vec()) + .into_ref_with_type(vm, cls.clone()); + } else { + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error + ); + } + } else { + return Err(vm.new_type_error("encodin is not string".to_string())); + } } else { return Err(vm.new_type_error(format!( "bytes() argument 2 must be str, not {}", @@ -141,31 +154,31 @@ impl PyBytesRef { } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } - } - - let value = if let OptionalArg::Present(ival) = val_option { - println!("{:?}", ival); - match_class!(ival.clone(), - _i @ PyInt => { - let size = objint::get_value(&ival).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, - _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _l @ PyString=> load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _obj => {return Err(vm.new_type_error(format!( - "int() argument must be a string or a number, not " - )));} - ) } else { - Ok(vec![]) - }; - match value { - Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), - Err(err) => Err(err), + let value = if let OptionalArg::Present(ival) = val_option { + println!("{:?}", ival); + match_class!(ival.clone(), + _i @ PyInt => { + let size = objint::get_value(&ival).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _l @ PyString=> {return Err(vm.new_type_error(format!( + "string argument without an encoding" + )));}, + _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _obj => {} + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), + Err(err) => Err(err), + } } } From f03a164c146d15b3016c2a8e08d14f3df6b16d5f Mon Sep 17 00:00:00 2001 From: andrew Date: Sun, 7 Apr 2019 12:22:10 +0200 Subject: [PATCH 160/658] use a code generator for not implemented tests --- tests/README.md | 6 +- tests/generator/not_impl_footer.txt | 12 + tests/generator/not_impl_header.txt | 2 + tests/not_impl_gen.py | 33 + tests/snippets/whats_left_to_implement.py | 873 ---------------------- 5 files changed, 51 insertions(+), 875 deletions(-) create mode 100644 tests/generator/not_impl_footer.txt create mode 100644 tests/generator/not_impl_header.txt create mode 100644 tests/not_impl_gen.py delete mode 100644 tests/snippets/whats_left_to_implement.py diff --git a/tests/README.md b/tests/README.md index a225ba3b6..ba24ab8bc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,3 @@ - # Test snippets This directory contains two sets of test snippets which can be run in @@ -6,6 +5,10 @@ Python. The `snippets/` directory contains functional tests, and the `benchmarks/` directory contains snippets for use in benchmarking RustPython's performance. +## Generates the test for not implemented methods + +run using cpython not_impl_gen.py it automatically generate a +test snippet to check not yet implemented methods ## Running with CPython + RustPython @@ -15,4 +18,3 @@ compilation to bytecode. When this is done, run the bytecode with rustpython. ## Running with RustPython The other option is to run all snippets with RustPython. - diff --git a/tests/generator/not_impl_footer.txt b/tests/generator/not_impl_footer.txt new file mode 100644 index 000000000..5cd500999 --- /dev/null +++ b/tests/generator/not_impl_footer.txt @@ -0,0 +1,12 @@ +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] + +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: + print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_header.txt new file mode 100644 index 000000000..cd6f8f5a1 --- /dev/null +++ b/tests/generator/not_impl_header.txt @@ -0,0 +1,2 @@ +import platform + diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py new file mode 100644 index 000000000..2e20f5609 --- /dev/null +++ b/tests/not_impl_gen.py @@ -0,0 +1,33 @@ +objects = [ + bool, + bytearray, + bytes, + complex, + dict, + float, + frozenset, + int, + iter, + list, + memoryview, + range, + set, + str, + tuple, + object +] + +header = open("generator/not_impl_header.txt") +footer = open("generator/not_impl_footer.txt") +output = open("snippets/whats_left_to_implement.py", "w") + +output.write(header.read()) +output.write("expected_methods = {\n") + +for obj in objects: + output.write(f" '{obj.__name__}': ({obj.__name__}, [\n") + output.write("\n".join(f" '{attr}'," for attr in dir(obj))) + output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n")) + +output.write("}\n\n") +output.write(footer.read()) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py deleted file mode 100644 index 6abade9df..000000000 --- a/tests/snippets/whats_left_to_implement.py +++ /dev/null @@ -1,873 +0,0 @@ -import platform - -expected_methods = { - 'bool': (bool, [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', - ]), - 'bytearray': (bytearray, [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', - ]), - 'bytes': (bytes, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', - ]), - 'complex': (complex, [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', - ]), - 'dict': (dict, [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', - ]), - 'float': (float, [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', - ]), - 'frozenset': (frozenset, [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', - ]), - 'int': (int, [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', - ]), - 'iter': ([].__iter__(), [ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', - ]), - 'list': (list, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', - ]), - 'memoryview': (memoryview, [ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', - ]), - 'range': (range, [ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', - ]), - 'set': (set, [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', - ]), - 'str': (str, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' - ]), - 'tuple': (tuple, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - ]), - 'object': (object, [ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' - ]), -} - -not_implemented = [(name, method) - for name, (val, methods) in expected_methods.items() - for method in methods - if not hasattr(val, method)] - -for r in not_implemented: - print(r[0], ".", r[1], sep="") -if not not_implemented: - print("Not much \\o/") - -if platform.python_implementation() == "CPython": - assert len(not_implemented) == 0, "CPython should have all the methods" From 2eb8e7bf2b49ba578f0180588a5e8592d38c5fd0 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 18:55:35 +0200 Subject: [PATCH 161/658] new ok for pybytes --- tests/snippets/bytes.py | 18 ++++++++++++ vm/src/obj/objbytes.rs | 61 ++++++++++++++++++----------------------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 19ebef68a..1a4b43dfb 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -1,3 +1,21 @@ +from testutils import assertRaises + +# new +assert bytes([1,2,3]) +assert bytes((1,2,3)) +assert bytes(range(4)) +assert b'bla' +assert bytes(3) +assert bytes("bla", "utf8") +try: + bytes("bla") +except TypeError: + assert True +else: + assert False + +# + assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented assert b'foobar'.__gt__(2) == NotImplemented diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 1b06a9641..74795088f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -97,26 +97,6 @@ extend_class!(context, bytesiterator_type, { });*/ //} -fn load_byte( - elements: PyResult>, - vm: &VirtualMachine, -) -> Result, PyObjectRef> { - if let Ok(value) = elements { - let mut data_bytes = vec![]; - for elem in value.iter() { - let v = objint::to_int(vm, &elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); - } - } - Ok(data_bytes) - } else { - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) - } -} - #[pyimpl(__inside_vm)] impl PyBytesRef { #[pymethod(name = "__new__")] @@ -126,14 +106,14 @@ impl PyBytesRef { enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - // Create bytes data: - + // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = enc_option { if let OptionalArg::Present(eval) = val_option { if let Ok(input) = eval.downcast::() { - if let Ok(encoding) = enc.downcast::() { + if let Ok(encoding) = enc.clone().downcast::() { if encoding.value.to_lowercase() == "utf8".to_string() || encoding.value.to_lowercase() == "utf-8".to_string() + // TODO: different encoding { return PyBytes::new(input.value.as_bytes().to_vec()) .into_ref_with_type(vm, cls.clone()); @@ -143,23 +123,23 @@ impl PyBytesRef { ); } } else { - return Err(vm.new_type_error("encodin is not string".to_string())); + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); } } else { - return Err(vm.new_type_error(format!( - "bytes() argument 2 must be str, not {}", - enc.class().name - ))); + return Err(vm.new_type_error("encoding without a string argument".to_string())); } } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } + // On ly one argument } else { let value = if let OptionalArg::Present(ival) = val_option { - println!("{:?}", ival); match_class!(ival.clone(), - _i @ PyInt => { - let size = objint::get_value(&ival).to_usize().unwrap(); + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); let mut res: Vec = Vec::with_capacity(size); for _ in 0..size { res.push(0) @@ -168,9 +148,22 @@ impl PyBytesRef { _l @ PyString=> {return Err(vm.new_type_error(format!( "string argument without an encoding" )));}, - _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _obj => {} + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)));}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } ) } else { Ok(vec![]) From a0aa88d2fb3aba561e0a085a14f618f7008f1f92 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 20:02:27 +0300 Subject: [PATCH 162/658] clear return implicit None --- vm/src/obj/objset.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 6c67312fb..c3c580bc4 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -290,9 +290,8 @@ impl PySetInner { perform_action_with_hash(vm, &mut self.elements, &item, &discard) } - fn clear(&mut self, vm: &VirtualMachine) -> PyResult { + fn clear(&mut self) -> () { self.elements.clear(); - Ok(vm.get_none()) } fn pop(&mut self, vm: &VirtualMachine) -> PyResult { @@ -498,8 +497,8 @@ impl PySetRef { self.inner.borrow_mut().discard(&item, vm) } - fn clear(self, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().clear(vm) + fn clear(self, _vm: &VirtualMachine) -> () { + self.inner.borrow_mut().clear() } fn pop(self, vm: &VirtualMachine) -> PyResult { From aec33634ca1323b68fbbc8ae75deda857b0e10b2 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:27:03 -0700 Subject: [PATCH 163/658] int: take self by ref --- vm/src/obj/objint.rs | 198 +++++++++++++++++++++++++------------------ 1 file changed, 115 insertions(+), 83 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 3c0c3b707..706b4ecfa 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -95,12 +95,8 @@ impl_try_from_object_int!( (u64, to_u64), ); -impl PyIntRef { - fn pass_value(self, _vm: &VirtualMachine) -> Self { - self - } - - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +impl PyInt { + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) } else { @@ -108,7 +104,7 @@ impl PyIntRef { } } - fn ne(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) } else { @@ -116,7 +112,7 @@ impl PyIntRef { } } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) } else { @@ -124,7 +120,7 @@ impl PyIntRef { } } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) } else { @@ -132,7 +128,7 @@ impl PyIntRef { } } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) } else { @@ -140,7 +136,7 @@ impl PyIntRef { } } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) } else { @@ -148,7 +144,7 @@ impl PyIntRef { } } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) } else { @@ -156,7 +152,7 @@ impl PyIntRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) } else { @@ -164,7 +160,7 @@ impl PyIntRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) } else { @@ -172,7 +168,7 @@ impl PyIntRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) } else { @@ -180,7 +176,7 @@ impl PyIntRef { } } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) } else { @@ -188,7 +184,7 @@ impl PyIntRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) } else { @@ -196,7 +192,7 @@ impl PyIntRef { } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -209,7 +205,7 @@ impl PyIntRef { } } - fn lshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -228,7 +224,7 @@ impl PyIntRef { } } - fn rshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -247,7 +243,7 @@ impl PyIntRef { } } - fn xor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) } else { @@ -255,7 +251,7 @@ impl PyIntRef { } } - fn rxor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) ^ (&self.value)) } else { @@ -263,7 +259,7 @@ impl PyIntRef { } } - fn or(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) } else { @@ -271,7 +267,7 @@ impl PyIntRef { } } - fn and(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); vm.ctx.new_int((&self.value) & v2) @@ -280,7 +276,7 @@ impl PyIntRef { } } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); vm.ctx.new_int(self.value.pow(v2)) @@ -292,7 +288,7 @@ impl PyIntRef { } } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -305,7 +301,7 @@ impl PyIntRef { } } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -321,37 +317,65 @@ impl PyIntRef { } } - fn neg(self, _vm: &VirtualMachine) -> BigInt { + fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } - fn hash(self, _vm: &VirtualMachine) -> u64 { + fn hash(&self, _vm: &VirtualMachine) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() } - fn abs(self, _vm: &VirtualMachine) -> BigInt { + fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } - fn round(self, _precision: OptionalArg, _vm: &VirtualMachine) -> Self { - self + fn round( + zelf: PyRef, + _precision: OptionalArg, + _vm: &VirtualMachine, + ) -> PyIntRef { + zelf } - fn float(self, _vm: &VirtualMachine) -> f64 { - self.value.to_f64().unwrap() + fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf } - fn invert(self, _vm: &VirtualMachine) -> BigInt { + fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn format(self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { + fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -359,15 +383,23 @@ impl PyIntRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } - fn bit_length(self, _vm: &VirtualMachine) -> usize { + fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } - fn imag(self, _vm: &VirtualMachine) -> usize { + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } } @@ -484,49 +516,49 @@ Base 0 means to interpret the base from the string as an integer literal. let int_type = &context.int_type; extend_class!(context, int_type, { "__doc__" => context.new_str(int_doc.to_string()), - "__eq__" => context.new_rustfunc(PyIntRef::eq), - "__ne__" => context.new_rustfunc(PyIntRef::ne), - "__lt__" => context.new_rustfunc(PyIntRef::lt), - "__le__" => context.new_rustfunc(PyIntRef::le), - "__gt__" => context.new_rustfunc(PyIntRef::gt), - "__ge__" => context.new_rustfunc(PyIntRef::ge), - "__abs__" => context.new_rustfunc(PyIntRef::abs), - "__add__" => context.new_rustfunc(PyIntRef::add), - "__radd__" => context.new_rustfunc(PyIntRef::add), - "__and__" => context.new_rustfunc(PyIntRef::and), - "__divmod__" => context.new_rustfunc(PyIntRef::divmod), - "__float__" => context.new_rustfunc(PyIntRef::float), - "__round__" => context.new_rustfunc(PyIntRef::round), - "__ceil__" => context.new_rustfunc(PyIntRef::pass_value), - "__floor__" => context.new_rustfunc(PyIntRef::pass_value), - "__index__" => context.new_rustfunc(PyIntRef::pass_value), - "__trunc__" => context.new_rustfunc(PyIntRef::pass_value), - "__int__" => context.new_rustfunc(PyIntRef::pass_value), - "__floordiv__" => context.new_rustfunc(PyIntRef::floordiv), - "__hash__" => context.new_rustfunc(PyIntRef::hash), - "__lshift__" => context.new_rustfunc(PyIntRef::lshift), - "__rshift__" => context.new_rustfunc(PyIntRef::rshift), + "__eq__" => context.new_rustfunc(PyInt::eq), + "__ne__" => context.new_rustfunc(PyInt::ne), + "__lt__" => context.new_rustfunc(PyInt::lt), + "__le__" => context.new_rustfunc(PyInt::le), + "__gt__" => context.new_rustfunc(PyInt::gt), + "__ge__" => context.new_rustfunc(PyInt::ge), + "__abs__" => context.new_rustfunc(PyInt::abs), + "__add__" => context.new_rustfunc(PyInt::add), + "__radd__" => context.new_rustfunc(PyInt::add), + "__and__" => context.new_rustfunc(PyInt::and), + "__divmod__" => context.new_rustfunc(PyInt::divmod), + "__float__" => context.new_rustfunc(PyInt::float), + "__round__" => context.new_rustfunc(PyInt::round), + "__ceil__" => context.new_rustfunc(PyInt::ceil), + "__floor__" => context.new_rustfunc(PyInt::floor), + "__index__" => context.new_rustfunc(PyInt::index), + "__trunc__" => context.new_rustfunc(PyInt::trunc), + "__int__" => context.new_rustfunc(PyInt::int), + "__floordiv__" => context.new_rustfunc(PyInt::floordiv), + "__hash__" => context.new_rustfunc(PyInt::hash), + "__lshift__" => context.new_rustfunc(PyInt::lshift), + "__rshift__" => context.new_rustfunc(PyInt::rshift), "__new__" => context.new_rustfunc(int_new), - "__mod__" => context.new_rustfunc(PyIntRef::mod_), - "__mul__" => context.new_rustfunc(PyIntRef::mul), - "__rmul__" => context.new_rustfunc(PyIntRef::mul), - "__or__" => context.new_rustfunc(PyIntRef::or), - "__neg__" => context.new_rustfunc(PyIntRef::neg), - "__pos__" => context.new_rustfunc(PyIntRef::pass_value), - "__pow__" => context.new_rustfunc(PyIntRef::pow), - "__repr__" => context.new_rustfunc(PyIntRef::repr), - "__sub__" => context.new_rustfunc(PyIntRef::sub), - "__rsub__" => context.new_rustfunc(PyIntRef::rsub), - "__format__" => context.new_rustfunc(PyIntRef::format), - "__truediv__" => context.new_rustfunc(PyIntRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyIntRef::rtruediv), - "__xor__" => context.new_rustfunc(PyIntRef::xor), - "__rxor__" => context.new_rustfunc(PyIntRef::rxor), - "__bool__" => context.new_rustfunc(PyIntRef::bool), - "__invert__" => context.new_rustfunc(PyIntRef::invert), - "bit_length" => context.new_rustfunc(PyIntRef::bit_length), - "conjugate" => context.new_rustfunc(PyIntRef::pass_value), - "real" => context.new_property(PyIntRef::pass_value), - "imag" => context.new_property(PyIntRef::imag) + "__mod__" => context.new_rustfunc(PyInt::mod_), + "__mul__" => context.new_rustfunc(PyInt::mul), + "__rmul__" => context.new_rustfunc(PyInt::mul), + "__or__" => context.new_rustfunc(PyInt::or), + "__neg__" => context.new_rustfunc(PyInt::neg), + "__pos__" => context.new_rustfunc(PyInt::pos), + "__pow__" => context.new_rustfunc(PyInt::pow), + "__repr__" => context.new_rustfunc(PyInt::repr), + "__sub__" => context.new_rustfunc(PyInt::sub), + "__rsub__" => context.new_rustfunc(PyInt::rsub), + "__format__" => context.new_rustfunc(PyInt::format), + "__truediv__" => context.new_rustfunc(PyInt::truediv), + "__rtruediv__" => context.new_rustfunc(PyInt::rtruediv), + "__xor__" => context.new_rustfunc(PyInt::xor), + "__rxor__" => context.new_rustfunc(PyInt::rxor), + "__bool__" => context.new_rustfunc(PyInt::bool), + "__invert__" => context.new_rustfunc(PyInt::invert), + "bit_length" => context.new_rustfunc(PyInt::bit_length), + "conjugate" => context.new_rustfunc(PyInt::conjugate), + "real" => context.new_property(PyInt::real), + "imag" => context.new_property(PyInt::imag) }); } From e70f174eb6f2802f5fb935a82a8b387228ffd4ae Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:35:39 -0700 Subject: [PATCH 164/658] str: take self by ref --- vm/src/obj/objstr.rs | 117 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 62cc923b1..0e16a8b3b 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -67,7 +67,7 @@ impl TryIntoRef for &str { } #[pyimpl(__inside_vm)] -impl PyStringRef { +impl PyString { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') @@ -89,7 +89,7 @@ impl PyStringRef { } } #[pymethod(name = "__add__")] - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn add(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) } else { @@ -98,12 +98,12 @@ impl PyStringRef { } #[pymethod(name = "__bool__")] - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() } #[pymethod(name = "__eq__")] - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) } else { @@ -112,17 +112,17 @@ impl PyStringRef { } #[pymethod(name = "__contains__")] - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + fn contains(&self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } #[pymethod(name = "__getitem__")] - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } #[pymethod(name = "__gt__")] - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn gt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) } else { @@ -131,7 +131,7 @@ impl PyStringRef { } #[pymethod(name = "__ge__")] - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ge(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) } else { @@ -140,7 +140,7 @@ impl PyStringRef { } #[pymethod(name = "__lt__")] - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) } else { @@ -149,7 +149,7 @@ impl PyStringRef { } #[pymethod(name = "__le__")] - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn le(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) } else { @@ -158,19 +158,19 @@ impl PyStringRef { } #[pymethod(name = "__hash__")] - fn hash(self, _vm: &VirtualMachine) -> usize { + fn hash(&self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() as usize } #[pymethod(name = "__len__")] - fn len(self, _vm: &VirtualMachine) -> usize { + fn len(&self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } #[pymethod(name = "__mul__")] - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); @@ -185,12 +185,12 @@ impl PyStringRef { } #[pymethod(name = "__str__")] - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + fn str(zelf: PyRef, _vm: &VirtualMachine) -> PyStringRef { + zelf } #[pymethod(name = "__repr__")] - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' @@ -221,30 +221,30 @@ impl PyStringRef { } #[pymethod] - fn lower(self, _vm: &VirtualMachine) -> String { + fn lower(&self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower #[pymethod] - fn casefold(self, _vm: &VirtualMachine) -> String { + fn casefold(&self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } #[pymethod] - fn upper(self, _vm: &VirtualMachine) -> String { + fn upper(&self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } #[pymethod] - fn capitalize(self, _vm: &VirtualMachine) -> String { + fn capitalize(&self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } #[pymethod] fn split( - self, + &self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, @@ -266,7 +266,7 @@ impl PyStringRef { #[pymethod] fn rsplit( - self, + &self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, @@ -287,23 +287,23 @@ impl PyStringRef { } #[pymethod] - fn strip(self, _vm: &VirtualMachine) -> String { + fn strip(&self, _vm: &VirtualMachine) -> String { self.value.trim().to_string() } #[pymethod] - fn lstrip(self, _vm: &VirtualMachine) -> String { + fn lstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_start().to_string() } #[pymethod] - fn rstrip(self, _vm: &VirtualMachine) -> String { + fn rstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_end().to_string() } #[pymethod] fn endswith( - self, + &self, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -318,7 +318,7 @@ impl PyStringRef { #[pymethod] fn startswith( - self, + &self, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -332,17 +332,17 @@ impl PyStringRef { } #[pymethod] - fn isalnum(self, _vm: &VirtualMachine) -> bool { + fn isalnum(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } #[pymethod] - fn isnumeric(self, _vm: &VirtualMachine) -> bool { + fn isnumeric(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } #[pymethod] - fn isdigit(self, _vm: &VirtualMachine) -> bool { + fn isdigit(&self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, @@ -359,7 +359,7 @@ impl PyStringRef { } #[pymethod] - fn isdecimal(self, _vm: &VirtualMachine) -> bool { + fn isdecimal(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -397,12 +397,12 @@ impl PyStringRef { } #[pymethod] - fn title(self, _vm: &VirtualMachine) -> String { + fn title(&self, _vm: &VirtualMachine) -> String { make_title(&self.value) } #[pymethod] - fn swapcase(self, _vm: &VirtualMachine) -> String { + fn swapcase(&self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char @@ -418,13 +418,13 @@ impl PyStringRef { } #[pymethod] - fn isalpha(self, _vm: &VirtualMachine) -> bool { + fn isalpha(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } #[pymethod] fn replace( - self, + &self, old: PyStringRef, new: PyStringRef, num: OptionalArg, @@ -439,12 +439,12 @@ impl PyStringRef { // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower #[pymethod] - fn isspace(self, _vm: &VirtualMachine) -> bool { + fn isspace(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } #[pymethod] - fn isupper(self, _vm: &VirtualMachine) -> bool { + fn isupper(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -454,7 +454,7 @@ impl PyStringRef { } #[pymethod] - fn islower(self, _vm: &VirtualMachine) -> bool { + fn islower(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -464,13 +464,13 @@ impl PyStringRef { } #[pymethod] - fn isascii(self, _vm: &VirtualMachine) -> bool { + fn isascii(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet #[pymethod] - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + fn splitlines(&self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value .split('\n') @@ -480,7 +480,7 @@ impl PyStringRef { } #[pymethod] - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { @@ -496,7 +496,7 @@ impl PyStringRef { #[pymethod] fn find( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -515,7 +515,7 @@ impl PyStringRef { #[pymethod] fn rfind( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -534,7 +534,7 @@ impl PyStringRef { #[pymethod] fn index( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -553,7 +553,7 @@ impl PyStringRef { #[pymethod] fn rindex( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -571,7 +571,7 @@ impl PyStringRef { } #[pymethod] - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + fn partition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -590,7 +590,7 @@ impl PyStringRef { } #[pymethod] - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + fn rpartition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -610,7 +610,7 @@ impl PyStringRef { } #[pymethod] - fn istitle(self, _vm: &VirtualMachine) -> bool { + fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -620,7 +620,7 @@ impl PyStringRef { #[pymethod] fn count( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -635,7 +635,7 @@ impl PyStringRef { } #[pymethod] - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + fn zfill(&self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { value.to_string() @@ -644,7 +644,10 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + fn get_fill_char<'a>( + rep: &'a OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<&'a str> { let rep_str = match rep { OptionalArg::Present(ref st) => &st.value, OptionalArg::Missing => " ", @@ -660,7 +663,7 @@ impl PyStringRef { #[pymethod] fn ljust( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -672,7 +675,7 @@ impl PyStringRef { #[pymethod] fn rjust( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -684,7 +687,7 @@ impl PyStringRef { #[pymethod] fn center( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -702,7 +705,7 @@ impl PyStringRef { } #[pymethod] - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(&self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; @@ -726,7 +729,7 @@ impl PyStringRef { } #[pymethod] - fn isidentifier(self, _vm: &VirtualMachine) -> bool { + fn isidentifier(&self, _vm: &VirtualMachine) -> bool { let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) @@ -769,7 +772,7 @@ impl IntoPyObject for &String { } pub fn init(ctx: &PyContext) { - PyStringRef::extend_class(ctx, &ctx.str_type); + PyString::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { From ea61599da46f302079e633ee80ea8acb1016c6b0 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:41:52 -0700 Subject: [PATCH 165/658] range: take self by ref --- vm/src/obj/objrange.rs | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 71adb4118..2820dd9b4 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -101,20 +101,20 @@ pub fn init(context: &PyContext) { When step is given, it specifies the increment (or decrement)."; extend_class!(context, range_type, { - "__bool__" => context.new_rustfunc(PyRangeRef::bool), - "__contains__" => context.new_rustfunc(PyRangeRef::contains), + "__bool__" => context.new_rustfunc(PyRange::bool), + "__contains__" => context.new_rustfunc(PyRange::contains), "__doc__" => context.new_str(range_doc.to_string()), - "__getitem__" => context.new_rustfunc(PyRangeRef::getitem), - "__iter__" => context.new_rustfunc(PyRangeRef::iter), - "__len__" => context.new_rustfunc(PyRangeRef::len), + "__getitem__" => context.new_rustfunc(PyRange::getitem), + "__iter__" => context.new_rustfunc(PyRange::iter), + "__len__" => context.new_rustfunc(PyRange::len), "__new__" => context.new_rustfunc(range_new), - "__repr__" => context.new_rustfunc(PyRangeRef::repr), - "__reversed__" => context.new_rustfunc(PyRangeRef::reversed), - "count" => context.new_rustfunc(PyRangeRef::count), - "index" => context.new_rustfunc(PyRangeRef::index), - "start" => context.new_property(PyRangeRef::start), - "stop" => context.new_property(PyRangeRef::stop), - "step" => context.new_property(PyRangeRef::step), + "__repr__" => context.new_rustfunc(PyRange::repr), + "__reversed__" => context.new_rustfunc(PyRange::reversed), + "count" => context.new_rustfunc(PyRange::count), + "index" => context.new_rustfunc(PyRange::index), + "start" => context.new_property(PyRange::start), + "stop" => context.new_property(PyRange::stop), + "step" => context.new_property(PyRange::step), }); let rangeiterator_type = &context.rangeiterator_type; @@ -126,7 +126,7 @@ pub fn init(context: &PyContext) { type PyRangeRef = PyRef; -impl PyRangeRef { +impl PyRange { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { start: PyInt::new(BigInt::zero()).into_ref(vm), @@ -153,26 +153,26 @@ impl PyRangeRef { .into_ref_with_type(vm, cls) } - fn start(self, _vm: &VirtualMachine) -> PyIntRef { + fn start(&self, _vm: &VirtualMachine) -> PyIntRef { self.start.clone() } - fn stop(self, _vm: &VirtualMachine) -> PyIntRef { + fn stop(&self, _vm: &VirtualMachine) -> PyIntRef { self.stop.clone() } - fn step(self, _vm: &VirtualMachine) -> PyIntRef { + fn step(&self, _vm: &VirtualMachine) -> PyIntRef { self.step.clone() } - fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyRangeIterator { + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIterator { PyRangeIterator { position: Cell::new(0), - range: self, + range: zelf, } } - fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyRangeIterator { + fn reversed(&self, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -204,7 +204,7 @@ impl PyRangeRef { } } - fn len(self, _vm: &VirtualMachine) -> PyInt { + fn len(&self, _vm: &VirtualMachine) -> PyInt { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -217,7 +217,7 @@ impl PyRangeRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { if self.step.as_bigint().is_one() { format!("range({}, {})", self.start, self.stop) } else { @@ -225,11 +225,11 @@ impl PyRangeRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.is_empty() } - fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { + fn contains(&self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { match self.offset(int.as_bigint()) { Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), @@ -240,7 +240,7 @@ impl PyRangeRef { } } - fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { match self.index_of(int.as_bigint()) { Some(idx) => Ok(PyInt::new(idx)), @@ -251,7 +251,7 @@ impl PyRangeRef { } } - fn count(self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { + fn count(&self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { if let Ok(int) = item.downcast::() { if self.index_of(int.as_bigint()).is_some() { PyInt::new(1) @@ -263,7 +263,7 @@ impl PyRangeRef { } } - fn getitem(self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { + fn getitem(&self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { RangeIndex::Int(index) => { if let Some(value) = self.get(index.as_bigint()) { @@ -314,10 +314,10 @@ impl PyRangeRef { fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let range = if args.args.len() <= 2 { let (cls, stop) = args.bind(vm)?; - PyRangeRef::new(cls, stop, vm) + PyRange::new(cls, stop, vm) } else { let (cls, start, stop, step) = args.bind(vm)?; - PyRangeRef::new_from(cls, start, stop, step, vm) + PyRange::new_from(cls, start, stop, step, vm) }?; Ok(range.into_object()) From 3bc1e3598c9999a4f1aca5ea72a3d4f749702317 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 21:26:20 +0300 Subject: [PATCH 166/658] Use PyIterable --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 48 +++++++++++++++++++------------------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 011716335..11d62944a 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -49,6 +49,7 @@ assert len(a) == 0 assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) +assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) @@ -181,6 +182,7 @@ assert b == a assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index c3c580bc4..46ba1db66 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,7 +8,9 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{ + PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, +}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -61,14 +63,13 @@ struct PySetInner { } impl PySetInner { - fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { let elements: HashMap = match iterable { OptionalArg::Missing => HashMap::new(), OptionalArg::Present(iterable) => { let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; } elements } @@ -175,9 +176,11 @@ impl PySetInner { ) } - fn union(&self, other: &PySetInner, _vm: &VirtualMachine) -> PyResult { + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.clone(); - elements.extend(other.elements.clone()); + for item in other.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; + } Ok(PySetInner { elements }) } @@ -302,10 +305,9 @@ impl PySetInner { } } - fn update(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut self.elements, &v)?; + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut self.elements, &item?)?; } Ok(vm.get_none()) } @@ -350,7 +352,7 @@ impl PySetInner { impl PySetRef { fn new( cls: PyClassRef, - iterable: OptionalArg, + iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { PySet { @@ -413,14 +415,10 @@ impl PySetRef { ) } - fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().union(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().union(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().union(other, vm)?), }, PySet::class(vm), None, @@ -505,12 +503,12 @@ impl PySetRef { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ior(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(self.as_object().clone()) } - fn update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(vm.get_none()) } @@ -565,7 +563,7 @@ impl PySetRef { impl PyFrozenSetRef { fn new( cls: PyClassRef, - iterable: OptionalArg, + iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { PyFrozenSet { @@ -628,14 +626,10 @@ impl PyFrozenSetRef { ) } - fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.union(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.union(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.union(other, vm)?, }, PyFrozenSet::class(vm), None, From f538a92007f375c07dd018f295fd34b101be4d4b Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 20:34:27 +0200 Subject: [PATCH 167/658] pybytes.__new__ ok --- vm/src/obj/objbyteinner.rs | 91 +++++++++++++++++++++++++++++++++++++- vm/src/obj/objbytes.rs | 69 ++--------------------------- 2 files changed, 92 insertions(+), 68 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 0607621f1..fd08e4b93 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,9 +1,96 @@ +use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyObjectRef; + +use crate::function::OptionalArg; + +use crate::vm::VirtualMachine; + +use crate::pyobject::{PyResult, TypeProtocol}; + +use crate::obj::objstr::PyString; + +use super::objint; +use crate::obj::objint::PyInt; +use num_traits::ToPrimitive; + #[derive(Debug, Default, Clone)] pub struct PyByteInner { pub elements: Vec, } + impl PyByteInner { - pub fn new(data: Vec) -> Self { - PyByteInner { elements: data } + pub fn new( + val_option: OptionalArg, + enc_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // First handle bytes(string, encoding[, errors]) + if let OptionalArg::Present(enc) = enc_option { + if let OptionalArg::Present(eval) = val_option { + if let Ok(input) = eval.downcast::() { + if let Ok(encoding) = enc.clone().downcast::() { + if encoding.value.to_lowercase() == "utf8".to_string() + || encoding.value.to_lowercase() == "utf-8".to_string() + // TODO: different encoding + { + return Ok(PyByteInner { + elements: input.value.as_bytes().to_vec(), + }); + } else { + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error + ); + } + } else { + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + // On ly one argument + } else { + let value = if let OptionalArg::Present(ival) = val_option { + match_class!(ival.clone(), + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _l @ PyString=> {return Err(vm.new_type_error(format!( + "string argument without an encoding" + )));}, + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)));}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => Ok(PyByteInner { elements: val }), + Err(err) => Err(err), + } + } } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 74795088f..07e71e23c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -106,73 +106,10 @@ impl PyBytesRef { enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - // First handle bytes(string, encoding[, errors]) - if let OptionalArg::Present(enc) = enc_option { - if let OptionalArg::Present(eval) = val_option { - if let Ok(input) = eval.downcast::() { - if let Ok(encoding) = enc.clone().downcast::() { - if encoding.value.to_lowercase() == "utf8".to_string() - || encoding.value.to_lowercase() == "utf-8".to_string() - // TODO: different encoding - { - return PyBytes::new(input.value.as_bytes().to_vec()) - .into_ref_with_type(vm, cls.clone()); - } else { - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error - ); - } - } else { - return Err(vm.new_type_error(format!( - "bytes() argument 2 must be str, not {}", - enc.class().name - ))); - } - } else { - return Err(vm.new_type_error("encoding without a string argument".to_string())); - } - } else { - return Err(vm.new_type_error("encoding without a string argument".to_string())); - } - // On ly one argument - } else { - let value = if let OptionalArg::Present(ival) = val_option { - match_class!(ival.clone(), - i @ PyInt => { - let size = objint::get_value(&i.into_object()).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, - _l @ PyString=> {return Err(vm.new_type_error(format!( - "string argument without an encoding" - )));}, - obj => { - let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( - "cannot convert {} object to bytes", obj.class().name)));}); - - let mut data_bytes = vec![]; - for elem in elements.unwrap(){ - let v = objint::to_int(vm, &elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); - } - - } - Ok(data_bytes) - } - ) - } else { - Ok(vec![]) - }; - match value { - Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), - Err(err) => Err(err), - } + PyBytes { + inner: PyByteInner::new(val_option, enc_option, vm)?, } + .into_ref_with_type(vm, cls) } #[pymethod(name = "__repr__")] From c2e03ed34287ab9b9cd827b894b3d8fffa82efd1 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 11:55:13 -0700 Subject: [PATCH 168/658] fix int.__float__ --- vm/src/obj/objint.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 706b4ecfa..de211668f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -347,8 +347,11 @@ impl PyInt { zelf } - fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { - zelf + fn float(&self, vm: &VirtualMachine) -> PyResult { + self.value + .to_f64() + .map(PyFloat::from) + .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) } fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { From fa369ff77964b53d34a40a3a481bc728c2e0be47 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 21:58:37 +0300 Subject: [PATCH 169/658] Limit non-operator versions to set and frozenset --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 11d62944a..174426aa2 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -53,6 +53,7 @@ assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) @@ -186,6 +187,7 @@ assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 46ba1db66..2aa00431b 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -9,7 +9,8 @@ use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -17,6 +18,7 @@ use super::objbool; use super::objint; use super::objiter; use super::objlist::PyListIterator; +use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] @@ -467,6 +469,10 @@ impl PySetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -636,6 +642,10 @@ impl PyFrozenSetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { @@ -732,6 +742,27 @@ enum SetCombineOperation { Difference, } +struct SetIterable { + iterable: PyIterable, +} + +impl TryFromObject for SetIterable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if objtype::issubclass(&obj.class(), &vm.ctx.set_type()) + || objtype::issubclass(&obj.class(), &vm.ctx.frozenset_type()) + { + Ok(SetIterable { + iterable: PyIterable::try_from_object(vm, obj)?, + }) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) + } + } +} + fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } @@ -757,7 +788,7 @@ pub fn init(context: &PyContext) { "issubset" => context.new_rustfunc(PySetRef::le), "issuperset" => context.new_rustfunc(PySetRef::ge), "union" => context.new_rustfunc(PySetRef::union), - "__or__" => context.new_rustfunc(PySetRef::union), + "__or__" => context.new_rustfunc(PySetRef::or), "intersection" => context.new_rustfunc(PySetRef::intersection), "__and__" => context.new_rustfunc(PySetRef::intersection), "difference" => context.new_rustfunc(PySetRef::difference), @@ -798,7 +829,7 @@ pub fn init(context: &PyContext) { "issubset" => context.new_rustfunc(PyFrozenSetRef::le), "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), "union" => context.new_rustfunc(PyFrozenSetRef::union), - "__or__" => context.new_rustfunc(PyFrozenSetRef::union), + "__or__" => context.new_rustfunc(PyFrozenSetRef::or), "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), "difference" => context.new_rustfunc(PyFrozenSetRef::difference), From e011e3f32721fa1e7b3cd749152956cde336a5ee Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 22:36:56 +0300 Subject: [PATCH 170/658] intersection and difference support iterable --- tests/snippets/set.py | 8 +++ vm/src/obj/objset.rs | 142 +++++++++++++++++++++--------------------- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 174426aa2..a7977f35d 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -57,15 +57,19 @@ assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) +assert set([1,2,3]).intersection([1,2]) == set([1,2]) assert set([1,2,3]) & set([4,5]) == set([]) assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) +assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) assert set([1,2,3]).difference(set([1,2])) == set([3]) assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) +assert set([1,2,3]).difference([1,2]) == set([3]) assert set([1,2,3]) - set([4,5]) == set([1,2,3]) assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) +assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) @@ -191,15 +195,19 @@ assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) +assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) +assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 2aa00431b..508400694 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -187,33 +187,27 @@ impl PySetInner { Ok(PySetInner { elements }) } - fn intersection(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { - self._combine_inner(other, vm, SetCombineOperation::Intersection) - } - - fn difference(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { - self._combine_inner(other, vm, SetCombineOperation::Difference) - } - - fn _combine_inner( - &self, - other: &PySetInner, - vm: &VirtualMachine, - op: SetCombineOperation, - ) -> PyResult { + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = HashMap::new(); - - for element in self.elements.iter() { - let value = other.contains(element.1.clone(), vm)?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), - }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); + for item in other.iter(vm)? { + if let Ok(obj) = item { + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + insert_into_set(vm, &mut elements, &obj)?; + } } } + Ok(PySetInner { elements }) + } + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + for item in other.iter(vm)? { + if let Ok(obj) = item { + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + remove_from_set(vm, &mut elements, &obj)?; + } + } + } Ok(PySetInner { elements }) } @@ -265,21 +259,7 @@ impl PySetInner { } fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut self.elements, &item, &remove) + remove_from_set(vm, &mut self.elements, &item) } fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -427,28 +407,20 @@ impl PySetRef { )) } - fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().intersection(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().intersection(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), }, PySet::class(vm), None, )) } - fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), }, PySet::class(vm), None, @@ -473,6 +445,14 @@ impl PySetRef { self.union(other.iterable, vm) } + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -642,32 +622,20 @@ impl PyFrozenSetRef { )) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { - self.union(other.iterable, vm) - } - - fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.intersection(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.intersection(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.intersection(other, vm)?, }, PyFrozenSet::class(vm), None, )) } - fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.difference(other, vm)?, }, PyFrozenSet::class(vm), None, @@ -688,6 +656,18 @@ impl PyFrozenSetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } @@ -737,6 +717,28 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } +fn remove_from_set( + vm: &VirtualMachine, + elements: &mut HashMap, + item: &PyObjectRef, +) -> PyResult { + fn remove( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + value: &PyObjectRef, + ) -> PyResult { + match elements.remove(&key) { + None => { + let item_str = format!("{:?}", value); + Err(vm.new_key_error(item_str)) + } + Some(_) => Ok(vm.get_none()), + } + } + perform_action_with_hash(vm, elements, item, &remove) +} + enum SetCombineOperation { Intersection, Difference, @@ -790,9 +792,9 @@ pub fn init(context: &PyContext) { "union" => context.new_rustfunc(PySetRef::union), "__or__" => context.new_rustfunc(PySetRef::or), "intersection" => context.new_rustfunc(PySetRef::intersection), - "__and__" => context.new_rustfunc(PySetRef::intersection), + "__and__" => context.new_rustfunc(PySetRef::and), "difference" => context.new_rustfunc(PySetRef::difference), - "__sub__" => context.new_rustfunc(PySetRef::difference), + "__sub__" => context.new_rustfunc(PySetRef::sub), "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), "__doc__" => context.new_str(set_doc.to_string()), @@ -831,9 +833,9 @@ pub fn init(context: &PyContext) { "union" => context.new_rustfunc(PyFrozenSetRef::union), "__or__" => context.new_rustfunc(PyFrozenSetRef::or), "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), - "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), + "__and__" => context.new_rustfunc(PyFrozenSetRef::and), "difference" => context.new_rustfunc(PyFrozenSetRef::difference), - "__sub__" => context.new_rustfunc(PyFrozenSetRef::difference), + "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), From 3ed8727ee5b8252941d19e6c7e318510826ae903 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:01:49 +0300 Subject: [PATCH 171/658] symmetric_difference support iterable --- tests/snippets/set.py | 4 +++ vm/src/obj/objset.rs | 69 ++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a7977f35d..42237dd2e 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -73,9 +73,11 @@ assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) +assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -211,9 +213,11 @@ assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) +assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: frozenset([[]])) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 508400694..f8c37fdd6 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -190,10 +190,9 @@ impl PySetInner { fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = HashMap::new(); for item in other.iter(vm)? { - if let Ok(obj) = item { - if objbool::get_value(&self.contains(obj.clone(), vm)?) { - insert_into_set(vm, &mut elements, &obj)?; - } + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + insert_into_set(vm, &mut elements, &obj)?; } } Ok(PySetInner { elements }) @@ -202,37 +201,27 @@ impl PySetInner { fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.clone(); for item in other.iter(vm)? { - if let Ok(obj) = item { - if objbool::get_value(&self.contains(obj.clone(), vm)?) { - remove_from_set(vm, &mut elements, &obj)?; - } + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + remove_from_set(vm, &mut elements, &obj)?; } } Ok(PySetInner { elements }) } - fn symmetric_difference( - &self, - other: &PySetInner, - vm: &VirtualMachine, - ) -> PyResult { - let mut elements = HashMap::new(); + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut new_inner = self.clone(); - for element in self.elements.iter() { - let value = other.contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); + for item in other.iter(vm)? { + let obj = item?; + if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + new_inner.add(&obj, vm)?; + } else { + new_inner.remove(&obj, vm)?; } } - for element in other.elements.iter() { - let value = self.contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - Ok(PySetInner { elements }) + Ok(new_inner) } fn iter(&self, vm: &VirtualMachine) -> PyListIterator { @@ -427,14 +416,10 @@ impl PySetRef { )) } - fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().symmetric_difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().symmetric_difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), }, PySet::class(vm), None, @@ -453,6 +438,10 @@ impl PySetRef { self.difference(other.iterable, vm) } + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -642,14 +631,10 @@ impl PyFrozenSetRef { )) } - fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.symmetric_difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.symmetric_difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.symmetric_difference(other, vm)?, }, PyFrozenSet::class(vm), None, @@ -668,6 +653,10 @@ impl PyFrozenSetRef { self.difference(other.iterable, vm) } + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } @@ -796,7 +785,7 @@ pub fn init(context: &PyContext) { "difference" => context.new_rustfunc(PySetRef::difference), "__sub__" => context.new_rustfunc(PySetRef::sub), "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PySetRef::xor), "__doc__" => context.new_str(set_doc.to_string()), "add" => context.new_rustfunc(PySetRef::add), "remove" => context.new_rustfunc(PySetRef::remove), @@ -837,7 +826,7 @@ pub fn init(context: &PyContext) { "difference" => context.new_rustfunc(PyFrozenSetRef::difference), "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PyFrozenSetRef::xor), "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), "__len__" => context.new_rustfunc(PyFrozenSetRef::len), "__doc__" => context.new_str(frozenset_doc.to_string()), From 848350d3347bac1bc404816a1dfefe0c908b7b3c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:06:42 +0300 Subject: [PATCH 172/658] ior support only set and frozenset --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 42237dd2e..a44847f9b 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -119,6 +119,8 @@ a |= set([3,4,5]) assert a == set([1,2,3,4,5]) with assertRaises(TypeError): a |= 1 +with assertRaises(TypeError): + a |= [1,2,3] a = set([1,2,3]) a.intersection_update([2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index f8c37fdd6..888ffae53 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -478,8 +478,8 @@ impl PySetRef { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().update(iterable, vm)?; + fn ior(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } From 5ae921dc571273343d981a20f469ce2b64bc4303 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:26:59 +0300 Subject: [PATCH 173/658] update function support iterable --- tests/snippets/set.py | 6 +++ vm/src/obj/objset.rs | 106 +++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 59 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a44847f9b..393db0754 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -132,6 +132,8 @@ a &= set([2,3,4,5]) assert a == set([2,3]) with assertRaises(TypeError): a &= 1 +with assertRaises(TypeError): + a &= [1,2,3] a = set([1,2,3]) a.difference_update([3,4,5]) @@ -143,6 +145,8 @@ a -= set([3,4,5]) assert a == set([1,2]) with assertRaises(TypeError): a -= 1 +with assertRaises(TypeError): + a -= [1,2,3] a = set([1,2,3]) a.symmetric_difference_update([3,4,5]) @@ -154,6 +158,8 @@ a ^= set([3,4,5]) assert a == set([1,2,4,5]) with assertRaises(TypeError): a ^= 1 +with assertRaises(TypeError): + a ^= [1,2,3] # frozen set diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 888ffae53..4b60b5b5c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -16,7 +16,6 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter; use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; @@ -283,39 +282,41 @@ impl PySetInner { Ok(vm.get_none()) } - fn combine_update_inner( - &mut self, - iterable: &PyObjectRef, - vm: &VirtualMachine, - op: SetCombineOperation, - ) -> PyResult { - let elements = &mut self.elements; - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + let temp_inner = self.copy(); + self.clear(); + for item in iterable.iter(vm)? { + let obj = item?; + if objbool::get_value(&temp_inner.contains(obj.clone(), vm)?) { + self.add(&obj, vm)?; } } Ok(vm.get_none()) } - fn ixor(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let elements_original = self.elements.clone(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut self.elements, &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(&iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - self.elements.remove(&element.0.clone()); + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + self.remove(&obj, vm)?; } } + Ok(vm.get_none()) + } + fn symmetric_difference_update( + &mut self, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + self.add(&obj, vm)?; + } else { + self.remove(&obj, vm)?; + } + } Ok(vm.get_none()) } } @@ -488,49 +489,41 @@ impl PySetRef { Ok(vm.get_none()) } - fn intersection_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Intersection, - )?; + fn intersection_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().intersection_update(iterable, vm)?; Ok(vm.get_none()) } - fn iand(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Intersection, - )?; + fn iand(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .intersection_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } - fn difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Difference, - )?; + fn difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn isub(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Difference, - )?; + fn isub(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .difference_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } - fn symmetric_difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().ixor(iterable, vm)?; + fn symmetric_difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn ixor(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().ixor(iterable, vm)?; + fn ixor(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } } @@ -728,11 +721,6 @@ fn remove_from_set( perform_action_with_hash(vm, elements, item, &remove) } -enum SetCombineOperation { - Intersection, - Difference, -} - struct SetIterable { iterable: PyIterable, } From ec98b4d6bf509abffe66d47ef43ef60f3bdc7248 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 22:36:13 +0200 Subject: [PATCH 174/658] repr len eq --- tests/snippets/bytes.py | 23 ++++++++++++++++++++++- vm/src/obj/objbyteinner.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 1a4b43dfb..85469b756 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -14,7 +14,28 @@ except TypeError: else: assert False -# +a = b"abcd" +b = b"ab" +c = b"abcd" + +# +# repr +assert repr(bytes([0, 1, 2])) == repr(b'\x00\x01\x02') +assert ( +repr(bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) +== "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'") +) +assert repr(b"abcd") == "b'abcd'" + +#len +assert len(bytes("abcdé", "utf8")) == 6 + +# +assert a == b"abcd" +# assert a > b +# assert a >= b +# assert b < a +# assert b <= a assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index fd08e4b93..8d32aeabf 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,4 +1,3 @@ -use crate::obj::objtype::PyClassRef; use crate::pyobject::PyObjectRef; use crate::function::OptionalArg; @@ -93,4 +92,33 @@ impl PyByteInner { } } } + + pub fn repr(&self) -> PyResult { + let mut res = String::with_capacity(self.elements.len()); + for i in self.elements.iter() { + match i { + 0..=8 => res.push_str(&format!("\\x0{}", i)), + 9 => res.push_str("\\t"), + 10 => res.push_str("\\n"), + 13 => res.push_str("\\r"), + 32..=126 => res.push(*(i) as char), + _ => res.push_str(&format!("\\x{:x}", i)), + } + } + Ok(res) + } + + pub fn len(&self) -> usize { + self.elements.len() + } + + pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements == other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } } +// TODO +// fix b"é" not allowed should be bytes("é", "utf8") From dddf9fee3906348063b9298b4621ae32adfd0e72 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 22:42:47 +0200 Subject: [PATCH 175/658] gt lt ge le --- tests/snippets/bytes.py | 8 +- vm/src/obj/objbyteinner.rs | 32 +++++++ vm/src/obj/objbytes.rs | 169 +++++++++++++++---------------------- 3 files changed, 104 insertions(+), 105 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 85469b756..a0371b789 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -32,10 +32,10 @@ assert len(bytes("abcdé", "utf8")) == 6 # assert a == b"abcd" -# assert a > b -# assert a >= b -# assert b < a -# assert b <= a +assert a > b +assert a >= b +assert b < a +assert b <= a assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8d32aeabf..81c70d440 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -119,6 +119,38 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } + + pub fn ge(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements >= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn le(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements <= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn gt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements > other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn lt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements < other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } } // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 07e71e23c..54427fc57 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,30 +1,11 @@ -use crate::obj::objint::PyInt; -use crate::obj::objlist; -use crate::obj::objlist::PyList; -use crate::obj::objstr::PyString; -use crate::obj::objtuple::PyTuple; -use crate::obj::objtype; -use std::cell::Cell; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use crate::vm::VirtualMachine; use std::ops::Deref; -use num_traits::ToPrimitive; - use crate::function::OptionalArg; -use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, -}; -use crate::vm::VirtualMachine; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::PyByteInner; -use super::objint; -use super::objiter; -use super::objstr; use super::objtype::PyClassRef; -use std::clone::Clone; - /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -35,7 +16,7 @@ use std::clone::Clone; /// - any object implementing the buffer API.\n \ /// - an integer"; #[pyclass(name = "bytes", __inside_vm)] -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct PyBytes { inner: PyByteInner, } @@ -113,102 +94,88 @@ impl PyBytesRef { } #[pymethod(name = "__repr__")] - fn repr(self, _vm: &VirtualMachine) -> String { - // TODO: don't just unwrap - let data = self.inner.elements.clone(); - format!("b'{:?}'", data) + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("b'{}'", self.inner.repr()?))) } #[pymethod(name = "__len__")] fn len(self, _vm: &VirtualMachine) -> usize { - self.inner.elements.len() - } -} -/* - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value == other.value) - } else { - vm.ctx.not_implemented() - } + self.inner.len() } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value >= other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.eq(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value > other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.ge(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value <= other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.le(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value < other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.gt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - - fn hash(self, _vm: &VirtualMachine) -> u64 { - let mut hasher = DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() - } - - - fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { - PyBytesIterator { - position: Cell::new(0), - bytes: self, - } + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } } +// fn hash(self, _vm: &VirtualMachine) -> u64 { +// let mut hasher = DefaultHasher::new(); +// self.value.hash(&mut hasher); +// hasher.finish() +// } +// fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { +// PyBytesIterator { +// position: Cell::new(0), +// bytes: self, +// } +// } +// } -#[derive(Debug)] -pub struct PyBytesIterator { - position: Cell, - bytes: PyBytesRef, -} +// #[derive(Debug)] +// pub struct PyBytesIterator { +// position: Cell, +// bytes: PyBytesRef, +// } -impl PyValue for PyBytesIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.bytesiterator_type() - } -} +// impl PyValue for PyBytesIterator { +// fn class(vm: &VirtualMachine) -> PyClassRef { +// vm.ctx.bytesiterator_type() +// } +// } -type PyBytesIteratorRef = PyRef; +// type PyBytesIteratorRef = PyRef; -impl PyBytesIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { - if self.position.get() < self.bytes.value.len() { - let ret = self.bytes[self.position.get()]; - self.position.set(self.position.get() + 1); - Ok(ret) - } else { - Err(objiter::new_stop_iteration(vm)) - } - } +// impl PyBytesIteratorRef { +// fn next(self, vm: &VirtualMachine) -> PyResult { +// if self.position.get() < self.bytes.value.len() { +// let ret = self.bytes[self.position.get()]; +// self.position.set(self.position.get() + 1); +// Ok(ret) +// } else { +// Err(objiter::new_stop_iteration(vm)) +// } +// } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -}*/ +// fn iter(self, _vm: &VirtualMachine) -> Self { +// self +// } +// } From abc72e999217e0de9318a04904562182a79ff62b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:54:42 +0300 Subject: [PATCH 176/658] contains return bool --- vm/src/obj/objset.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 4b60b5b5c..5ec4fca46 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -88,14 +88,14 @@ impl PySetInner { } } - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { for element in self.elements.iter() { let value = vm._eq(needle.clone(), element.1.clone())?; if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); + return Ok(true); } } - Ok(vm.new_bool(false)) + Ok(false) } fn _compare_inner( @@ -124,8 +124,7 @@ impl PySetInner { return Ok(vm.new_bool(false)); } for element in get_other(swap).elements.iter() { - let value = get_zelf(swap).contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { + if !get_zelf(swap).contains(element.1.clone(), vm)? { return Ok(vm.new_bool(false)); } } @@ -190,7 +189,7 @@ impl PySetInner { let mut elements = HashMap::new(); for item in other.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { insert_into_set(vm, &mut elements, &obj)?; } } @@ -201,7 +200,7 @@ impl PySetInner { let mut elements = self.elements.clone(); for item in other.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { remove_from_set(vm, &mut elements, &obj)?; } } @@ -213,7 +212,7 @@ impl PySetInner { for item in other.iter(vm)? { let obj = item?; - if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + if !self.contains(obj.clone(), vm)? { new_inner.add(&obj, vm)?; } else { new_inner.remove(&obj, vm)?; @@ -287,7 +286,7 @@ impl PySetInner { self.clear(); for item in iterable.iter(vm)? { let obj = item?; - if objbool::get_value(&temp_inner.contains(obj.clone(), vm)?) { + if temp_inner.contains(obj.clone(), vm)? { self.add(&obj, vm)?; } } @@ -297,7 +296,7 @@ impl PySetInner { fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { for item in iterable.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { self.remove(&obj, vm)?; } } @@ -311,7 +310,7 @@ impl PySetInner { ) -> PyResult { for item in iterable.iter(vm)? { let obj = item?; - if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + if !self.contains(obj.clone(), vm)? { self.add(&obj, vm)?; } else { self.remove(&obj, vm)?; @@ -343,7 +342,7 @@ impl PySetRef { } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow().contains(needle, vm) } @@ -550,7 +549,7 @@ impl PyFrozenSetRef { } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.contains(needle, vm) } From 3c736c1f940f65e95275512e87d3ea28f46238e7 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 00:11:20 +0200 Subject: [PATCH 177/658] iter --- tests/snippets/bytes.py | 8 ++- vm/src/obj/objbyteinner.rs | 9 +++ vm/src/obj/objbytes.rs | 114 +++++++++++++++---------------------- 3 files changed, 63 insertions(+), 68 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a0371b789..bb72847b6 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -30,7 +30,7 @@ assert repr(b"abcd") == "b'abcd'" #len assert len(bytes("abcdé", "utf8")) == 6 -# +#comp assert a == b"abcd" assert a > b assert a >= b @@ -43,3 +43,9 @@ assert b'foobar'.__gt__(2) == NotImplemented assert b'foobar'.__ge__(2) == NotImplemented assert b'foobar'.__lt__(2) == NotImplemented assert b'foobar'.__le__(2) == NotImplemented + +#hash +hash(a) == hash(b"abcd") + +#iter +[i for i in b"abcd"] == ["a", "b", "c", "d"] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 81c70d440..93293e627 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -7,6 +7,8 @@ use crate::vm::VirtualMachine; use crate::pyobject::{PyResult, TypeProtocol}; use crate::obj::objstr::PyString; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use super::objint; use crate::obj::objint::PyInt; @@ -151,6 +153,13 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } + + pub fn hash(&self) -> usize { + let mut hasher = DefaultHasher::new(); + self.elements.hash(&mut hasher); + hasher.finish() as usize + } } + // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 54427fc57..8c64944ca 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,11 +1,14 @@ use crate::vm::VirtualMachine; +use core::cell::Cell; use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::PyByteInner; +use super::objiter; use super::objtype::PyClassRef; + /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -44,39 +47,18 @@ impl PyValue for PyBytes { } } -// Binary data support - -// Fill bytes class methods: - pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().inner.elements } -// pub fn init(context: &PyContext) { -// let bytes_doc = -pub fn init(ctx: &PyContext) { - PyBytesRef::extend_class(ctx, &ctx.bytes_type); +pub fn init(context: &PyContext) { + PyBytesRef::extend_class(context, &context.bytes_type); + let bytesiterator_type = &context.bytesiterator_type; + extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } -//extend_class!(context, &context.bytes_type, { -//"__new__" => context.new_rustfunc(bytes_new), -/* "__eq__" => context.new_rustfunc(PyBytesRef::eq), -"__lt__" => context.new_rustfunc(PyBytesRef::lt), -"__le__" => context.new_rustfunc(PyBytesRef::le), -"__gt__" => context.new_rustfunc(PyBytesRef::gt), -"__ge__" => context.new_rustfunc(PyBytesRef::ge), -"__hash__" => context.new_rustfunc(PyBytesRef::hash), -"__repr__" => context.new_rustfunc(PyBytesRef::repr), -"__len__" => context.new_rustfunc(PyBytesRef::len), -"__iter__" => context.new_rustfunc(PyBytesRef::iter), -"__doc__" => context.new_str(bytes_doc.to_string())*/ -// }); - -/* let bytesiterator_type = &context.bytesiterator_type; -extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), -});*/ -//} #[pyimpl(__inside_vm)] impl PyBytesRef { @@ -134,48 +116,46 @@ impl PyBytesRef { bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), _ => Ok(vm.ctx.not_implemented())) } + #[pymethod(name = "__hash__")] + fn hash(self, _vm: &VirtualMachine) -> usize { + self.inner.hash() + } + + #[pymethod(name = "__iter__")] + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { + position: Cell::new(0), + bytes: self, + } + } } -// fn hash(self, _vm: &VirtualMachine) -> u64 { -// let mut hasher = DefaultHasher::new(); -// self.value.hash(&mut hasher); -// hasher.finish() -// } +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, +} -// fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { -// PyBytesIterator { -// position: Cell::new(0), -// bytes: self, -// } -// } -// } +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } +} -// #[derive(Debug)] -// pub struct PyBytesIterator { -// position: Cell, -// bytes: PyBytesRef, -// } +type PyBytesIteratorRef = PyRef; -// impl PyValue for PyBytesIterator { -// fn class(vm: &VirtualMachine) -> PyClassRef { -// vm.ctx.bytesiterator_type() -// } -// } +impl PyBytesIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.inner.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } -// type PyBytesIteratorRef = PyRef; - -// impl PyBytesIteratorRef { -// fn next(self, vm: &VirtualMachine) -> PyResult { -// if self.position.get() < self.bytes.value.len() { -// let ret = self.bytes[self.position.get()]; -// self.position.set(self.position.get() + 1); -// Ok(ret) -// } else { -// Err(objiter::new_stop_iteration(vm)) -// } -// } - -// fn iter(self, _vm: &VirtualMachine) -> Self { -// self -// } -// } + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} From ec65b8480fe35ece7182544eaf75d0582a257266 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 00:45:53 +0200 Subject: [PATCH 178/658] add contains --- tests/snippets/bytes.py | 14 ++++++++++++++ vm/src/obj/objbyteinner.rs | 26 ++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index bb72847b6..2fd32153d 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -49,3 +49,17 @@ hash(a) == hash(b"abcd") #iter [i for i in b"abcd"] == ["a", "b", "c", "d"] + +#add +assert a + b == b"abcdab" + +#contains +# contains +assert b"ab" in b"abcd" +assert b"cd" in b"abcd" +assert b"abcd" in b"abcd" +assert b"a" in b"abcd" +assert b"d" in b"abcd" +assert b"dc" not in b"abcd" +# assert 97 in b"abcd" +# assert 150 not in b"abcd" \ No newline at end of file diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 93293e627..03876095c 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -159,6 +159,32 @@ impl PyByteInner { self.elements.hash(&mut hasher); hasher.finish() as usize } + + pub fn add(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + let elements: Vec = self + .elements + .iter() + .chain(other.elements.iter()) + .cloned() + .collect(); + Ok(vm.ctx.new_bytes(elements)) + } + + pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + for (n, i) in self.elements.iter().enumerate() { + if n + other.len() <= self.len() && *i == other.elements[0] { + if &self.elements[n..n + other.len()] == other.elements.as_slice() { + return Ok(vm.new_bool(true)); + } + } + } + Ok(vm.new_bool(false)) + } + + pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + self.elements.contains(int); + Ok(vm.new_bool(false)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8c64944ca..e052654bf 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -128,6 +128,33 @@ impl PyBytesRef { bytes: self, } } + + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.add(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + + #[pymethod(name = "__contains__")] + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // no new style since objint is not. + match_class!(needle, + bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), + int @ PyInt => self.inner.contains_int(&int, vm), + _ => Ok(vm.ctx.not_implemented())) + // if objtype::isinstance(&needle, &vm.ctx.bytes_type()) { + // let result = vec_contains(&self.value, &get_value(&needle)); + // vm.ctx.new_bool(result) + // } else if objtype::isinstance(&needle, &vm.ctx.int_type()) { + // let result = self + // .value + // .contains(&objint::get_value(&needle).to_u8().unwrap()); + // vm.ctx.new_bool(result) + // } else { + // vm.new_type_error(format!("Cannot add {:?} and {:?}", self, needle)) + // } + } } #[derive(Debug)] From 4d8a56aea1d86d861c488e1eb4fd8ee7669090b4 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:09:44 -0600 Subject: [PATCH 179/658] `list.__setitem__` with slice and int indexing optimisation is uncertain correctness is uncertain but it does appear to work --- vm/src/obj/objlist.rs | 173 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 159 insertions(+), 14 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 60c14b107..3a4035409 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -11,11 +11,11 @@ use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyVal use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; +//use super::objint; use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, SequenceIndex, + SequenceIndex, }; use super::objslice::PySliceRef; use super::objtype; @@ -181,22 +181,167 @@ impl PyListRef { } } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.borrow_mut(); + fn setitem( + self, + subscript: SequenceIndex, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.setindex(index, value, vm), + SequenceIndex::Slice(slice) => { + // TODO check special case of if a == b, may need copy of b first? + self.setslice(slice, value, vm) + } + } + } - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(pos_index) = elements.get_pos(idx) { - elements[pos_index] = value; - Ok(vm.get_none()) + fn setindex(self, index: i32, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut()[pos_index] = value; + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list assignment index out of range".to_string())) + } + } + + fn setslice(self, slice: PySliceRef, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&slice.start, &slice.stop); + if range.start < range.end { + match step.to_i32() { + Some(1) => self._set_slice(range, sec, vm), + Some(num) => { + // assign to extended slice + self._set_stepped_slice(range, num as usize, sec, vm) + } + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list, no question + #[allow(clippy::range_plus_one)] + self._set_stepped_slice(range.start..(range.start + 1), 1, sec, vm) + } + } } else { - Err(vm.new_index_error("list index out of range".to_string())) + // this functions as an insert of sec before range.start + self._set_slice(range.start..range.start, sec, vm) + // TODO this will take some special processing } } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = &slice.start.as_ref().map(|x| x + 1); + let stop = &slice.stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + match (-step).to_i32() { + Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list no question + self._set_stepped_slice_reverse(range.end - 1..range.end, 1, sec, vm) + } + } + } + } + + fn _set_slice(self, range: Range, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // consume the iter, we don't need it's size + // but if it's going to fail we want that to happen *before* we start modifing + if let Ok(items) = vm.extract_elements(&sec) { + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _set_stepped_slice( + self, + range: Range, + step: usize, + sec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let slicelen = ((range.end - range.start - 1) / step) + 1; + if let Ok(items) = vm.extract_elements(&sec) { + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _set_stepped_slice_reverse( + self, + range: Range, + step: usize, + sec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let slicelen = ((range.end - range.start - 1) / step) + 1; + if let Ok(items) = vm.extract_elements(&sec) { + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _replace_indexes(self, indexes: I, items: &[PyObjectRef]) + where + I: Iterator, + { + let mut elements = self.elements.borrow_mut(); + + for (i, value) in indexes.zip(items) { + // clone for refrence count + elements[i] = value.clone(); } } From 118c98ccc76977945e78495b28124b42290162f6 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:12:03 -0600 Subject: [PATCH 180/658] exhaustive (hopefully) test snippets I attempted to cover every case with the snippets but I easily could of missed some. --- tests/snippets/list.py | 198 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 3d9743c0b..71f7dea06 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -201,3 +201,201 @@ assert_raises(TypeError, bad_del_1) def bad_del_2(): del ['a', 'b'][2] assert_raises(IndexError, bad_del_2) + +# __setitem__ + +# simple index +x = [1, 2, 3, 4, 5] +x[0] = 'a' +assert x == ['a', 2, 3, 4, 5] +x[-1] = 'b' +assert x == ['a', 2, 3, 4, 'b'] +# make sure refrences are assigned correctly +y = [] +x[1] = y +y.append(100) +assert x[1] == y +assert x[1] == [100] + +#index bounds +def set_index_out_of_bounds_high(): + x = [0, 1, 2, 3, 4] + x[5] = 'a' + +def set_index_out_of_bounds_low(): + x = [0, 1, 2, 3, 4] + x[-6] = 'a' + +assert_raises(IndexError, set_index_out_of_bounds_high) +assert_raises(IndexError, set_index_out_of_bounds_low) + +# non stepped slice index +a = list(range(10)) +x = a[:] +y = a[:] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +# replace whole list +x[:] = ['a', 'b', 'c'] +y[::1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c'] +assert x == y +# splice list start +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[:1] = ['a', 'b', 'c'] +y[0:1] = ['a', 'b', 'c'] +z[:1:1] = ['a', 'b', 'c'] +zz[0:1:1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# splice list end +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[5:] = ['a', 'b', 'c'] +y[5::1] = ['a', 'b', 'c'] +z[5:10] = ['a', 'b', 'c'] +zz[5:10:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +assert x == y +assert x == z +assert x == zz +# insert sec +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[1:1] = ['a', 'b', 'c'] +y[1:0] = ['a', 'b', 'c'] +z[1:1:1] = ['a', 'b', 'c'] +zz[1:0:1] = ['a', 'b', 'c'] +assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# same but negative indexes? +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[-1:-1] = ['a', 'b', 'c'] +y[-1:9] = ['a', 'b', 'c'] +z[-1:-1:1] = ['a', 'b', 'c'] +zz[-1:9:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +assert x == y +assert x == z +assert x == zz +# splice mid +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', 'e'] +y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +assert x == y +x = a[:] +x[3:5] = ['a'] +assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +# assign empty to non stepped empty slice does nothing +x = a[:] +y = a[:] +x[5:2] = [] +y[5:2:1] = [] +assert x == a +assert y == a +# assign empty to non stepped slice removes elems +x = a[:] +y = a[:] +x[2:8] = [] +y[2:8:1] = [] +assert x == [0, 1, 8, 9] +assert x == y +# make sure refrences are assigned correctly +yy = [] +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', yy] +y[3:5:1] = ['a', 'b', 'c', 'd', yy] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +assert x == y +yy.append(100) +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == y +assert x[7] == yy +assert x[7] == [100] +assert y[7] == yy +assert y[7] == [100] + +# no zero step +def no_zero_step_set(): + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] +assert_raises(ValueError, no_zero_step_set) + +# stepped slice index +# forward slice +x = a[:] +x[2:8:2] = ['a', 'b', 'c'] +assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e'] +x[::2] = c +y[-10::2] = c +z[0:10:2] = c +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +assert x == y +assert x == z +assert x == zz +# backward slice +x = a[:] +x[8:2:-2] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +x[::-1] = c +y[9:-11:-1] = c +z[9::-1] = c +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +assert x == y +assert x == z +assert x == zz +# step size bigger than len +x = a[:] +x[::200] = ['a'] +assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x = a[:] +x[5::200] = ['a'] +assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] + +# bad stepped slices +def stepped_slice_assign_too_big(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b', 'c', 'd'] + +assert_raises(ValueError, stepped_slice_assign_too_big) + +def stepped_slice_assign_too_small(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b'] + +assert_raises(ValueError, stepped_slice_assign_too_small) + +# must assign iter t0 slice +def must_assign_iter_to_slice(): + x = [0, 1, 2, 3, 4] + x[::2] = 42 + +assert_raises(TypeError, must_assign_iter_to_slice) From a23b5bc470f0196079529b11351169cc543a362f Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:32:11 -0600 Subject: [PATCH 181/658] cleanup and cargo fmt --- vm/src/obj/objlist.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 3a4035409..1ef01ca3e 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -189,10 +189,7 @@ impl PyListRef { ) -> PyResult { match subscript { SequenceIndex::Int(index) => self.setindex(index, value, vm), - SequenceIndex::Slice(slice) => { - // TODO check special case of if a == b, may need copy of b first? - self.setslice(slice, value, vm) - } + SequenceIndex::Slice(slice) => self.setslice(slice, value, vm), } } @@ -229,7 +226,6 @@ impl PyListRef { } else { // this functions as an insert of sec before range.start self._set_slice(range.start..range.start, sec, vm) - // TODO this will take some special processing } } else { // calculate the range for the reverse slice, first the bounds needs to be made From 0e264426c9b876c4706042eff8cfd50415d81849 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Mon, 8 Apr 2019 09:12:22 +0200 Subject: [PATCH 182/658] fix import --- tests/snippets/builtin_slice.py | 16 ++++++++-------- tests/test_snippets.py | 22 +++++----------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index d61698034..b12ad8021 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -79,13 +79,13 @@ def test_all_slices(): test all possible slices except big number """ - import resources - from resources.cpython_generated_slices import SLICES_RES, START, END, STEP, LL + mod = __import__('cpython_generated_slices') - ll = LL - start = START - end = END - step = STEP + ll = mod.LL + start = mod.START + end = mod.END + step = mod.STEP + slices_res = mod.SLICES_RES count = 0 failures = [] @@ -94,11 +94,11 @@ def test_all_slices(): for t in step: lhs = ll[s:e:t] try: - assert lhs == SLICES_RES[count] + assert lhs == slices_res[count] except AssertionError: failures.append( "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( - s, e, t, lhs, SLICES_RES[count] + s, e, t, lhs, slices_res[count] ) ) count += 1 diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7a888f093..9322485ce 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -26,10 +26,6 @@ TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) -RESOURCES_DIR = os.path.abspath( - os.path.join(TEST_DIRS[_TestType.functional], "resources") -) - @contextlib.contextmanager def pushd(path): @@ -149,17 +145,11 @@ def generate_slices(path): class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - # setup resource dir - cls.resources = Path(RESOURCES_DIR) - if cls.resources.exists(): - shutil.rmtree( - RESOURCES_DIR, ignore_errors=True - ) # we don't care if dir doesnt exist - cls.resources.mkdir() - (cls.resources / "__init__.py").touch() - # Here add resource files - cls.slices_resource_path = Path(RESOURCES_DIR) / "cpython_generated_slices.py" + cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + if cls.slices_resource_path.exists(): + cls.slices_resource_path.unlink() + generate_slices(cls.slices_resource_path) # cargo stuff @@ -168,6 +158,4 @@ class SampleTestCase(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree( - RESOURCES_DIR, ignore_errors=True - ) # we don't care if dir doesnt exist + cls.slices_resource_path.unlink() From 86928114fa1505ea6dadfc39cb096e73249ac0ff Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Mon, 8 Apr 2019 10:33:56 +0200 Subject: [PATCH 183/658] resolve conflict --- vm/src/obj/objsequence.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 1e389bafd..32c723a26 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -2,9 +2,6 @@ use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; - use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; From 9c32de954bf586cd2425bc27014ca6f85e6bab6b Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 17:09:41 +0100 Subject: [PATCH 184/658] Add iterator for dictionary keys. --- vm/src/dictdatatype.rs | 12 +++++++++++ vm/src/obj/objdict.rs | 49 +++++++++++++++++++++++++++++++----------- vm/src/pyobject.rs | 3 +++ 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 8a6a8df1b..dc25e6eeb 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -120,6 +120,18 @@ impl Dict { self.len() == 0 } + pub fn next_entry(&self, position: usize) -> Option<(usize, &PyObjectRef, &T)> { + let mut new_position = position; + while position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[position] { + return Some((new_position + 1, key, value)); + } else { + new_position += 1; + } + } + None + } + pub fn iter_items(&self) -> impl Iterator + '_ { self.entries .iter() diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e84f7b178..4a1b6bbb1 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -132,20 +132,11 @@ impl PyDictRef { } /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + fn iter(self, _vm: &VirtualMachine) -> PyDictIterator { // TODO: separate type, not a list iterator - let keys = self - .entries - .borrow() - .get_items() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); - let key_list = vm.ctx.new_list(keys); - - PyListIterator { + PyDictIterator { position: Cell::new(0), - list: key_list.downcast().unwrap(), + dict: self, } } @@ -284,6 +275,35 @@ impl ItemProtocol for PyDictRef { } } +#[derive(Debug)] +struct PyDictIterator { + pub dict: PyDictRef, + pub position: Cell, +} +type PyDictIteratorRef = PyRef; + +impl PyDictIteratorRef { + fn next(self: PyDictIteratorRef, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, _value)) => { + self.position.set(new_position); + Ok(key.clone()) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictiterator_type.clone() + } +} + pub fn init(context: &PyContext) { extend_class!(context, &context.dict_type, { "__bool__" => context.new_rustfunc(PyDictRef::bool), @@ -305,4 +325,9 @@ pub fn init(context: &PyContext) { "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), }); + + extend_class!(context, &context.dictiterator_type, { + "__next__" => context.new_rustfunc(PyDictIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictIteratorRef::iter), + }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4d2c8353b..2c2371f21 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,6 +129,7 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, + pub dictiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -254,6 +255,7 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); + let dictiterator_type = create_type("dict_iterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -313,6 +315,7 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, + dictiterator_type, set_type, frozenset_type, true_value, From 5c4755ffd77f66371e102efee16528086da8c883 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 11:08:45 +0100 Subject: [PATCH 185/658] Separate iterators for items/keys/values. --- tests/snippets/dict.py | 9 ++- vm/src/dictdatatype.rs | 7 +- vm/src/obj/objdict.rs | 155 ++++++++++++++++++++++++++++------------- vm/src/pyobject.rs | 12 +++- 4 files changed, 128 insertions(+), 55 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f924bb0e7..f57fc37b6 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -37,11 +37,18 @@ for key in a.keys(): res.add(key) assert res == set(['a','b']) +x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} +del x['c'] +it = iter(x.items()) +assert ('a', 1) == next(it) +assert ('b', 2) == next(it) +assert ('d', 3) == next(it) + + x = {} x[1] = 1 assert x[1] == 1 - x[7] = 7 x[2] = 2 x[(5, 6)] = 5 diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index dc25e6eeb..0612bc8a3 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -120,13 +120,12 @@ impl Dict { self.len() == 0 } - pub fn next_entry(&self, position: usize) -> Option<(usize, &PyObjectRef, &T)> { - let mut new_position = position; + pub fn next_entry(&self, mut position: usize) -> Option<(usize, &PyObjectRef, &T)> { while position < self.entries.len() { if let Some(DictEntry { key, value, .. }) = &self.entries[position] { - return Some((new_position + 1, key, value)); + return Some((position + 1, key, value)); } else { - new_position += 1; + position += 1; } } None diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4a1b6bbb1..a0d9210b5 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -11,7 +11,6 @@ use crate::vm::{ReprGuard, VirtualMachine}; use crate::dictdatatype; use super::objiter; -use super::objlist::PyListIterator; use super::objstr; use crate::obj::objtype::PyClassRef; @@ -131,47 +130,16 @@ impl PyDictRef { self.entries.borrow_mut().clear() } - /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, _vm: &VirtualMachine) -> PyDictIterator { - // TODO: separate type, not a list iterator - PyDictIterator { - position: Cell::new(0), - dict: self, - } + fn iter(self, vm: &VirtualMachine) -> PyDictKeysIteratorRef { + PyDictKeysIteratorRef::new(self, vm) } - fn values(self, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type. `values` should be a live view over the collection, not an iterator. - let values = self - .entries - .borrow() - .get_items() - .iter() - .map(|(_k, v)| v.clone()) - .collect(); - let values_list = vm.ctx.new_list(values); - - PyListIterator { - position: Cell::new(0), - list: values_list.downcast().unwrap(), - } + fn values(self, vm: &VirtualMachine) -> PyDictValuesIteratorRef { + PyDictValuesIteratorRef::new(self, vm) } - fn items(self, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type. `items` should be a live view over the collection, not an iterator. - let items = self - .entries - .borrow() - .get_items() - .iter() - .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) - .collect(); - let items_list = vm.ctx.new_list(items); - - PyListIterator { - position: Cell::new(0), - list: items_list.downcast().unwrap(), - } + fn items(self, vm: &VirtualMachine) -> PyDictItemsIteratorRef { + PyDictItemsIteratorRef::new(self, vm) } pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -276,14 +244,22 @@ impl ItemProtocol for PyDictRef { } #[derive(Debug)] -struct PyDictIterator { +struct PyDictKeysIterator { pub dict: PyDictRef, pub position: Cell, } -type PyDictIteratorRef = PyRef; +type PyDictKeysIteratorRef = PyRef; -impl PyDictIteratorRef { - fn next(self: PyDictIteratorRef, vm: &VirtualMachine) -> PyResult { +impl PyDictKeysIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictKeysIteratorRef { + PyDictKeysIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self, vm: &VirtualMachine) -> PyResult { match self.dict.entries.borrow().next_entry(self.position.get()) { Some((new_position, key, _value)) => { self.position.set(new_position); @@ -298,9 +274,84 @@ impl PyDictIteratorRef { } } -impl PyValue for PyDictIterator { +impl PyValue for PyDictKeysIterator { fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictiterator_type.clone() + vm.ctx.dictkeysiterator_type.clone() + } +} + +#[derive(Debug)] +struct PyDictValuesIterator { + pub dict: PyDictRef, + pub position: Cell, +} +type PyDictValuesIteratorRef = PyRef; + +impl PyDictValuesIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictValuesIteratorRef { + PyDictValuesIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, _key, value)) => { + self.position.set(new_position); + Ok(value.clone()) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictValuesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictvaluesiterator_type.clone() + } +} + +#[derive(Debug)] +struct PyDictItemsIterator { + pub dict: PyDictRef, + pub position: Cell, +} + +type PyDictItemsIteratorRef = PyRef; + +impl PyDictItemsIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictItemsIteratorRef { + PyDictItemsIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self: PyDictItemsIteratorRef, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, value)) => { + self.position.set(new_position); + Ok(vm.ctx.new_tuple(vec![key.clone(), value.clone()])) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictItemsIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictitemsiterator_type.clone() } } @@ -326,8 +377,18 @@ pub fn init(context: &PyContext) { "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictiterator_type, { - "__next__" => context.new_rustfunc(PyDictIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictIteratorRef::iter), + extend_class!(context, &context.dictkeysiterator_type, { + "__next__" => context.new_rustfunc(PyDictKeysIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictKeysIteratorRef::iter), + }); + + extend_class!(context, &context.dictvaluesiterator_type, { + "__next__" => context.new_rustfunc(PyDictValuesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictValuesIteratorRef::iter), + }); + + extend_class!(context, &context.dictitemsiterator_type, { + "__next__" => context.new_rustfunc(PyDictItemsIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictItemsIteratorRef::iter), }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 2c2371f21..8c405c5b1 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,7 +129,9 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, - pub dictiterator_type: PyClassRef, + pub dictkeysiterator_type: PyClassRef, + pub dictvaluesiterator_type: PyClassRef, + pub dictitemsiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -255,7 +257,9 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); - let dictiterator_type = create_type("dict_iterator", &type_type, &object_type); + let dictkeysiterator_type = create_type("dict_keys", &type_type, &object_type); + let dictvaluesiterator_type = create_type("dict_values", &type_type, &object_type); + let dictitemsiterator_type = create_type("dict_items", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -315,7 +319,9 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, - dictiterator_type, + dictkeysiterator_type, + dictvaluesiterator_type, + dictitemsiterator_type, set_type, frozenset_type, true_value, From 2d127b983422e68a9d9fe26b509107fbaaf2bfce Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 11:58:11 +0100 Subject: [PATCH 186/658] Use macro to generate full set of views and iterators. --- vm/src/obj/objdict.rs | 217 ++++++++++++++++++++---------------------- vm/src/pyobject.rs | 27 ++++-- 2 files changed, 121 insertions(+), 123 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index a0d9210b5..e80fa7d8a 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -130,16 +130,20 @@ impl PyDictRef { self.entries.borrow_mut().clear() } - fn iter(self, vm: &VirtualMachine) -> PyDictKeysIteratorRef { - PyDictKeysIteratorRef::new(self, vm) + fn iter(self, _vm: &VirtualMachine) -> PyDictKeyIterator { + PyDictKeyIterator::new(self) } - fn values(self, vm: &VirtualMachine) -> PyDictValuesIteratorRef { - PyDictValuesIteratorRef::new(self, vm) + fn keys(self, _vm: &VirtualMachine) -> PyDictKeys { + PyDictKeys::new(self) } - fn items(self, vm: &VirtualMachine) -> PyDictItemsIteratorRef { - PyDictItemsIteratorRef::new(self, vm) + fn values(self, _vm: &VirtualMachine) -> PyDictValues { + PyDictValues::new(self) + } + + fn items(self, _vm: &VirtualMachine) -> PyDictItems { + PyDictItems::new(self) } pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -243,116 +247,89 @@ impl ItemProtocol for PyDictRef { } } -#[derive(Debug)] -struct PyDictKeysIterator { - pub dict: PyDictRef, - pub position: Cell, -} -type PyDictKeysIteratorRef = PyRef; - -impl PyDictKeysIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictKeysIteratorRef { - PyDictKeysIterator { - position: Cell::new(0), - dict, +macro_rules! dict_iterator { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $result_fn: expr) => { + #[derive(Debug)] + struct $name { + pub dict: PyDictRef, } - .into_ref(vm) - } - fn next(self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, _value)) => { - self.position.set(new_position); - Ok(key.clone()) + impl $name { + fn new(dict: PyDictRef) -> Self { + $name { dict: dict } } - None => Err(objiter::new_stop_iteration(vm)), - } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -} - -impl PyValue for PyDictKeysIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictkeysiterator_type.clone() - } -} - -#[derive(Debug)] -struct PyDictValuesIterator { - pub dict: PyDictRef, - pub position: Cell, -} -type PyDictValuesIteratorRef = PyRef; - -impl PyDictValuesIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictValuesIteratorRef { - PyDictValuesIterator { - position: Cell::new(0), - dict, - } - .into_ref(vm) - } - - fn next(self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, _key, value)) => { - self.position.set(new_position); - Ok(value.clone()) + fn iter(&self, _vm: &VirtualMachine) -> $iter_name { + $iter_name::new(self.dict.clone()) } - None => Err(objiter::new_stop_iteration(vm)), } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -} - -impl PyValue for PyDictValuesIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictvaluesiterator_type.clone() - } -} - -#[derive(Debug)] -struct PyDictItemsIterator { - pub dict: PyDictRef, - pub position: Cell, -} - -type PyDictItemsIteratorRef = PyRef; - -impl PyDictItemsIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictItemsIteratorRef { - PyDictItemsIterator { - position: Cell::new(0), - dict, - } - .into_ref(vm) - } - - fn next(self: PyDictItemsIteratorRef, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, value)) => { - self.position.set(new_position); - Ok(vm.ctx.new_tuple(vec![key.clone(), value.clone()])) + impl PyValue for $name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$class.clone() } - None => Err(objiter::new_stop_iteration(vm)), } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } + #[derive(Debug)] + struct $iter_name { + pub dict: PyDictRef, + pub position: Cell, + } + + impl $iter_name { + fn new(dict: PyDictRef) -> Self { + $iter_name { + position: Cell::new(0), + dict, + } + } + + fn next(&self, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, value)) => { + self.position.set(new_position); + Ok($result_fn(vm, key, value)) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + } + + impl PyValue for $iter_name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$iter_class.clone() + } + } + }; } -impl PyValue for PyDictItemsIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictitemsiterator_type.clone() - } +dict_iterator! { + PyDictKeys, + PyDictKeyIterator, + dictkeys_type, + dictkeyiterator_type, + |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() +} + +dict_iterator! { + PyDictValues, + PyDictValueIterator, + dictvalues_type, + dictvalueiterator_type, + |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() +} + +dict_iterator! { + PyDictItems, + PyDictItemIterator, + dictitems_type, + dictitemiterator_type, + |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| + vm.ctx.new_tuple(vec![key.clone(), value.clone()]) } pub fn init(context: &PyContext) { @@ -371,24 +348,36 @@ pub fn init(context: &PyContext) { "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), // TODO: separate type. `keys` should be a live view over the collection, not an iterator. - "keys" => context.new_rustfunc(PyDictRef::iter), + "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictkeysiterator_type, { - "__next__" => context.new_rustfunc(PyDictKeysIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictKeysIteratorRef::iter), + extend_class!(context, &context.dictkeys_type, { + "__iter__" => context.new_rustfunc(PyDictKeys::iter), }); - extend_class!(context, &context.dictvaluesiterator_type, { - "__next__" => context.new_rustfunc(PyDictValuesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictValuesIteratorRef::iter), + extend_class!(context, &context.dictkeyiterator_type, { + "__next__" => context.new_rustfunc(PyDictKeyIterator::next), + "__iter__" => context.new_rustfunc(PyDictKeyIterator::iter), }); - extend_class!(context, &context.dictitemsiterator_type, { - "__next__" => context.new_rustfunc(PyDictItemsIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictItemsIteratorRef::iter), + extend_class!(context, &context.dictvalues_type, { + "__iter__" => context.new_rustfunc(PyDictValues::iter), + }); + + extend_class!(context, &context.dictvalueiterator_type, { + "__next__" => context.new_rustfunc(PyDictValueIterator::next), + "__iter__" => context.new_rustfunc(PyDictValueIterator::iter), + }); + + extend_class!(context, &context.dictitems_type, { + "__iter__" => context.new_rustfunc(PyDictItems::iter), + }); + + extend_class!(context, &context.dictitemiterator_type, { + "__next__" => context.new_rustfunc(PyDictItemIterator::next), + "__iter__" => context.new_rustfunc(PyDictItemIterator::iter), }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 8c405c5b1..434693ea6 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,9 +129,12 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, - pub dictkeysiterator_type: PyClassRef, - pub dictvaluesiterator_type: PyClassRef, - pub dictitemsiterator_type: PyClassRef, + pub dictkeyiterator_type: PyClassRef, + pub dictvalueiterator_type: PyClassRef, + pub dictitemiterator_type: PyClassRef, + pub dictkeys_type: PyClassRef, + pub dictvalues_type: PyClassRef, + pub dictitems_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -257,9 +260,12 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); - let dictkeysiterator_type = create_type("dict_keys", &type_type, &object_type); - let dictvaluesiterator_type = create_type("dict_values", &type_type, &object_type); - let dictitemsiterator_type = create_type("dict_items", &type_type, &object_type); + let dictkeys_type = create_type("dict_keys", &type_type, &object_type); + let dictvalues_type = create_type("dict_values", &type_type, &object_type); + let dictitems_type = create_type("dict_items", &type_type, &object_type); + let dictkeyiterator_type = create_type("dict_keyiterator", &type_type, &object_type); + let dictvalueiterator_type = create_type("dict_valueiterator", &type_type, &object_type); + let dictitemiterator_type = create_type("dict_itemiterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -319,9 +325,12 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, - dictkeysiterator_type, - dictvaluesiterator_type, - dictitemsiterator_type, + dictkeys_type, + dictvalues_type, + dictitems_type, + dictkeyiterator_type, + dictvalueiterator_type, + dictitemiterator_type, set_type, frozenset_type, true_value, From 29e9753e51fa127aec3efea7a68503cf02145b7b Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 15:02:04 +0100 Subject: [PATCH 187/658] Even newer style class definitions. --- vm/src/obj/objdict.rs | 51 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e80fa7d8a..612fa2cb7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -8,11 +8,11 @@ use crate::pyobject::{ }; use crate::vm::{ReprGuard, VirtualMachine}; -use crate::dictdatatype; - use super::objiter; use super::objstr; +use crate::dictdatatype; use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyClassImpl; pub type DictContentType = dictdatatype::Dict; @@ -248,17 +248,20 @@ impl ItemProtocol for PyDictRef { } macro_rules! dict_iterator { - ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $result_fn: expr) => { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { + #[pyclass(name = $class_name, __inside_vm)] #[derive(Debug)] struct $name { pub dict: PyDictRef, } + #[pyimpl(__inside_vm)] impl $name { fn new(dict: PyDictRef) -> Self { $name { dict: dict } } + #[pymethod(name = "__iter__")] fn iter(&self, _vm: &VirtualMachine) -> $iter_name { $iter_name::new(self.dict.clone()) } @@ -270,12 +273,14 @@ macro_rules! dict_iterator { } } + #[pyclass(name = $iter_class_name, __inside_vm)] #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, pub position: Cell, } + #[pyimpl(__inside_vm)] impl $iter_name { fn new(dict: PyDictRef) -> Self { $iter_name { @@ -284,6 +289,7 @@ macro_rules! dict_iterator { } } + #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { match self.dict.entries.borrow().next_entry(self.position.get()) { Some((new_position, key, value)) => { @@ -294,6 +300,7 @@ macro_rules! dict_iterator { } } + #[pymethod(name = "__iter__")] fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { zelf } @@ -312,6 +319,8 @@ dict_iterator! { PyDictKeyIterator, dictkeys_type, dictkeyiterator_type, + "dictkeys", + "dictkeyiterator", |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() } @@ -320,6 +329,8 @@ dict_iterator! { PyDictValueIterator, dictvalues_type, dictvalueiterator_type, + "dictvalues", + "dictvalueiterator", |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() } @@ -328,6 +339,8 @@ dict_iterator! { PyDictItemIterator, dictitems_type, dictitemiterator_type, + "dictitems", + "dictitemiterator", |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| vm.ctx.new_tuple(vec![key.clone(), value.clone()]) } @@ -354,30 +367,10 @@ pub fn init(context: &PyContext) { "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictkeys_type, { - "__iter__" => context.new_rustfunc(PyDictKeys::iter), - }); - - extend_class!(context, &context.dictkeyiterator_type, { - "__next__" => context.new_rustfunc(PyDictKeyIterator::next), - "__iter__" => context.new_rustfunc(PyDictKeyIterator::iter), - }); - - extend_class!(context, &context.dictvalues_type, { - "__iter__" => context.new_rustfunc(PyDictValues::iter), - }); - - extend_class!(context, &context.dictvalueiterator_type, { - "__next__" => context.new_rustfunc(PyDictValueIterator::next), - "__iter__" => context.new_rustfunc(PyDictValueIterator::iter), - }); - - extend_class!(context, &context.dictitems_type, { - "__iter__" => context.new_rustfunc(PyDictItems::iter), - }); - - extend_class!(context, &context.dictitemiterator_type, { - "__next__" => context.new_rustfunc(PyDictItemIterator::next), - "__iter__" => context.new_rustfunc(PyDictItemIterator::iter), - }); + PyDictKeys::extend_class(context, &context.dictkeys_type); + PyDictKeyIterator::extend_class(context, &context.dictkeyiterator_type); + PyDictValues::extend_class(context, &context.dictvalues_type); + PyDictValueIterator::extend_class(context, &context.dictvalueiterator_type); + PyDictItems::extend_class(context, &context.dictitems_type); + PyDictItemIterator::extend_class(context, &context.dictitemiterator_type); } From 2811f27bb546a811ea54628df5b73cf6e61a8d48 Mon Sep 17 00:00:00 2001 From: andrew Date: Mon, 8 Apr 2019 16:03:24 +0200 Subject: [PATCH 188/658] add a warning --- tests/generator/not_impl_header.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_header.txt index cd6f8f5a1..4ceefff4d 100644 --- a/tests/generator/not_impl_header.txt +++ b/tests/generator/not_impl_header.txt @@ -1,2 +1,7 @@ +# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE +# EDIT tests/not_impl_gen.py, NOT THIS FILE. +# RESULTS OF THIS TEST DEPEND ON THE CPYTHON +# VERSION USED TO RUN not_impl_gen.py + import platform From 3aa110a2960a7572852f98b8f97a544dcb379d1c Mon Sep 17 00:00:00 2001 From: andrew Date: Mon, 8 Apr 2019 16:03:35 +0200 Subject: [PATCH 189/658] add .gitignore --- tests/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/.gitignore diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 000000000..05ba24c43 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +snippets/whats_left_to_implement.py From b1a95f669f9a5fcb5a439ce240b7c767d36cd04b Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Mon, 8 Apr 2019 17:02:27 +0200 Subject: [PATCH 190/658] Initial version of symbol table builder. --- vm/src/compile.rs | 11 +- vm/src/lib.rs | 1 + vm/src/symboltable.rs | 272 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 vm/src/symboltable.rs diff --git a/vm/src/compile.rs b/vm/src/compile.rs index dd045c3bf..cc52661a4 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -10,6 +10,7 @@ use crate::error::CompileError; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; +use crate::symboltable::SymbolTable; use crate::VirtualMachine; use num_complex::Complex64; use rustpython_parser::{ast, parser}; @@ -37,6 +38,8 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; + let mut symbol_table = SymbolTable::new(); + symbol_table.scan_program(&ast); compiler.compile_program(&ast) } Mode::Eval => { @@ -194,11 +197,8 @@ impl Compiler { // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::Global { names } => { - unimplemented!("global {:?}", names); - } - ast::Statement::Nonlocal { names } => { - unimplemented!("nonlocal {:?}", names); + ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => { + // Handled during symbol table construction. } ast::Statement::If { test, body, orelse } => { let end_label = self.new_label(); @@ -1624,6 +1624,7 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option< (body, None) } + #[cfg(test)] mod tests { use super::Compiler; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 89b4dd872..578f1d99a 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -55,6 +55,7 @@ pub mod import; pub mod obj; pub mod pyobject; pub mod stdlib; +mod symboltable; mod sysmodule; mod traceback; pub mod util; diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs new file mode 100644 index 000000000..6525e71e1 --- /dev/null +++ b/vm/src/symboltable.rs @@ -0,0 +1,272 @@ +/* Python code is pre-scanned for symbols in the ast. + +This ensures that global and nonlocal keywords are picked up. +Then the compiler can use the symbol table to generate proper +load and store instructions for names. +*/ + +use rustpython_parser::ast; +use std::collections::HashMap; + +pub enum SymbolRole { + Global, + Nonlocal, + Used, + Assigned, +} + +pub struct SymbolTable { + // TODO: split-up into nested scopes. + symbols: HashMap, +} + +impl SymbolTable { + pub fn new() -> Self { + SymbolTable { + symbols: HashMap::new(), + } + } + + pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { + return self.symbols.get(name); + } + + pub fn scan_program(&mut self, program: &ast::Program) { + self.scan_statements(&program.statements); + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) { + for statement in statements { + self.scan_statement(statement) + } + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) { + match &statement.node { + ast::Statement::Global { names } => { + for name in names { + self.register_name(name, SymbolRole::Global); + } + } + ast::Statement::Nonlocal { names } => { + for name in names { + self.register_name(name, SymbolRole::Nonlocal); + } + } + ast::Statement::FunctionDef { + name, + body, + args, + decorator_list, + returns, + } => { + self.scan_expressions(decorator_list); + self.register_name(name, SymbolRole::Assigned); + for parameter in &args.args {} + for parameter in &args.kwonlyargs {} + + self.scan_expressions(&args.defaults); + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression); + } + } + + self.scan_statements(body); + if let Some(expression) = returns { + self.scan_expression(expression); + } + } + ast::Statement::ClassDef { + name, + body, + bases, + keywords, + decorator_list, + } => { + self.register_name(name, SymbolRole::Assigned); + self.scan_statements(body); + self.scan_expressions(bases); + for keyword in keywords { + self.scan_expression(&keyword.value); + } + self.scan_expressions(decorator_list); + } + ast::Statement::Expression { expression } => self.scan_expression(expression), + ast::Statement::If { test, body, orelse } => { + self.scan_expression(test); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::For { + target, + iter, + body, + orelse, + } => { + self.scan_expression(target); + self.scan_expression(iter); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::While { test, body, orelse } => { + self.scan_expression(test); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { + // No symbols here. + } + ast::Statement::Import { import_parts } => for part in import_parts {}, + ast::Statement::Return { value } => { + if let Some(expression) = value { + self.scan_expression(expression); + } + } + ast::Statement::Assert { test, msg } => { + self.scan_expression(test); + if let Some(expression) = msg { + self.scan_expression(expression); + } + } + ast::Statement::Delete { targets } => { + self.scan_expressions(targets); + } + ast::Statement::Assign { targets, value } => { + self.scan_expressions(targets); + self.scan_expression(value); + } + ast::Statement::AugAssign { + target, + op: _, + value, + } => { + self.scan_expression(target); + self.scan_expression(value); + } + ast::Statement::With { items, body } => { + for item in items { + self.scan_expression(&item.context_expr); + if let Some(expression) = &item.optional_vars { + self.scan_expression(expression); + } + } + self.scan_statements(body); + } + ast::Statement::Try { + body, + handlers, + orelse, + finalbody, + } => { + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + if let Some(code) = finalbody { + self.scan_statements(code); + } + } + ast::Statement::Raise { exception, cause } => { + if let Some(expression) = exception { + self.scan_expression(expression); + } + if let Some(expression) = cause { + self.scan_expression(expression); + } + } + } + } + + fn scan_expressions(&mut self, expressions: &[ast::Expression]) { + for expression in expressions { + self.scan_expression(expression); + } + } + + fn scan_expression(&mut self, expression: &ast::Expression) { + match expression { + ast::Expression::Binop { a, op: _, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::BoolOp { a, op: _, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::Compare { vals, ops: _ } => { + self.scan_expressions(vals); + } + ast::Expression::Subscript { a, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::Attribute { value, name: _ } => { + self.scan_expression(value); + } + ast::Expression::Dict { elements } => { + for (key, value) in elements { + self.scan_expression(key); + self.scan_expression(value); + } + } + ast::Expression::Yield { value } => { + if let Some(expression) = value { + self.scan_expression(expression); + } + } + ast::Expression::YieldFrom { value } => { + self.scan_expression(value); + } + ast::Expression::Unop { op: _, a } => { + self.scan_expression(a); + } + ast::Expression::True + | ast::Expression::False + | ast::Expression::None + | ast::Expression::Ellipsis => {} + ast::Expression::Number { .. } => {} + ast::Expression::Starred { value } => { + self.scan_expression(value); + } + ast::Expression::Bytes { .. } => {} + ast::Expression::Tuple { elements } + | ast::Expression::Set { elements } + | ast::Expression::List { elements } + | ast::Expression::Slice { elements } => { + self.scan_expressions(elements); + } + ast::Expression::Comprehension { kind, generators } => {} + ast::Expression::Call { + function, + args, + keywords, + } => { + self.scan_expression(function); + self.scan_expressions(args); + } + ast::Expression::String { value } => {} + ast::Expression::Identifier { name } => { + self.register_name(name, SymbolRole::Used); + } + ast::Expression::Lambda { args, body } => { + self.scan_expression(body); + } + ast::Expression::IfExpression { test, body, orelse } => { + self.scan_expression(test); + self.scan_expression(body); + self.scan_expression(orelse); + } + } + } + + fn register_name(&mut self, name: &String, role: SymbolRole) { + self.symbols.insert(name.clone(), role); + } +} From e7e126e31ded0b1da77d945f0f92d631b0055d57 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 8 Apr 2019 12:30:53 -0500 Subject: [PATCH 191/658] Fix whats_left.sh --- whats_left.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/whats_left.sh b/whats_left.sh index c6f191bd5..e05051d28 100755 --- a/whats_left.sh +++ b/whats_left.sh @@ -1,5 +1,11 @@ #!/bin/sh +set -e -cd "$(dirname "$0")" || exit +cd "$(dirname "$0")" +cd tests + +python3 not_impl_gen.py + +cd .. cargo run -- tests/snippets/whats_left_to_implement.py From 4e277cfcc2d6eac7aa15cc493dacaa46fa8971d8 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 12:41:28 -0600 Subject: [PATCH 192/658] move to useing PyIterable --- vm/src/obj/objlist.rs | 108 ++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 1ef01ca3e..4c3c16383 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -7,7 +7,10 @@ use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{ + IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, +}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -189,7 +192,12 @@ impl PyListRef { ) -> PyResult { match subscript { SequenceIndex::Int(index) => self.setindex(index, value, vm), - SequenceIndex::Slice(slice) => self.setslice(slice, value, vm), + SequenceIndex::Slice(slice) => { + if let Ok(sec) = PyIterable::try_from_object(vm, value) { + return self.setslice(slice, sec, vm); + } + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } } } @@ -202,7 +210,7 @@ impl PyListRef { } } - fn setslice(self, slice: PySliceRef, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn setslice(self, slice: PySliceRef, sec: PyIterable, vm: &VirtualMachine) -> PyResult { let step = slice.step.clone().unwrap_or_else(BigInt::one); if step.is_zero() { @@ -244,52 +252,50 @@ impl PyListRef { } } - fn _set_slice(self, range: Range, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // consume the iter, we don't need it's size - // but if it's going to fail we want that to happen *before* we start modifing - if let Ok(items) = vm.extract_elements(&sec) { - // replace the range of elements with the full sequence - self.elements.borrow_mut().splice(range, items); + fn _set_slice(self, range: Range, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - Ok(vm.get_none()) - } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) - } + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) } fn _set_stepped_slice( self, range: Range, step: usize, - sec: PyObjectRef, + sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { + let slicelen = ((range.end - range.start - 1) / step) + 1; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing - let slicelen = ((range.end - range.start - 1) / step) + 1; - if let Ok(items) = vm.extract_elements(&sec) { - let n = items.len(); + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - if range.start < range.end { - if n == slicelen { - let indexes = range.step_by(step); - self._replace_indexes(indexes, &items); - Ok(vm.get_none()) - } else { - Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size {}", - n, slicelen - ))) - } + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) } else { - // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size 0", - n + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen ))) } } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) } } @@ -297,35 +303,35 @@ impl PyListRef { self, range: Range, step: usize, - sec: PyObjectRef, + sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { + let slicelen = ((range.end - range.start - 1) / step) + 1; + // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing - let slicelen = ((range.end - range.start - 1) / step) + 1; - if let Ok(items) = vm.extract_elements(&sec) { - let n = items.len(); + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - if range.start < range.end { - if n == slicelen { - let indexes = range.rev().step_by(step); - self._replace_indexes(indexes, &items); - Ok(vm.get_none()) - } else { - Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size {}", - n, slicelen - ))) - } + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) } else { - // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size 0", - n + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen ))) } } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) } } From d0e6d2fa323992ce1bce5ad998a55c1d04c7883c Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 13:18:37 -0600 Subject: [PATCH 193/658] add tests for custom iters --- tests/snippets/list.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 71f7dea06..55c84fdf9 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -399,3 +399,55 @@ def must_assign_iter_to_slice(): x[::2] = 42 assert_raises(TypeError, must_assign_iter_to_slice) + +# other iterables? +a = list(range(10)) + +# string +x = a[:] +x[3:8] = "abcdefghi" +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] + +# tuple +x = a[:] +x[3:8] = (11, 12, 13, 14, 15) +assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + +# class +class CIterNext: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + def __iter__(self): + return self + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + +x = list(range(10)) +x[3:8] = CIterNext() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +class CIter: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __iter__(self): + for n in self.sec: + yield n + +x = list(range(10)) +x[3:8] = CIter() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +class CGetItem: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __getitem__(self, sub): + return self.sec[sub] + +x = list(range(10)) +x[3:8] = CGetItem() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] From 8232a4d28512323125f853e3d448cc190c2c3a6f Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 21:27:44 +0200 Subject: [PATCH 194/658] finish contains --- tests/snippets/bytes.py | 45 ++++++++++++++++++++------------------ vm/src/obj/objbyteinner.rs | 14 ++++++++---- vm/src/obj/objbytes.rs | 12 +--------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2fd32153d..b03eb3808 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -1,10 +1,10 @@ from testutils import assertRaises # new -assert bytes([1,2,3]) -assert bytes((1,2,3)) +assert bytes([1, 2, 3]) +assert bytes((1, 2, 3)) assert bytes(range(4)) -assert b'bla' +assert b"bla" assert bytes(3) assert bytes("bla", "utf8") try: @@ -20,40 +20,39 @@ c = b"abcd" # # repr -assert repr(bytes([0, 1, 2])) == repr(b'\x00\x01\x02') -assert ( -repr(bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) -== "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'") +assert repr(bytes([0, 1, 2])) == repr(b"\x00\x01\x02") +assert repr( + bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) + == "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'" ) assert repr(b"abcd") == "b'abcd'" -#len +# len assert len(bytes("abcdé", "utf8")) == 6 -#comp +# comp assert a == b"abcd" assert a > b assert a >= b assert b < a assert b <= a -assert b'foobar'.__eq__(2) == NotImplemented -assert b'foobar'.__ne__(2) == NotImplemented -assert b'foobar'.__gt__(2) == NotImplemented -assert b'foobar'.__ge__(2) == NotImplemented -assert b'foobar'.__lt__(2) == NotImplemented -assert b'foobar'.__le__(2) == NotImplemented +assert b"foobar".__eq__(2) == NotImplemented +assert b"foobar".__ne__(2) == NotImplemented +assert b"foobar".__gt__(2) == NotImplemented +assert b"foobar".__ge__(2) == NotImplemented +assert b"foobar".__lt__(2) == NotImplemented +assert b"foobar".__le__(2) == NotImplemented -#hash +# hash hash(a) == hash(b"abcd") -#iter +# iter [i for i in b"abcd"] == ["a", "b", "c", "d"] -#add +# add assert a + b == b"abcdab" -#contains # contains assert b"ab" in b"abcd" assert b"cd" in b"abcd" @@ -61,5 +60,9 @@ assert b"abcd" in b"abcd" assert b"a" in b"abcd" assert b"d" in b"abcd" assert b"dc" not in b"abcd" -# assert 97 in b"abcd" -# assert 150 not in b"abcd" \ No newline at end of file +assert 97 in b"abcd" +assert 150 not in b"abcd" +try: + 350 in b"abcd" +except ValueError: + pass diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 03876095c..95df84421 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -181,11 +181,17 @@ impl PyByteInner { Ok(vm.new_bool(false)) } - pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { - self.elements.contains(int); - Ok(vm.new_bool(false)) + pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(int) = int.as_bigint().to_u8() { + if self.elements.contains(&int) { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } else { + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } } } - // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index e052654bf..f376dbc67 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,3 +1,4 @@ +use crate::obj::objint::PyInt; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -143,17 +144,6 @@ impl PyBytesRef { bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), int @ PyInt => self.inner.contains_int(&int, vm), _ => Ok(vm.ctx.not_implemented())) - // if objtype::isinstance(&needle, &vm.ctx.bytes_type()) { - // let result = vec_contains(&self.value, &get_value(&needle)); - // vm.ctx.new_bool(result) - // } else if objtype::isinstance(&needle, &vm.ctx.int_type()) { - // let result = self - // .value - // .contains(&objint::get_value(&needle).to_u8().unwrap()); - // vm.ctx.new_bool(result) - // } else { - // vm.new_type_error(format!("Cannot add {:?} and {:?}", self, needle)) - // } } } From 4c50defa37d5a38e75247d6ac32fd3c680f9e7fc Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 13:46:23 -0600 Subject: [PATCH 195/658] extend PyIterable::try_from_object to work with sequences --- vm/src/pyobject.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index ee765f987..be51128ef 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,4 +1,5 @@ use std::any::Any; +use std::cell::Cell; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; @@ -1043,10 +1044,24 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - Ok(PyIterable { - method: vm.get_method(obj, "__iter__")?, - _item: std::marker::PhantomData, - }) + if let Ok(method) = vm.get_method(obj.clone(), "__iter__") { + Ok(PyIterable { + method: method, + _item: std::marker::PhantomData, + }) + } else if vm.get_method(obj.clone(), "__getitem__").is_ok() { + Self::try_from_object( + vm, + objiter::PySequenceIterator { + position: Cell::new(0), + obj: obj.clone(), + } + .into_ref(vm) + .into_object(), + ) + } else { + Err(vm.new_type_error(format!("'{}' object is not iterable", obj.class().name))) + } } } From 4d53ddedb020c100eeb6e45675aebd572909f9dd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 14:45:34 -0600 Subject: [PATCH 196/658] test for if iter raises error --- tests/snippets/list.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 55c84fdf9..217e79089 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -414,6 +414,7 @@ x[3:8] = (11, 12, 13, 14, 15) assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] # class +# __next__ class CIterNext: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -431,6 +432,7 @@ x = list(range(10)) x[3:8] = CIterNext() assert x == [0, 1, 2, 1, 2, 3, 8, 9] +# __iter__ yield class CIter: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -442,6 +444,7 @@ x = list(range(10)) x[3:8] = CIter() assert x == [0, 1, 2, 1, 2, 3, 8, 9] +# __getitem but no __iter__ sequence class CGetItem: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -451,3 +454,17 @@ class CGetItem: x = list(range(10)) x[3:8] = CGetItem() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# iter raises error +class CIterError: + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + +def bad_iter_assign(): + x = list(range(10)) + x[3:8] = CIterError() + +assert_raises(RuntimeError, bad_iter_assign) From b53595831005023bdfeb6e5c93eb40b3286775f9 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 23:12:40 +0200 Subject: [PATCH 197/658] add getitem, fix contain error_message --- tests/snippets/bytes.py | 11 +++++++++++ vm/src/obj/objbyteinner.rs | 18 +++++++++++++++++- vm/src/obj/objbytes.rs | 12 ++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index b03eb3808..770f4f373 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -66,3 +66,14 @@ try: 350 in b"abcd" except ValueError: pass + + +# getitem +d = b"abcdefghij" + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == b"cdef" +assert d[-6:] == b"efghij" +assert d[1:8:2] == b"bdfh" +assert d[8:1:-2] == b"igec" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 95df84421..bb9966ebf 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -11,6 +11,7 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use super::objint; +use super::objsequence::PySliceableSequence; use crate::obj::objint::PyInt; use num_traits::ToPrimitive; @@ -189,9 +190,24 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } else { - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + Err(vm.new_value_error("byte mu st be in range(0, 256)".to_string())) } } + + pub fn getitem_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + Ok(vm.new_int(self.elements[idx])) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + pub fn getitem_slice(&self, slice: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) + } } + // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f376dbc67..2c15e60cd 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -8,6 +8,7 @@ use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyVa use super::objbyteinner::PyByteInner; use super::objiter; +use super::objslice::PySlice; use super::objtype::PyClassRef; /// "bytes(iterable_of_ints) -> bytes\n\ @@ -139,11 +140,18 @@ impl PyBytesRef { #[pymethod(name = "__contains__")] fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // no new style since objint is not. match_class!(needle, bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), int @ PyInt => self.inner.contains_int(&int, vm), - _ => Ok(vm.ctx.not_implemented())) + obj => Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))) + } + + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(needle, + int @ PyInt => self.inner.getitem_int(&int, vm), + slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), + obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) } } From b2a1f6580b6dcf3eec299ec1423513c682aa18eb Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 15:56:06 -0600 Subject: [PATCH 198/658] fix error when start or stop of slice is -1 and step != 1 --- tests/snippets/list.py | 10 ++++++++++ vm/src/obj/objlist.rs | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 217e79089..dc0582d4c 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -468,3 +468,13 @@ def bad_iter_assign(): x[3:8] = CIterError() assert_raises(RuntimeError, bad_iter_assign) + +# slice assign when step or stop is -1 + +a = list(range(10)) +x = a[:] +x[-1:-5:-1] = ['a', 'b', 'c', 'd'] +assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x = a[:] +x[-5:-1:-1] = [] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 4c3c16383..6de08dfef 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -3,7 +3,7 @@ use std::fmt; use std::ops::Range; -use num_bigint::BigInt; +use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; @@ -238,8 +238,20 @@ impl PyListRef { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = &slice.start.as_ref().map(|x| x + 1); - let stop = &slice.stop.as_ref().map(|x| x + 1); + let start = &slice.start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = &slice.stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); match (-step).to_i32() { Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), @@ -271,7 +283,11 @@ impl PyListRef { sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { - let slicelen = ((range.end - range.start - 1) / step) + 1; + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing let items: Result, _> = sec.iter(vm)?.collect(); @@ -290,6 +306,9 @@ impl PyListRef { n, slicelen ))) } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) } else { // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( @@ -306,7 +325,11 @@ impl PyListRef { sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { - let slicelen = ((range.end - range.start - 1) / step) + 1; + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing @@ -326,6 +349,9 @@ impl PyListRef { n, slicelen ))) } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) } else { // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( From 1101b6571ccded8b05e252b47658448d04382783 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 16:17:57 -0600 Subject: [PATCH 199/658] fix error related to #746 but for list.__delitem__ --- tests/snippets/list.py | 8 ++++++++ vm/src/obj/objlist.rs | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 3d9743c0b..41de1b5fd 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -201,3 +201,11 @@ assert_raises(TypeError, bad_del_1) def bad_del_2(): del ['a', 'b'][2] assert_raises(IndexError, bad_del_2) + +# is step != 1 and start or stop of slice == -1 +x = list(range(10)) +del x[-1:-5:-1] +assert x == [0, 1, 2, 3, 4, 5] +x = list(range(10)) +del x[-5:-1:-1] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 60c14b107..c5278e1fb 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -3,7 +3,7 @@ use std::fmt; use std::ops::Range; -use num_bigint::BigInt; +use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; @@ -411,8 +411,20 @@ impl PyListRef { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { From 422ab69b99af4611cd8d3f416cfe24e94894782c Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 8 Apr 2019 21:50:38 -0500 Subject: [PATCH 200/658] Use impl style classes --- wasm/lib/src/browser_module.rs | 54 ++++++++++++++++------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 607cfc886..d302ec99a 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -10,7 +10,7 @@ use rustpython_vm::obj::{ objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; @@ -161,6 +161,7 @@ fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult Ok(vm.get_none()) } +#[pyclass(name = "Promise")] #[derive(Debug)] pub struct PyPromise { value: Promise, @@ -173,6 +174,7 @@ impl PyValue for PyPromise { } } +#[pyimpl] impl PyPromise { pub fn new(value: Promise) -> PyPromise { PyPromise { value } @@ -187,15 +189,16 @@ impl PyPromise { self.value.clone() } + #[pymethod] fn then( - zelf: PyPromiseRef, + &self, on_fulfill: PyFunctionRef, on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); @@ -220,10 +223,11 @@ impl PyPromise { PyPromise::from_future(ret_future).into_ref(vm) } - fn catch(zelf: PyPromiseRef, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + #[pymethod] + fn catch(&self, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { res.or_else(|err| { let stored_vm = weak_vm .upgrade() @@ -239,11 +243,11 @@ impl PyPromise { } } +#[pyclass] #[derive(Debug)] struct Document { doc: web_sys::Document, } -type DocumentRef = PyRef; impl PyValue for Document { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -251,9 +255,11 @@ impl PyValue for Document { } } +#[pyimpl] impl Document { - fn query(zelf: DocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { - let elem = zelf + #[pymethod] + fn query(&self, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = self .doc .query_selector(&query.value) .map_err(|err| convert::js_py_typeerror(vm, err))?; @@ -265,11 +271,11 @@ impl Document { } } +#[pyclass] #[derive(Debug)] struct Element { elem: web_sys::Element, } -type ElementRef = PyRef; impl PyValue for Element { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -277,26 +283,24 @@ impl PyValue for Element { } } +#[pyimpl] impl Element { + #[pymethod] fn get_attr( - zelf: ElementRef, + &self, attr: PyStringRef, default: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - match zelf.elem.get_attribute(&attr.value) { + match self.elem.get_attribute(&attr.value) { Some(s) => vm.new_str(s), None => default.into_option().unwrap_or_else(|| vm.get_none()), } } - fn set_attr( - zelf: ElementRef, - attr: PyStringRef, - value: PyStringRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.elem + #[pymethod] + fn set_attr(&self, attr: PyStringRef, value: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + self.elem .set_attribute(&attr.value, &value.value) .map_err(|err| convert::js_to_py(vm, err)) } @@ -340,14 +344,9 @@ fn browser_prompt( pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let promise = py_class!(ctx, "Promise", ctx.object(), { - "then" => ctx.new_rustfunc(PyPromise::then), - "catch" => ctx.new_rustfunc(PyPromise::catch), - }); + let promise = PyPromise::make_class(ctx); - let document_class = py_class!(ctx, "Document", ctx.object(), { - "query" => ctx.new_rustfunc(Document::query), - }); + let document_class = Document::make_class(ctx); let document = PyObject::new( Document { @@ -357,10 +356,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { None, ); - let element = py_class!(ctx, "Element", ctx.object(), { - "get_attr" => ctx.new_rustfunc(Element::get_attr), - "set_attr" => ctx.new_rustfunc(Element::set_attr), - }); + let element = Element::make_class(ctx); py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), From 046e4b3fe37648e86ad69436fb7cee26b2a2c3f9 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 9 Apr 2019 20:43:59 +1200 Subject: [PATCH 201/658] Cleanup dict test snippet --- tests/snippets/dict.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f924bb0e7..23d6e933b 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -110,16 +110,10 @@ assert list(x) == ['a', 'b'] y = x.copy() x['c'] = 12 -assert list(y) == ['a', 'b'] -assert y['a'] == 2 -assert y['b'] == 10 +assert dict_eq(y, {'a': 2, 'b': 10}) y.update({'c': 19, "d": -1, 'b': 12}) -assert list(y) == ['a', 'b', 'c', 'd'] -assert y['a'] == 2 -assert y['b'] == 12 -assert y['c'] == 19 -assert y['d'] == -1 +assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) y.update(y) -assert list(y) == ['a', 'b', 'c', 'd'] +assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) # hasn't changed From 23a4b1e9d86bed3acd95ed3917023ace2456fad3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:13:44 +0100 Subject: [PATCH 202/658] Additional tests for dictionary iteration. --- tests/snippets/dict.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f57fc37b6..44d1c382f 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -37,13 +37,36 @@ for key in a.keys(): res.add(key) assert res == set(['a','b']) +# Deleted values are correctly skipped over: x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} del x['c'] it = iter(x.items()) assert ('a', 1) == next(it) assert ('b', 2) == next(it) assert ('d', 3) == next(it) +with assertRaises(StopIteration): + next(it) +# Iterating a dictionary is just its keys: +assert ['a', 'b', 'd'] == list(x) + +# Iterating view captures dictionary when iterated. +data = {1: 2, 3: 4} +items = data.items() +assert list(items) == [(1, 2), (3, 4)] +data[5] = 6 +assert list(items) == [(1, 2), (3, 4), (5, 6)] + +# Values can be changed during iteration. +data = {1: 2, 3: 4} +items = iter(data.items()) +assert (1, 2) == next(items) +data[3] = "changed" +assert (3, "changed") == next(items) + +# View isn't itself an iterator. +with assertRaises(TypeError): + next(data.keys()) x = {} x[1] = 1 From d9216c87f1acde0f0a7f092d943f0ffe8bdebd41 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:17:17 +0100 Subject: [PATCH 203/658] Remove done todo. --- vm/src/obj/objdict.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 612fa2cb7..53445e01f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -360,7 +360,6 @@ pub fn init(context: &PyContext) { "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), - // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), "copy" => context.new_rustfunc(PyDictRef::copy), From 25d3f83e4df7548010c2d090e8fc5193c3d75b08 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:44:04 +0100 Subject: [PATCH 204/658] Add __len__ to dictionary views. --- tests/snippets/dict.py | 2 ++ vm/src/obj/objdict.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 091b03405..45c6c3b43 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -68,6 +68,8 @@ assert (3, "changed") == next(items) with assertRaises(TypeError): next(data.keys()) +assert len(data.keys()) == 2 + x = {} x[1] = 1 assert x[1] == 1 diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 53445e01f..6d2aefd78 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -265,6 +265,11 @@ macro_rules! dict_iterator { fn iter(&self, _vm: &VirtualMachine) -> $iter_name { $iter_name::new(self.dict.clone()) } + + #[pymethod(name = "__len__")] + fn len(&self, vm: &VirtualMachine) -> usize { + self.dict.clone().len(vm) + } } impl PyValue for $name { From ec5fd550debd3c2bbdc646215484dfc81fb3de3f Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 11:25:18 +0100 Subject: [PATCH 205/658] Implement IntoIterator for PyDictRef. --- vm/src/dictdatatype.rs | 12 ------- vm/src/frame.rs | 15 ++++----- vm/src/obj/objdict.rs | 73 ++++++++++++++++++++++++++++++----------- vm/src/obj/objmodule.rs | 6 +--- vm/src/obj/objobject.rs | 2 +- vm/src/stdlib/json.rs | 2 +- 6 files changed, 63 insertions(+), 47 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 0612bc8a3..6dc49125b 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -131,18 +131,6 @@ impl Dict { None } - pub fn iter_items(&self) -> impl Iterator + '_ { - self.entries - .iter() - .filter(|e| e.is_some()) - .map(|e| e.as_ref().unwrap()) - .map(|e| (e.key.clone(), e.value.clone())) - } - - pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { - self.iter_items().collect() - } - /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { let hash_value = calc_hash(vm, key)?; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2570c0934..ff9baba35 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -393,9 +393,8 @@ impl Frame { // Take all key-value pairs from the dict: let dict: PyDictRef = obj.downcast().expect("Need a dictionary to build a map."); - let dict_elements = dict.get_key_value_pairs(); - for (key, value) in dict_elements.iter() { - map_obj.set_item(key.clone(), value.clone(), vm).unwrap(); + for (key, value) in dict { + map_obj.set_item(key, value, vm).unwrap(); } } } else { @@ -636,8 +635,7 @@ impl Frame { let kwargs = if *has_kwargs { let kw_dict: PyDictRef = self.pop_value().downcast().expect("Kwargs must be a dict."); - let dict_elements = kw_dict.get_key_value_pairs(); - dict_elements + kw_dict .into_iter() .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() @@ -862,8 +860,8 @@ impl Frame { // Grab all the names from the module and put them in the context if let Some(dict) = &module.dict { - for (k, v) in dict.get_key_value_pairs().iter() { - self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + for (k, v) in dict { + self.scope.store_name(&vm, &objstr::get_value(&k), v); } } Ok(None) @@ -1230,8 +1228,7 @@ impl fmt::Debug for Frame { .collect::(); let dict = self.scope.get_locals(); let local_str = dict - .get_key_value_pairs() - .iter() + .into_iter() .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) .collect::(); write!( diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 53445e01f..b0d99d4e7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,8 +3,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, - PyValue, + IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -63,9 +62,8 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - let mut dict_borrowed = dict.borrow_mut(); - for (key, value) in dict_obj.entries.borrow().iter_items() { - dict_borrowed.insert(vm, &key, value)?; + for (key, value) in dict_obj { + dict.borrow_mut().insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -105,7 +103,7 @@ impl PyDictRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; - for (key, value) in self.get_key_value_pairs() { + for (key, value) in self { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -146,10 +144,6 @@ impl PyDictRef { PyDictItems::new(self) } - pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - self.entries.borrow().get_items() - } - fn inner_setitem( self, key: PyObjectRef, @@ -194,24 +188,17 @@ impl PyDictRef { fn update( self, - mut dict_obj: OptionalArg, + dict_obj: OptionalArg, kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult<()> { - if let OptionalArg::Present(ref other) = dict_obj { - if self.is(other) { - // Updating yourself is a noop, and this avoids a borrow error - dict_obj = OptionalArg::Missing; - } - } - PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) } /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); - for (key, value) in self.get_key_value_pairs() { + for (key, value) in self { let key = objstr::get_value(&key); attrs.insert(key, value); } @@ -247,6 +234,54 @@ impl ItemProtocol for PyDictRef { } } +// Implement IntoIterator so that we can easily iterate dictionaries from rust code. +impl IntoIterator for PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIterator; + + fn into_iter(self) -> Self::IntoIter { + DictIterator::new(self) + } +} + +impl IntoIterator for &PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIterator; + + fn into_iter(self) -> Self::IntoIter { + DictIterator::new(self.clone()) + } +} + +pub struct DictIterator { + dict: PyDictRef, + position: Cell, +} + +impl DictIterator { + pub fn new(dict: PyDictRef) -> DictIterator { + DictIterator { + dict, + position: Cell::new(0), + } + } +} + +impl Iterator for DictIterator { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option { + self.dict + .entries + .borrow() + .next_entry(self.position.get()) + .map(|(new_position, key, value)| { + self.position.set(new_position); + (key.clone(), value.clone()) + }) + } +} + macro_rules! dict_iterator { ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { #[pyclass(name = $class_name, __inside_vm)] diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index 75749d557..904487e4d 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -17,11 +17,7 @@ impl PyValue for PyModule { impl PyModuleRef { fn dir(self: PyModuleRef, vm: &VirtualMachine) -> PyResult { if let Some(dict) = &self.into_object().dict { - let keys = dict - .get_key_value_pairs() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); + let keys = dict.into_iter().map(|(k, _v)| k.clone()).collect(); Ok(vm.ctx.new_list(keys)) } else { panic!("Modules should definitely have a dict."); diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index a3bc41103..f8e86d0a1 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -237,7 +237,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in dict.get_key_value_pairs() { + for (key, value) in dict.into_iter() { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index f4061bc2a..3fa30e16b 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -64,7 +64,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); - let pairs = dict.get_key_value_pairs(); + let pairs: Vec<(PyObjectRef, PyObjectRef)> = dict.into_iter().collect(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; From 2940c7cc280143de25777198e1bb3631b3f54104 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 14:33:29 +0200 Subject: [PATCH 206/658] add islpaha isalnum isdigit islower isupper isspace istitle --- tests/snippets/bytes.py | 35 +++++++++++++++ vm/src/obj/objbyteinner.rs | 90 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 33 ++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 770f4f373..ceb52921c 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -77,3 +77,38 @@ assert d[2:6] == b"cdef" assert d[-6:] == b"efghij" assert d[1:8:2] == b"bdfh" assert d[8:1:-2] == b"igec" + + +# is_xx methods + +assert bytes(b"1a23").isalnum() +assert not bytes(b"1%a23").isalnum() + +assert bytes(b"abc").isalpha() +assert not bytes(b"abc1").isalpha() + +# travis doesn't like this +# assert bytes(b'xyz').isascii() +# assert not bytes([128, 157, 32]).isascii() + +assert bytes(b"1234567890").isdigit() +assert not bytes(b"12ab").isdigit() + +l = bytes(b"lower") +b = bytes(b"UPPER") + +assert l.islower() +assert not l.isupper() +assert b.isupper() +assert not bytes(b"Super Friends").islower() + +assert bytes(b" \n\t").isspace() +assert not bytes(b"\td\n").isspace() + +assert b.isupper() +assert not b.islower() +assert l.islower() +assert not bytes(b"tuPpEr").isupper() + +assert bytes(b"Is Title Case").istitle() +assert not bytes(b"is Not title casE").istitle() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bb9966ebf..14da2521e 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -207,6 +207,96 @@ impl PyByteInner { .ctx .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) } + + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_alphanumeric()), + )) + } + + pub fn isalpha(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_alphabetic()), + )) + } + + pub fn isascii(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_ascii()), + )) + } + + pub fn isdigit(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_digit(10)), + )) + } + + pub fn islower(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_lowercase()), + )) + } + + pub fn isspace(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_whitespace()), + )) + } + + pub fn isupper(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_uppercase()), + )) + } + + pub fn istitle(&self, vm: &VirtualMachine) -> PyResult { + if self.elements.is_empty() { + return Ok(vm.new_bool(false)); + } + + let mut iter = self.elements.iter().peekable(); + let mut prev_cased = false; + + while let Some(c) = iter.next() { + let current = char::from(*c); + let next = if let Some(k) = iter.peek() { + char::from(**k) + } else if current.is_uppercase() { + return Ok(vm.new_bool(!prev_cased)); + } else { + return Ok(vm.new_bool(prev_cased)); + }; + + let is_cased = current.to_uppercase().next().unwrap() != current + || current.to_lowercase().next().unwrap() != current; + if (is_cased && next.is_uppercase() && !prev_cased) + || (!is_cased && next.is_lowercase()) + { + return Ok(vm.new_bool(false)); + } + + prev_cased = is_cased; + } + + Ok(vm.new_bool(true)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2c15e60cd..83ce3b8fb 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -153,6 +153,39 @@ impl PyBytesRef { slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) } + + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalnum(vm) + } + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalpha(vm) + } + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.isascii(vm) + } + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.isdigit(vm) + } + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.islower(vm) + } + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.isspace(vm) + } + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.isupper(vm) + } + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.istitle(vm) + } } #[derive(Debug)] From 6c745f68dd6d123fc491cc513429ede529fa3a2e Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 14:43:13 +0200 Subject: [PATCH 207/658] fix typo, fix bytesinner.add now return Vec[u8] --- vm/src/obj/objbyteinner.rs | 6 +++--- vm/src/obj/objbytes.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 14da2521e..bcb1f8207 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -55,7 +55,7 @@ impl PyByteInner { } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } - // On ly one argument + // Only one argument } else { let value = if let OptionalArg::Present(ival) = val_option { match_class!(ival.clone(), @@ -161,14 +161,14 @@ impl PyByteInner { hasher.finish() as usize } - pub fn add(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + pub fn add(&self, other: &PyByteInner, _vm: &VirtualMachine) -> Vec { let elements: Vec = self .elements .iter() .chain(other.elements.iter()) .cloned() .collect(); - Ok(vm.ctx.new_bytes(elements)) + elements } pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 83ce3b8fb..0b257cb77 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -134,7 +134,7 @@ impl PyBytesRef { #[pymethod(name = "__add__")] fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, - bytes @ PyBytes => self.inner.add(&bytes.inner, vm), + bytes @ PyBytes => Ok(vm.ctx.new_bytes(self.inner.add(&bytes.inner, vm))), _ => Ok(vm.ctx.not_implemented())) } From 9b763072fe78d8b2dd9fcdeb31a3d1fafcbcc367 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 16:23:56 +0200 Subject: [PATCH 208/658] add upper lower --- tests/snippets/bytes.py | 6 ++++++ vm/src/obj/objbyteinner.rs | 8 ++++++++ vm/src/obj/objbytes.rs | 17 +++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index ceb52921c..3ccb2fc23 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -112,3 +112,9 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() + +# upper lower +l = bytes(b"lower") +b = bytes(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bcb1f8207..4771a91bd 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -297,6 +297,14 @@ impl PyByteInner { Ok(vm.new_bool(true)) } + + pub fn lower(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_lowercase() + } + + pub fn upper(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_uppercase() + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0b257cb77..0be821500 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -158,34 +158,51 @@ impl PyBytesRef { fn isalnum(self, vm: &VirtualMachine) -> PyResult { self.inner.isalnum(vm) } + #[pymethod(name = "isalpha")] fn isalpha(self, vm: &VirtualMachine) -> PyResult { self.inner.isalpha(vm) } + #[pymethod(name = "isascii")] fn isascii(self, vm: &VirtualMachine) -> PyResult { self.inner.isascii(vm) } + #[pymethod(name = "isdigit")] fn isdigit(self, vm: &VirtualMachine) -> PyResult { self.inner.isdigit(vm) } + #[pymethod(name = "islower")] fn islower(self, vm: &VirtualMachine) -> PyResult { self.inner.islower(vm) } + #[pymethod(name = "isspace")] fn isspace(self, vm: &VirtualMachine) -> PyResult { self.inner.isspace(vm) } + #[pymethod(name = "isupper")] fn isupper(self, vm: &VirtualMachine) -> PyResult { self.inner.isupper(vm) } + #[pymethod(name = "istitle")] fn istitle(self, vm: &VirtualMachine) -> PyResult { self.inner.istitle(vm) } + + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.upper(vm))) + } } #[derive(Debug)] From 5cc83a35aa6d9430a8f0d7c793d6cbbad17dfd66 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 17:08:38 +0200 Subject: [PATCH 209/658] hex --- tests/snippets/bytes.py | 3 ++- vm/src/obj/objbyteinner.rs | 9 +++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 3ccb2fc23..bf7906bc1 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -113,8 +113,9 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower hex l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() +assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4771a91bd..6994a58ac 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -305,6 +305,15 @@ impl PyByteInner { pub fn upper(&self, _vm: &VirtualMachine) -> Vec { self.elements.to_ascii_uppercase() } + + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { + let bla = self + .elements + .iter() + .map(|x| format!("{:02x}", x)) + .collect::(); + Ok(vm.ctx.new_str(bla)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0be821500..5859b10b4 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -203,6 +203,11 @@ impl PyBytesRef { fn upper(self, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.upper(vm))) } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.hex(vm) + } } #[derive(Debug)] From cbdab226d995fff81892af6eddd34b9e74f8cf48 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Tue, 9 Apr 2019 22:07:21 -0500 Subject: [PATCH 210/658] Change #[pymethod(property)] to #[pyproperty] --- derive/src/pyclass.rs | 246 ++++++++++++++++++++++++++++-------------- 1 file changed, 165 insertions(+), 81 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index e533e8009..710747291 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,91 +1,138 @@ use super::rustpython_path_attr; -use proc_macro2::{Span, TokenStream as TokenStream2}; +use proc_macro2::TokenStream as TokenStream2; use quote::quote; +use std::collections::HashMap; use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; -enum MethodKind { +#[derive(PartialEq, Clone, Copy)] +enum ClassItemKind { Method, - Property, + PropertyGetter, + PropertySetter, } -impl MethodKind { - fn to_ctx_constructor_fn(&self) -> Ident { - let f = match self { - MethodKind::Method => "new_rustfunc", - MethodKind::Property => "new_property", - }; - Ident::new(f, Span::call_site()) +struct ClassItem { + item_name: Ident, + py_name: String, + kind: ClassItemKind, +} + +fn meta_to_vec(meta: Meta) -> Result, Meta> { + match meta { + Meta::Word(_) => Ok(Vec::new()), + Meta::List(list) => Ok(list.nested.into_iter().collect()), + Meta::NameValue(_) => Err(meta), } } -struct Method { - fn_name: Ident, - py_name: String, - kind: MethodKind, -} - -impl Method { - fn from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { - let mut py_name = None; - let mut kind = MethodKind::Method; - let mut pymethod_to_remove = Vec::new(); - let metas = attrs +impl ClassItem { + fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + let mut item = None; + let mut attr_idx = None; + for (i, meta) in attrs .iter() + .filter_map(|attr| attr.parse_meta().ok()) .enumerate() - .filter_map(|(i, attr)| { - if attr.path.is_ident("pymethod") { - let meta = attr.parse_meta().expect("Invalid attribute"); - // remove #[pymethod] because there's no actual proc macro - // implementation for it - pymethod_to_remove.push(i); - match meta { - Meta::List(list) => Some(list), - Meta::Word(_) => None, - Meta::NameValue(_) => panic!( - "#[pymethod = ...] attribute on a method should be a list, like \ - #[pymethod(...)]" - ), - } - } else { - None + { + let name = meta.name(); + if name == "pymethod" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") } - }) - .flat_map(|attr| attr.nested); - for meta in metas { - if let NestedMeta::Meta(meta) = meta { - match meta { - Meta::NameValue(name_value) => { - if name_value.ident == "name" { - if let Lit::Str(s) = &name_value.lit { - py_name = Some(s.value()); - } else { - panic!("#[pymethod(name = ...)] must be a string"); + let nesteds = meta_to_vec(meta) + .expect("#[pymethod = ...] must be a list, e.g. #[pymethod(...)]"); + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pymethod(name = ...)] must be a string"); + } } } + _ => {} } - Meta::Word(ident) => { - if ident == "property" { - kind = MethodKind::Property - } - } - _ => {} } + item = Some(ClassItem { + item_name: sig.ident.clone(), + py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), + kind: ClassItemKind::Method, + }); + attr_idx = Some(i); + } else if name == "pyproperty" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") + } + let nesteds = meta_to_vec(meta) + .expect("#[pyproperty = ...] must be a list, e.g. #[pyproperty(...)]"); + let mut setter = false; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pyproperty(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => { + if ident == "setter" { + setter = true; + } + } + _ => {} + } + } + let kind = if setter { + ClassItemKind::PropertySetter + } else { + ClassItemKind::PropertyGetter + }; + let py_name = py_name.unwrap_or_else(|| { + let item_name = sig.ident.to_string(); + if item_name.starts_with("set_") { + let name = &item_name["set_".len()..]; + if name.is_empty() { + panic!( + "A #[pyproperty(setter)] fn with a set_* name have something \ + after \"set_\"" + ) + } else { + name.to_string() + } + } else { + panic!( + "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ + fn name along the lines of \"set_*\"" + ) + } + }); + item = Some(ClassItem { + py_name, + item_name: sig.ident.clone(), + kind, + }); + attr_idx = Some(i); } } - // if there are no #[pymethods]s, then it's not a method, so exclude it from - // the final result - if pymethod_to_remove.is_empty() { - return None; + if let Some(attr_idx) = attr_idx { + attrs.remove(attr_idx); } - for i in pymethod_to_remove { - attrs.remove(i); - } - let py_name = py_name.unwrap_or_else(|| sig.ident.to_string()); - Some(Method { - fn_name: sig.ident.clone(), - py_name, - kind, - }) + item } } @@ -98,30 +145,66 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let rp_path = rustpython_path_attr(&attr); - let methods = imp + let items = imp .items .iter_mut() .filter_map(|item| { if let ImplItem::Method(meth) = item { - Method::from_syn(&mut meth.attrs, &meth.sig) + ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig) } else { None } }) .collect::>(); let ty = &imp.self_ty; - let methods = methods.iter().map( - |Method { - py_name, - fn_name, - kind, - }| { - let constructor_fn = kind.to_ctx_constructor_fn(); - quote! { - class.set_str_attr(#py_name, ctx.#constructor_fn(Self::#fn_name)); + let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); + for item in items.iter() { + match item.kind { + ClassItemKind::PropertyGetter => { + let (ref mut getter, _) = properties.entry(&item.py_name).or_default(); + if getter.is_some() { + panic!("Multiple property getters with name {:?}", &item.py_name) + } + *getter = Some(&item.item_name); } - }, - ); + ClassItemKind::PropertySetter => { + let (_, ref mut setter) = properties.entry(&item.py_name).or_default(); + if setter.is_some() { + panic!("Multiple property getters with name {:?}", &item.py_name) + } + *setter = Some(&item.item_name); + } + ClassItemKind::Method => {} + } + } + let methods = items.iter().filter_map(|item| { + if let ClassItemKind::Method = item.kind { + let ClassItem { + py_name, item_name, .. + } = item; + Some(quote! { + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); + }) + } else { + None + } + }); + let properties = properties.iter().map(|(name, prop)| { + let getter = match prop.0 { + Some(getter) => getter, + None => panic!("Property {:?} is missing a getter", name), + }; + let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); + quote! { + class.set_str_attr( + #name, + #rp_path::obj::objproperty::PropertyBuilder + .add_getter(Self::#getter) + #add_setter + .create(), + ); + } + }); quote! { #imp @@ -131,6 +214,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { class: &#rp_path::obj::objtype::PyClassRef, ) { #(#methods)* + #(#properties)* } } } From 977f56ade1bc9f9aca692ff14dcb352b42508e10 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 10 Apr 2019 10:08:10 +0200 Subject: [PATCH 211/658] add bytes.fromhex --- tests/snippets/bytes.py | 14 +++++++++++++- vm/src/obj/objbyteinner.rs | 33 +++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 22 +++++++++++++++++++--- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index bf7906bc1..acd8d9e50 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -113,9 +113,21 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower hex +# upper lower l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() + +# hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +try: + bytes.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytes.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 6994a58ac..7504f2b74 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -314,6 +314,39 @@ impl PyByteInner { .collect::(); Ok(vm.ctx.new_str(bla)) } + + pub fn fromhex(string: String, vm: &VirtualMachine) -> Result, PyObjectRef> { + // first check for invalid character + for (i, c) in string.char_indices() { + if !c.is_digit(16) && !c.is_whitespace() { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + i + ))); + } + } + + // strip white spaces + let stripped = string.split_whitespace().collect::(); + + // Hex is evaluated on 2 digits + if stripped.len() % 2 != 0 { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + stripped.len() - 1 + ))); + } + + // parse even string + Ok(stripped + .chars() + .collect::>() + .chunks(2) + .map(|x| x.to_vec().iter().collect::()) + .map(|x| u8::from_str_radix(&x, 16)) + .map(|x| x.unwrap()) + .collect::>()) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 5859b10b4..2c8edca6a 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,5 @@ use crate::obj::objint::PyInt; +use crate::obj::objstr::PyString; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -55,11 +56,15 @@ pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a pub fn init(context: &PyContext) { PyBytesRef::extend_class(context, &context.bytes_type); + let bytes_type = &context.bytes_type; + extend_class!(context, bytes_type, { +"fromhex" => context.new_rustfunc(PyBytesRef::fromhex), +}); let bytesiterator_type = &context.bytesiterator_type; extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), - }); +"__next__" => context.new_rustfunc(PyBytesIteratorRef::next), +"__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), +}); } #[pyimpl(__inside_vm)] @@ -208,6 +213,17 @@ impl PyBytesRef { fn hex(self, vm: &VirtualMachine) -> PyResult { self.inner.hex(vm) } + + // #[pymethod(name = "fromhex")] + fn fromhex(string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(string, + s @ PyString => { + match PyByteInner::fromhex(s.to_string(), vm) { + Ok(x) => Ok(vm.ctx.new_bytes(x)), + Err(y) => Err(y)}}, + obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) + ) + } } #[derive(Debug)] From 9a92159413a9ef6de556e45e47bdea14fbe43dd0 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:26:03 +0100 Subject: [PATCH 212/658] Remove unneeded Cell on DictIter. --- vm/src/obj/objdict.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index b0d99d4e7..35925ff35 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -255,15 +255,12 @@ impl IntoIterator for &PyDictRef { pub struct DictIterator { dict: PyDictRef, - position: Cell, + position: usize, } impl DictIterator { pub fn new(dict: PyDictRef) -> DictIterator { - DictIterator { - dict, - position: Cell::new(0), - } + DictIterator { dict, position: 0 } } } @@ -271,14 +268,13 @@ impl Iterator for DictIterator { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { - self.dict - .entries - .borrow() - .next_entry(self.position.get()) - .map(|(new_position, key, value)| { - self.position.set(new_position); - (key.clone(), value.clone()) - }) + match self.dict.entries.borrow().next_entry(self.position) { + Some((new_position, key, value)) => { + self.position = new_position; + Some((key.clone(), value.clone())) + } + None => None, + } } } From 8f840d537610cb05e545a3545feb2011e230a095 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:26:18 +0100 Subject: [PATCH 213/658] Pick code review nits. --- vm/src/obj/objdict.rs | 18 +++++++++--------- vm/src/obj/objobject.rs | 2 +- vm/src/stdlib/json.rs | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 35925ff35..753baf786 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -237,34 +237,34 @@ impl ItemProtocol for PyDictRef { // Implement IntoIterator so that we can easily iterate dictionaries from rust code. impl IntoIterator for PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterator; + type IntoIter = DictIter; fn into_iter(self) -> Self::IntoIter { - DictIterator::new(self) + DictIter::new(self) } } impl IntoIterator for &PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterator; + type IntoIter = DictIter; fn into_iter(self) -> Self::IntoIter { - DictIterator::new(self.clone()) + DictIter::new(self.clone()) } } -pub struct DictIterator { +pub struct DictIter { dict: PyDictRef, position: usize, } -impl DictIterator { - pub fn new(dict: PyDictRef) -> DictIterator { - DictIterator { dict, position: 0 } +impl DictIter { + pub fn new(dict: PyDictRef) -> DictIter { + DictIter { dict, position: 0 } } } -impl Iterator for DictIterator { +impl Iterator for DictIter { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index f8e86d0a1..2be4ff4f4 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -237,7 +237,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in dict.into_iter() { + for (key, value) in dict { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 3fa30e16b..dceec8a7d 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -64,7 +64,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); - let pairs: Vec<(PyObjectRef, PyObjectRef)> = dict.into_iter().collect(); + let pairs: Vec<_> = dict.into_iter().collect(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; From e04275b1999d42749c92d029a109cfc2aa6699c7 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:48:53 +0100 Subject: [PATCH 214/658] Missed get_key_values_pairs removal in wasm. --- wasm/lib/src/browser_module.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index d302ec99a..478ebb9c5 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -83,7 +83,7 @@ fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyRe if let Some(headers) = headers { let h = request.headers(); - for (key, value) in headers.get_key_value_pairs() { + for (key, value) in headers { let key = vm.to_str(&key)?; let value = vm.to_str(&value)?; h.set(key.as_str(), value.as_str()) From 751c3d52e2a4deccd0443b84c0c70329a2b76f02 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Wed, 10 Apr 2019 13:24:31 +0200 Subject: [PATCH 215/658] use vec! for int arg in bytes new --- tests/snippets/bytes.py | 6 +++++- vm/src/obj/objbyteinner.rs | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index acd8d9e50..0a3521df3 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -4,8 +4,8 @@ from testutils import assertRaises assert bytes([1, 2, 3]) assert bytes((1, 2, 3)) assert bytes(range(4)) -assert b"bla" assert bytes(3) +assert b"bla" assert bytes("bla", "utf8") try: bytes("bla") @@ -14,6 +14,9 @@ except TypeError: else: assert False + + + a = b"abcd" b = b"ab" c = b"abcd" @@ -49,6 +52,7 @@ hash(a) == hash(b"abcd") # iter [i for i in b"abcd"] == ["a", "b", "c", "d"] +assert list(bytes(3)) == [0,0,0] # add assert a + b == b"abcdab" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 7504f2b74..fac093762 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -61,11 +61,7 @@ impl PyByteInner { match_class!(ival.clone(), i @ PyInt => { let size = objint::get_value(&i.into_object()).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, + Ok(vec![0; size])}, _l @ PyString=> {return Err(vm.new_type_error(format!( "string argument without an encoding" )));}, From 54c7335f1c32c0215b81f856cac26218a1223f5e Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 10 Apr 2019 18:05:28 +0200 Subject: [PATCH 216/658] SyntaxError for non ascii char --- parser/src/lexer.rs | 8 ++++++-- tests/snippets/bytes.py | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index edf21c973..561ea4d77 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -540,8 +540,12 @@ where let end_pos = self.get_pos(); let tok = if is_bytes { - Tok::Bytes { - value: string_content.as_bytes().to_vec(), + if string_content.is_ascii() { + Tok::Bytes { + value: string_content.as_bytes().to_vec(), + } + } else { + return Err(LexicalError::StringError); } } else { Tok::String { diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 0a3521df3..816eae495 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -11,10 +11,6 @@ try: bytes("bla") except TypeError: assert True -else: - assert False - - a = b"abcd" From 9d25a216de4d8240f72eb6f347228dc9b31269c1 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Wed, 10 Apr 2019 21:39:16 +0200 Subject: [PATCH 217/658] fix some tests, run clippy --- tests/snippets/bytes.py | 12 ++++-------- vm/src/obj/objbyteinner.rs | 30 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 816eae495..0ffbc29f8 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -7,10 +7,8 @@ assert bytes(range(4)) assert bytes(3) assert b"bla" assert bytes("bla", "utf8") -try: +with assertRaises(TypeError): bytes("bla") -except TypeError: - assert True a = b"abcd" @@ -48,7 +46,7 @@ hash(a) == hash(b"abcd") # iter [i for i in b"abcd"] == ["a", "b", "c", "d"] -assert list(bytes(3)) == [0,0,0] +assert list(bytes(3)) == [0, 0, 0] # add assert a + b == b"abcdab" @@ -62,10 +60,8 @@ assert b"d" in b"abcd" assert b"dc" not in b"abcd" assert 97 in b"abcd" assert 150 not in b"abcd" -try: +with assertRaises(ValueError): 350 in b"abcd" -except ValueError: - pass # getitem @@ -113,7 +109,7 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index fac093762..532504b02 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -31,8 +31,8 @@ impl PyByteInner { if let OptionalArg::Present(eval) = val_option { if let Ok(input) = eval.downcast::() { if let Ok(encoding) = enc.clone().downcast::() { - if encoding.value.to_lowercase() == "utf8".to_string() - || encoding.value.to_lowercase() == "utf-8".to_string() + if &encoding.value.to_lowercase() == "utf8" + || &encoding.value.to_lowercase() == "utf-8" // TODO: different encoding { return Ok(PyByteInner { @@ -62,12 +62,10 @@ impl PyByteInner { i @ PyInt => { let size = objint::get_value(&i.into_object()).to_usize().unwrap(); Ok(vec![0; size])}, - _l @ PyString=> {return Err(vm.new_type_error(format!( - "string argument without an encoding" - )));}, + _l @ PyString=> {return Err(vm.new_type_error("string argument without an encoding".to_string()));}, obj => { - let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( - "cannot convert {} object to bytes", obj.class().name)));}); + let elements = vm.extract_elements(&obj).or_else(|_| {Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)))}); let mut data_bytes = vec![]; for elem in elements.unwrap(){ @@ -111,6 +109,10 @@ impl PyByteInner { self.elements.len() } + pub fn is_empty(&self) -> bool { + self.elements.len() == 0 + } + pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { if self.elements == other.elements { Ok(vm.new_bool(true)) @@ -169,10 +171,11 @@ impl PyByteInner { pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { for (n, i) in self.elements.iter().enumerate() { - if n + other.len() <= self.len() && *i == other.elements[0] { - if &self.elements[n..n + other.len()] == other.elements.as_slice() { - return Ok(vm.new_bool(true)); - } + if n + other.len() <= self.len() + && *i == other.elements[0] + && &self.elements[n..n + other.len()] == other.elements.as_slice() + { + return Ok(vm.new_bool(true)); } } Ok(vm.new_bool(false)) @@ -186,7 +189,7 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } else { - Err(vm.new_value_error("byte mu st be in range(0, 256)".to_string())) + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) } } @@ -344,6 +347,3 @@ impl PyByteInner { .collect::>()) } } - -// TODO -// fix b"é" not allowed should be bytes("é", "utf8") From 47bd9680f9c473129d331b22f549bc40e18a0a56 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:41:25 -0500 Subject: [PATCH 218/658] Change ClassItem to enum --- derive/src/pyclass.rs | 64 ++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 710747291..3415da181 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -4,17 +4,16 @@ use quote::quote; use std::collections::HashMap; use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; -#[derive(PartialEq, Clone, Copy)] -enum ClassItemKind { - Method, - PropertyGetter, - PropertySetter, -} - -struct ClassItem { - item_name: Ident, - py_name: String, - kind: ClassItemKind, +enum ClassItem { + Method { + item_name: Ident, + py_name: String, + }, + Property { + item_name: Ident, + py_name: String, + setter: bool, + }, } fn meta_to_vec(meta: Meta) -> Result, Meta> { @@ -60,10 +59,9 @@ impl ClassItem { _ => {} } } - item = Some(ClassItem { + item = Some(ClassItem::Method { item_name: sig.ident.clone(), py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), - kind: ClassItemKind::Method, }); attr_idx = Some(i); } else if name == "pyproperty" { @@ -97,11 +95,6 @@ impl ClassItem { _ => {} } } - let kind = if setter { - ClassItemKind::PropertySetter - } else { - ClassItemKind::PropertyGetter - }; let py_name = py_name.unwrap_or_else(|| { let item_name = sig.ident.to_string(); if item_name.starts_with("set_") { @@ -121,10 +114,10 @@ impl ClassItem { ) } }); - item = Some(ClassItem { + item = Some(ClassItem::Property { py_name, item_name: sig.ident.clone(), - kind, + setter, }); attr_idx = Some(i); } @@ -159,29 +152,24 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let ty = &imp.self_ty; let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); for item in items.iter() { - match item.kind { - ClassItemKind::PropertyGetter => { - let (ref mut getter, _) = properties.entry(&item.py_name).or_default(); - if getter.is_some() { - panic!("Multiple property getters with name {:?}", &item.py_name) + match item { + ClassItem::Property { + item_name, + py_name, + setter, + } => { + let entry = properties.entry(py_name).or_default(); + let func = if *setter { &mut entry.1 } else { &mut entry.0 }; + if func.is_some() { + panic!("Multiple property accessors with name {:?}", py_name) } - *getter = Some(&item.item_name); + *func = Some(item_name); } - ClassItemKind::PropertySetter => { - let (_, ref mut setter) = properties.entry(&item.py_name).or_default(); - if setter.is_some() { - panic!("Multiple property getters with name {:?}", &item.py_name) - } - *setter = Some(&item.item_name); - } - ClassItemKind::Method => {} + _ => {} } } let methods = items.iter().filter_map(|item| { - if let ClassItemKind::Method = item.kind { - let ClassItem { - py_name, item_name, .. - } = item; + if let ClassItem::Method { item_name, py_name } = item { Some(quote! { class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); }) From d099165e3bf2b5e53bf41d087482bb05dd2800f6 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:41:45 -0500 Subject: [PATCH 219/658] Rename item_name -> item_ident --- derive/src/pyclass.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 3415da181..892b298b8 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -6,11 +6,11 @@ use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, enum ClassItem { Method { - item_name: Ident, + item_ident: Ident, py_name: String, }, Property { - item_name: Ident, + item_ident: Ident, py_name: String, setter: bool, }, @@ -60,7 +60,7 @@ impl ClassItem { } } item = Some(ClassItem::Method { - item_name: sig.ident.clone(), + item_ident: sig.ident.clone(), py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), }); attr_idx = Some(i); @@ -96,9 +96,9 @@ impl ClassItem { } } let py_name = py_name.unwrap_or_else(|| { - let item_name = sig.ident.to_string(); - if item_name.starts_with("set_") { - let name = &item_name["set_".len()..]; + let item_ident = sig.ident.to_string(); + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; if name.is_empty() { panic!( "A #[pyproperty(setter)] fn with a set_* name have something \ @@ -116,7 +116,7 @@ impl ClassItem { }); item = Some(ClassItem::Property { py_name, - item_name: sig.ident.clone(), + item_ident: sig.ident.clone(), setter, }); attr_idx = Some(i); @@ -154,7 +154,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { for item in items.iter() { match item { ClassItem::Property { - item_name, + item_ident, py_name, setter, } => { @@ -163,15 +163,19 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { if func.is_some() { panic!("Multiple property accessors with name {:?}", py_name) } - *func = Some(item_name); + *func = Some(item_ident); } _ => {} } } let methods = items.iter().filter_map(|item| { - if let ClassItem::Method { item_name, py_name } = item { + if let ClassItem::Method { + item_ident, + py_name, + } = item + { Some(quote! { - class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident)); }) } else { None From 2bdaa02700f88691826ad7edd666d632e1d1d26d Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:45:07 -0500 Subject: [PATCH 220/658] Improve error messages --- derive/src/pyclass.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 892b298b8..2387b4d79 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -28,6 +28,7 @@ impl ClassItem { fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { let mut item = None; let mut attr_idx = None; + // TODO: better error handling throughout this for (i, meta) in attrs .iter() .filter_map(|attr| attr.parse_meta().ok()) @@ -38,13 +39,15 @@ impl ClassItem { if item.is_some() { panic!("You can only have one #[py*] attribute on an impl item") } - let nesteds = meta_to_vec(meta) - .expect("#[pymethod = ...] must be a list, e.g. #[pymethod(...)]"); + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); let mut py_name = None; for meta in nesteds { let meta = match meta { NestedMeta::Meta(meta) => meta, - NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + NestedMeta::Literal(_) => continue, }; match meta { Meta::NameValue(name_value) => { @@ -68,14 +71,16 @@ impl ClassItem { if item.is_some() { panic!("You can only have one #[py*] attribute on an impl item") } - let nesteds = meta_to_vec(meta) - .expect("#[pyproperty = ...] must be a list, e.g. #[pyproperty(...)]"); + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); let mut setter = false; let mut py_name = None; for meta in nesteds { let meta = match meta { NestedMeta::Meta(meta) => meta, - NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + NestedMeta::Literal(_) => continue, }; match meta { Meta::NameValue(name_value) => { From c5f4eab1424e4b88493a98bf904e350c55982ebe Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:55:24 -0500 Subject: [PATCH 221/658] Change objint to impl style with properties --- derive/src/pyclass.rs | 28 ++++++++++++++++------------ vm/src/obj/objint.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 2387b4d79..433d5911e 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -102,21 +102,25 @@ impl ClassItem { } let py_name = py_name.unwrap_or_else(|| { let item_ident = sig.ident.to_string(); - if item_ident.starts_with("set_") { - let name = &item_ident["set_".len()..]; - if name.is_empty() { - panic!( - "A #[pyproperty(setter)] fn with a set_* name have something \ - after \"set_\"" - ) + if setter { + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; + if name.is_empty() { + panic!( + "A #[pyproperty(setter)] fn with a set_* name have something \ + after \"set_\"" + ) + } else { + name.to_string() + } } else { - name.to_string() - } - } else { - panic!( + panic!( "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ fn name along the lines of \"set_*\"" ) + } + } else { + item_ident } }); item = Some(ClassItem::Property { @@ -195,7 +199,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { class.set_str_attr( #name, - #rp_path::obj::objproperty::PropertyBuilder + #rp_path::obj::objproperty::PropertyBuilder::new(ctx) .add_getter(Self::#getter) #add_setter .create(), diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index de211668f..789d140d6 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -17,6 +17,7 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; +#[pyclass(__inside_vm)] #[derive(Debug)] pub struct PyInt { value: BigInt, @@ -95,7 +96,9 @@ impl_try_from_object_int!( (u64, to_u64), ); +#[pyimpl(__inside_vm)] impl PyInt { + #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) @@ -104,6 +107,7 @@ impl PyInt { } } + #[pymethod(name = "__ne__")] fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) @@ -112,6 +116,7 @@ impl PyInt { } } + #[pymethod(name = "__lt__")] fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) @@ -120,6 +125,7 @@ impl PyInt { } } + #[pymethod(name = "__le__")] fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) @@ -128,6 +134,7 @@ impl PyInt { } } + #[pymethod(name = "__gt__")] fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) @@ -136,6 +143,7 @@ impl PyInt { } } + #[pymethod(name = "__ge__")] fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) @@ -144,6 +152,7 @@ impl PyInt { } } + #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) @@ -152,6 +161,7 @@ impl PyInt { } } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) @@ -160,6 +170,7 @@ impl PyInt { } } + #[pymethod(name = "__rsub__")] fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) @@ -168,6 +179,7 @@ impl PyInt { } } + #[pymethod(name = "__mul__")] fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) @@ -176,6 +188,7 @@ impl PyInt { } } + #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) @@ -184,6 +197,7 @@ impl PyInt { } } + #[pymethod(name = "__rtruediv__")] fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) @@ -192,6 +206,7 @@ impl PyInt { } } + #[pymethod(name = "__floordiv__")] fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -205,6 +220,7 @@ impl PyInt { } } + #[pymethod(name = "__lshift__")] fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); @@ -224,6 +240,7 @@ impl PyInt { } } + #[pymethod(name = "__rshift__")] fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); @@ -243,6 +260,7 @@ impl PyInt { } } + #[pymethod(name = "__xor__")] fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) @@ -251,6 +269,7 @@ impl PyInt { } } + #[pymethod(name = "__rxor__")] fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) ^ (&self.value)) @@ -259,6 +278,7 @@ impl PyInt { } } + #[pymethod(name = "__or__")] fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) @@ -267,6 +287,7 @@ impl PyInt { } } + #[pymethod(name = "__and__")] fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -276,6 +297,7 @@ impl PyInt { } } + #[pymethod(name = "__pow__")] fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); @@ -288,6 +310,7 @@ impl PyInt { } } + #[pymethod(name = "__mod__")] fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -301,6 +324,7 @@ impl PyInt { } } + #[pymethod(name = "__divmod__")] fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -317,20 +341,24 @@ impl PyInt { } } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } + #[pymethod(name = "__hash__")] fn hash(&self, _vm: &VirtualMachine) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() } + #[pymethod(name = "__abs__")] fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } + #[pymethod(name = "__round__")] fn round( zelf: PyRef, _precision: OptionalArg, @@ -339,14 +367,17 @@ impl PyInt { zelf } + #[pymethod(name = "__int__")] fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__pos__")] fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__float__")] fn float(&self, vm: &VirtualMachine) -> PyResult { self.value .to_f64() @@ -354,30 +385,37 @@ impl PyInt { .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) } + #[pymethod(name = "__trunc__")] fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__floor__")] fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__ceil__")] fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__index__")] fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__invert__")] fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } + #[pymethod(name = "__repr__")] fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } + #[pymethod(name = "__format__")] fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { @@ -386,22 +424,27 @@ impl PyInt { } } + #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } + #[pymethod] fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } + #[pymethod] fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pyproperty] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pyproperty] fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } From dd4539b1a601f33ac77b1fdfdc74b909423dff73 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 18:02:30 -0500 Subject: [PATCH 222/658] Change to use PyClassImpl::extend_class --- vm/src/obj/objint.rs | 80 ++++++++++---------------------------------- 1 file changed, 18 insertions(+), 62 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 789d140d6..a51908209 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -8,7 +8,8 @@ use num_traits::{Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::OptionalArg; use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; @@ -17,6 +18,20 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; +/// int(x=0) -> integer +/// int(x, base=10) -> integer +/// +/// Convert a number or string to an integer, or return 0 if no arguments +/// are given. If x is a number, return x.__int__(). For floating point +/// numbers, this truncates towards zero. +/// +/// If x is not a number or if base is given, then x must be a string, +/// bytes, or bytearray instance representing an integer literal in the +/// given base. The literal can be preceded by '+' or '-' and be surrounded +/// by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. +/// Base 0 means to interpret the base from the string as an integer literal. +/// >>> int('0b100', base=0) +/// 4 #[pyclass(__inside_vm)] #[derive(Debug)] pub struct PyInt { @@ -543,68 +558,9 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { } } -#[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let int_doc = "int(x=0) -> integer -int(x, base=10) -> integer - -Convert a number or string to an integer, or return 0 if no arguments -are given. If x is a number, return x.__int__(). For floating point -numbers, this truncates towards zero. - -If x is not a number or if base is given, then x must be a string, -bytes, or bytearray instance representing an integer literal in the -given base. The literal can be preceded by '+' or '-' and be surrounded -by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. -Base 0 means to interpret the base from the string as an integer literal. ->>> int('0b100', base=0) -4"; - let int_type = &context.int_type; - extend_class!(context, int_type, { - "__doc__" => context.new_str(int_doc.to_string()), - "__eq__" => context.new_rustfunc(PyInt::eq), - "__ne__" => context.new_rustfunc(PyInt::ne), - "__lt__" => context.new_rustfunc(PyInt::lt), - "__le__" => context.new_rustfunc(PyInt::le), - "__gt__" => context.new_rustfunc(PyInt::gt), - "__ge__" => context.new_rustfunc(PyInt::ge), - "__abs__" => context.new_rustfunc(PyInt::abs), - "__add__" => context.new_rustfunc(PyInt::add), - "__radd__" => context.new_rustfunc(PyInt::add), - "__and__" => context.new_rustfunc(PyInt::and), - "__divmod__" => context.new_rustfunc(PyInt::divmod), - "__float__" => context.new_rustfunc(PyInt::float), - "__round__" => context.new_rustfunc(PyInt::round), - "__ceil__" => context.new_rustfunc(PyInt::ceil), - "__floor__" => context.new_rustfunc(PyInt::floor), - "__index__" => context.new_rustfunc(PyInt::index), - "__trunc__" => context.new_rustfunc(PyInt::trunc), - "__int__" => context.new_rustfunc(PyInt::int), - "__floordiv__" => context.new_rustfunc(PyInt::floordiv), - "__hash__" => context.new_rustfunc(PyInt::hash), - "__lshift__" => context.new_rustfunc(PyInt::lshift), - "__rshift__" => context.new_rustfunc(PyInt::rshift), + PyInt::extend_class(context, &context.int_type); + extend_class!(context, &context.int_type, { "__new__" => context.new_rustfunc(int_new), - "__mod__" => context.new_rustfunc(PyInt::mod_), - "__mul__" => context.new_rustfunc(PyInt::mul), - "__rmul__" => context.new_rustfunc(PyInt::mul), - "__or__" => context.new_rustfunc(PyInt::or), - "__neg__" => context.new_rustfunc(PyInt::neg), - "__pos__" => context.new_rustfunc(PyInt::pos), - "__pow__" => context.new_rustfunc(PyInt::pow), - "__repr__" => context.new_rustfunc(PyInt::repr), - "__sub__" => context.new_rustfunc(PyInt::sub), - "__rsub__" => context.new_rustfunc(PyInt::rsub), - "__format__" => context.new_rustfunc(PyInt::format), - "__truediv__" => context.new_rustfunc(PyInt::truediv), - "__rtruediv__" => context.new_rustfunc(PyInt::rtruediv), - "__xor__" => context.new_rustfunc(PyInt::xor), - "__rxor__" => context.new_rustfunc(PyInt::rxor), - "__bool__" => context.new_rustfunc(PyInt::bool), - "__invert__" => context.new_rustfunc(PyInt::invert), - "bit_length" => context.new_rustfunc(PyInt::bit_length), - "conjugate" => context.new_rustfunc(PyInt::conjugate), - "real" => context.new_property(PyInt::real), - "imag" => context.new_property(PyInt::imag) }); } From 6a190e39f348db73c76fc875fcdedb9211659514 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 20:41:06 -0500 Subject: [PATCH 223/658] Fix __radd__ and __rmul__ --- vm/src/obj/objint.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index a51908209..5be50f6c4 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -176,6 +176,11 @@ impl PyInt { } } + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.add(other, vm) + } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { @@ -203,6 +208,11 @@ impl PyInt { } } + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.mul(other, vm) + } + #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { From bd5772d9142f8bbd93e65a6ca4ec4cb214bbc4ac Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 12:03:02 +0100 Subject: [PATCH 224/658] Implement dict.__eq__ --- tests/snippets/dict.py | 26 ++++++++++++++------------ vm/src/obj/objdict.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 45c6c3b43..6507efb19 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,21 +1,23 @@ from testutils import assertRaises -def dict_eq(d1, d2): - return (all(k in d2 and d1[k] == d2[k] for k in d1) - and all(k in d1 and d1[k] == d2[k] for k in d2)) +assert dict(a=2, b=3) == {'a': 2, 'b': 3} +assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} +assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} - -assert dict_eq(dict(a=2, b=3), {'a': 2, 'b': 3}) -assert dict_eq(dict({'a': 2, 'b': 3}, b=4), {'a': 2, 'b': 4}) -assert dict_eq(dict([('a', 2), ('b', 3)]), {'a': 2, 'b': 3}) +assert {} == {} +assert not {'a': 2} == {} +assert not {} == {'a': 2} +assert not {'b': 2} == {'a': 2} +assert not {'a': 4} == {'a': 2} +assert {'a': 2} == {'a': 2} a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) c['d'] = 3 c['a']['g'] = 2 -assert dict_eq(a, {'g': 2}) -assert dict_eq(b, {'a': a, 'd': 9}) +assert a == {'g': 2} +assert b == {'a': a, 'd': 9} a.clear() assert len(a) == 0 @@ -142,10 +144,10 @@ assert list(x) == ['a', 'b'] y = x.copy() x['c'] = 12 -assert dict_eq(y, {'a': 2, 'b': 10}) +assert y == {'a': 2, 'b': 10} y.update({'c': 19, "d": -1, 'b': 12}) -assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} y.update(y) -assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) # hasn't changed +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4a503fdcf..8e34ca738 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -7,6 +7,7 @@ use crate::pyobject::{ }; use crate::vm::{ReprGuard, VirtualMachine}; +use super::objbool; use super::objiter; use super::objstr; use crate::dictdatatype; @@ -96,6 +97,35 @@ impl PyDictRef { !self.entries.borrow().is_empty() } + fn inner_eq(self, other: &PyDict, vm: &VirtualMachine) -> PyResult { + if other.entries.borrow().len() != self.entries.borrow().len() { + return Ok(false); + } + for (k, v1) in self { + match other.entries.borrow().get(vm, &k)? { + Some(v2) => { + let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; + if !value { + return Ok(false); + } + } + None => { + return Ok(false); + } + } + } + return Ok(true); + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + let eq = self.inner_eq(other, vm)?; + Ok(vm.ctx.new_bool(eq)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + fn len(self, _vm: &VirtualMachine) -> usize { self.entries.borrow().len() } @@ -387,6 +417,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__eq__" => context.new_rustfunc(PyDictRef::eq), "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), "__new__" => context.new_rustfunc(PyDictRef::new), From b70f989952f99b535750e34b904f0ea88d32ab7d Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 08:39:03 +0100 Subject: [PATCH 225/658] dict equality - objects always equal themselves. --- tests/snippets/dict.py | 3 +++ vm/src/obj/objdict.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 6507efb19..4c35af2f4 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -11,6 +11,9 @@ assert not {'b': 2} == {'a': 2} assert not {'a': 4} == {'a': 2} assert {'a': 2} == {'a': 2} +nan = float('nan') +assert {'a': nan} == {'a': nan} + a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 8e34ca738..be85c7b2f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,7 +3,8 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, + PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -104,6 +105,9 @@ impl PyDictRef { for (k, v1) in self { match other.entries.borrow().get(vm, &k)? { Some(v2) => { + if v1.is(&v2) { + continue; + } let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; if !value { return Ok(false); From 84eff4a7ac8a0010f6ee151637b1f5a90da4d566 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 10 Apr 2019 20:56:40 +1200 Subject: [PATCH 226/658] Add __name__ and __module__ to functions --- tests/snippets/class.py | 8 ++++++++ tests/snippets/function.py | 17 +++++++++++++++++ vm/src/frame.rs | 10 +++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 9dbce566d..0acd38fce 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -1,3 +1,6 @@ +__name__ = "class" + + class Foo: def __init__(self, x): assert x == 5 @@ -14,6 +17,11 @@ foo = Foo(5) assert foo.y == Foo.y assert foo.x == 5 assert foo.square() == 25 +assert Foo.__name__ == "Foo" +assert Foo.__qualname__ == "Foo" +assert Foo.__module__ == "class" +assert Foo.square.__name__ == "square" +assert Foo.square.__qualname__ == "Foo.square" class Bar: diff --git a/tests/snippets/function.py b/tests/snippets/function.py index e4d0ab544..b499c6776 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,9 +1,16 @@ + +__name__ = "function" + + def foo(): """test""" return 42 assert foo() == 42 assert foo.__doc__ == "test" +assert foo.__name__ == "foo" +assert foo.__qualname__ == "foo" +assert foo.__module__ == "function" def my_func(a,): return a+2 @@ -55,3 +62,13 @@ def f5(): assert f5.__doc__ == 'abcw00t', f5.__doc__ + +def f6(): + def nested(): + pass + + assert nested.__name__ == "nested" + assert nested.__qualname__ == "f6..nested" + + +f6() diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..d614691c9 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -566,7 +566,7 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let _qualified_name = self.pop_value(); + let qualified_name = self.pop_value(); let code_obj = self .pop_value() .downcast() @@ -606,6 +606,14 @@ impl Frame { .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); + vm.set_attr(&obj, "__name__", qualified_name.clone())?; + vm.set_attr(&obj, "__qualname__", qualified_name)?; + let module = self + .scope + .globals + .get_item_option("__name__", vm)? + .unwrap_or_else(|| vm.get_none()); + vm.set_attr(&obj, "__module__", module)?; vm.set_attr(&obj, "__annotations__", annotations)?; self.push_value(obj); From 2164cb5e65946c1eda326fdc6a5b8ba7817e917a Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 11 Apr 2019 20:11:26 +1200 Subject: [PATCH 227/658] Produce correct value for __qualname__ and add attributes to class. --- tests/snippets/class.py | 2 +- vm/src/builtins.rs | 28 +++++++++++++++++++--------- vm/src/compile.rs | 31 +++++++++++++++++++++++++++++-- vm/src/frame.rs | 14 ++++++++------ vm/src/function.rs | 14 ++++++++++++++ 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 0acd38fce..ded02dbfa 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -22,7 +22,7 @@ assert Foo.__qualname__ == "Foo" assert Foo.__module__ == "class" assert Foo.square.__name__ == "square" assert Foo.square.__qualname__ == "Foo.square" - +assert Foo.square.__module__ == "class" class Bar: """ W00t """ diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 27e024aaf..6a94b8362 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -18,7 +18,7 @@ use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; -use crate::function::{Args, OptionalArg, PyFuncArgs}; +use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -783,11 +783,17 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { }); } -pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { - let function = args.shift(); - let name_arg = args.shift(); - let bases = args.args.clone(); - let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { +pub fn builtin_build_class_( + function: PyObjectRef, + qualified_name: PyStringRef, + bases: Args, + mut kwargs: KwArgs, + vm: &VirtualMachine, +) -> PyResult { + let name = qualified_name.value.split('.').next_back().unwrap(); + let name_obj = vm.new_str(name.to_string()); + + let mut metaclass = if let Some(metaclass) = kwargs.pop_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? } else { vm.get_type() @@ -801,21 +807,25 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu } } - let bases = vm.context().new_tuple(bases); + let bases = bases.into_tuple(vm); // Prepare uses full __getattribute__ resolution chain. let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; - let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; + let namespace = vm.invoke(prepare, vec![name_obj.clone(), bases.clone()])?; let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?; let cells = vm.ctx.new_dict(); vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; + + namespace.set_item("__name__", name_obj.clone(), vm)?; + namespace.set_item("__qualname__", qualified_name.into_object(), vm)?; + let class = vm.call_method( metaclass.as_object(), "__call__", - vec![name_arg, bases, namespace.into_object()], + vec![name_obj, bases, namespace.into_object()], )?; cells.set_item("__class__", class.clone(), vm)?; Ok(class) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index dd045c3bf..d5a4b48ec 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -19,6 +19,7 @@ struct Compiler { nxt_label: usize, source_path: Option, current_source_location: ast::Location, + current_qualified_path: Option, in_loop: bool, in_function_def: bool, } @@ -75,6 +76,7 @@ impl Compiler { nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), + current_qualified_path: None, in_loop: false, in_function_def: false, } @@ -610,6 +612,10 @@ impl Compiler { self.in_loop = false; self.in_function_def = true; + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(self.create_qualified_name(name, ".")); + self.prepare_decorators(decorator_list)?; let mut flags = self.enter_function(name, args)?; @@ -668,7 +674,7 @@ impl Compiler { }); self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -681,6 +687,7 @@ impl Compiler { name: name.to_string(), }); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -696,6 +703,11 @@ impl Compiler { ) -> Result<(), CompileError> { let was_in_loop = self.in_loop; self.in_loop = false; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(qualified_name.clone()); + self.prepare_decorators(decorator_list)?; self.emit(Instruction::LoadBuildClass); let line_number = self.get_source_line_number(); @@ -711,6 +723,12 @@ impl Compiler { let (new_body, doc_str) = get_doc(body); + self.emit(Instruction::LoadName { + name: "__name__".to_string(), + }); + self.emit(Instruction::StoreName { + name: "__module__".to_string(), + }); self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, @@ -737,7 +755,7 @@ impl Compiler { self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -779,6 +797,7 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; Ok(()) } @@ -1604,6 +1623,14 @@ impl Compiler { self.current_source_location.get_row() } + fn create_qualified_name(&self, name: &str, suffix: &str) -> String { + if let Some(ref qualified_path) = self.current_qualified_path { + format!("{}.{}{}", qualified_path, name, suffix) + } else { + format!("{}{}", name, suffix) + } + } + fn mark_generator(&mut self) { self.current_code_object().is_generator = true; } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index d614691c9..6fde05a59 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -10,7 +10,6 @@ use crate::builtins; use crate::bytecode; use crate::function::PyFuncArgs; use crate::obj::objbool; -use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objint::PyInt; @@ -18,6 +17,7 @@ use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objstr::PyString; use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; @@ -566,7 +566,10 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let qualified_name = self.pop_value(); + let qualified_name = self + .pop_value() + .downcast::() + .expect("qualified name to be a string"); let code_obj = self .pop_value() .downcast() @@ -606,7 +609,8 @@ impl Frame { .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); - vm.set_attr(&obj, "__name__", qualified_name.clone())?; + let name = qualified_name.value.split('.').next_back().unwrap(); + vm.set_attr(&obj, "__name__", vm.new_str(name.to_string()))?; vm.set_attr(&obj, "__qualname__", qualified_name)?; let module = self .scope @@ -747,9 +751,7 @@ impl Frame { Ok(None) } bytecode::Instruction::LoadBuildClass => { - let rustfunc = - PyBuiltinFunction::new(Box::new(builtins::builtin_build_class_)).into_ref(vm); - self.push_value(rustfunc.into_object()); + self.push_value(vm.ctx.new_rustfunc(builtins::builtin_build_class_)); Ok(None) } bytecode::Instruction::UnpackSequence { size } => { diff --git a/vm/src/function.rs b/vm/src/function.rs index 305c4de5e..705a172cb 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -244,6 +244,12 @@ pub trait FromArgs: Sized { /// an appropriate FromArgs implementation must be created. pub struct KwArgs(HashMap); +impl KwArgs { + pub fn pop_kwarg(&mut self, name: &str) -> Option { + self.0.remove(name) + } +} + impl FromArgs for KwArgs where T: TryFromObject, @@ -274,8 +280,16 @@ impl IntoIterator for KwArgs { /// /// `Args` optionally accepts a generic type parameter to allow type checks /// or conversions of each argument. +#[derive(Clone)] pub struct Args(Vec); +impl Args> { + pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_tuple(self.0.into_iter().map(|obj| obj.into_object()).collect()) + } +} + impl FromArgs for Args where T: TryFromObject, From 664554bc5e0bf38287dd2bbe12ce1f509c03c359 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 15:21:56 +0100 Subject: [PATCH 228/658] Fix dir bug - not including object attributes correctly. --- tests/snippets/builtin_dir.py | 3 +++ vm/src/obj/objobject.rs | 38 ++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index a121fb718..b5fbf8d0b 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -8,6 +8,9 @@ a = A() assert "test" in dir(a), "test not in a" assert "test" in dir(A), "test not in A" +a.x = 3 +assert "x" in dir(a), "x not in a" + class B(A): def __dir__(self): return ('q', 'h') diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 2be4ff4f4..8f6c90a36 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,6 +1,6 @@ use super::objdict::PyDictRef; use super::objlist::PyList; -use super::objstr::PyStringRef; +use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; @@ -117,13 +117,29 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id()) } -pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { - let attributes = get_attributes(&obj); +pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut attributes: PyAttributes = objtype::get_attributes(obj.class()); + + // Get instance attributes: + if let Some(dict) = &obj.dict { + for (key, value) in dict { + if let Some(key_string) = key.payload::() { + attributes.insert(key_string.to_string(), value.clone()); + } else { + return Err(vm.new_type_error(format!( + "Attribute is not a string: {:?}", + vm.to_pystr(&key)? + ))); + } + } + } + let attributes: Vec = attributes .keys() .map(|k| vm.ctx.new_str(k.to_string())) .collect(); - PyList::from(attributes) + + Ok(PyList::from(attributes)) } fn object_format( @@ -230,17 +246,3 @@ fn object_getattr( Ok(None) } } - -pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { - // Get class attributes: - let mut attributes = objtype::get_attributes(obj.class()); - - // Get instance attributes: - if let Some(dict) = &obj.dict { - for (key, value) in dict { - attributes.insert(key.to_string(), value.clone()); - } - } - - attributes -} From aa38a1aa51913f74eeeea4089268650495db23f3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 10:59:04 +0100 Subject: [PATCH 229/658] Start refactor of ast. --- vm/src/stdlib/ast.rs | 594 +++++++++++++++---------------------------- 1 file changed, 203 insertions(+), 391 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index e7ac5308c..67c58a15d 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -10,14 +10,15 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; use crate::function::PyFuncArgs; +use crate::obj::objlist::PyListRef; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; #[derive(Debug)] struct AstNode; -// type AstNodeRef = PyRef; +type AstNodeRef = PyRef; impl PyValue for AstNode { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -41,32 +42,37 @@ impl AstToPyAst { } */ -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { - let mut body = vec![]; - for statement in &program.statements { - body.push(statement_to_ast(&vm, statement)); +macro_rules! node { + ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { + { + let node = create_node($vm, stringify!($node_name)); + $( + $vm.set_attr(&node, stringify!($attr_name), $attr_value).unwrap(); + )* + node + } + }; + ( $vm: expr, $node_name:ident) => { + create_node($vm, stringify!($node_name)) } - // TODO: create Module node: - // let ast_node = ctx.new_instance(this.Module); - let ast_node = create_node(vm, "program"); - let py_body = vm.ctx.new_list(body); - vm.set_attr(&ast_node, "body", py_body).unwrap(); - ast_node +} + +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { + let py_body = statements_to_ast(vm, &program.statements); + node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, _name: &str) -> PyObjectRef { - // TODO: instantiate a class of type given by name - // TODO: lookup in the current module? - PyObject::new(AstNode, AstNode::class(vm), Some(vm.ctx.new_dict())) +fn create_node(vm: &VirtualMachine, name: &str) -> PyObjectRef { + PyObject::new(AstNode, vm.class("ast", name), Some(vm.ctx.new_dict())) } -fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyObjectRef { - let mut py_statements = vec![]; - for statement in statements { - py_statements.push(statement_to_ast(vm, statement)); - } - vm.ctx.new_list(py_statements) +fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { + let body = statements + .iter() + .map(|statement| statement_to_ast(&vm, statement)) + .collect(); + vm.ctx.new_list(body).downcast().unwrap() } fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { @@ -76,22 +82,11 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P body, decorator_list, .. - } => { - let node = create_node(vm, "ClassDef"); - - // Set name: - vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) - .unwrap(); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.set_attr(&node, "decorator_list", py_decorator_list) - .unwrap(); - node - } + } => node!(vm, ClassDef, { + name => vm.ctx.new_str(name.to_string()), + body => statements_to_ast(vm, body), + decorator_list => expressions_to_ast(vm, decorator_list), + }), ast::Statement::FunctionDef { name, args, @@ -99,140 +94,86 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P decorator_list, returns, } => { - let node = create_node(vm, "FunctionDef"); - - // Set name: - vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) - .unwrap(); - - vm.set_attr(&node, "args", parameters_to_ast(vm, args)) - .unwrap(); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.set_attr(&node, "decorator_list", py_decorator_list) - .unwrap(); - let py_returns = if let Some(hint) = returns { expression_to_ast(vm, hint) } else { vm.ctx.none() }; - vm.set_attr(&node, "returns", py_returns).unwrap(); - node + node!(vm, FunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args), + body => statements_to_ast(vm, body), + decorator_list => expressions_to_ast(vm, decorator_list), + returns => py_returns + }) } - ast::Statement::Continue => create_node(vm, "Continue"), - ast::Statement::Break => create_node(vm, "Break"), - ast::Statement::Pass => create_node(vm, "Pass"), + ast::Statement::Continue => node!(vm, Continue), + ast::Statement::Break => node!(vm, Break), + ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { - let node = create_node(vm, "Pass"); - - vm.set_attr(&node, "test", expression_to_ast(vm, test)) - .unwrap(); - let py_msg = match msg { Some(msg) => expression_to_ast(vm, msg), None => vm.ctx.none(), }; - vm.set_attr(&node, "msg", py_msg).unwrap(); - - node + node!(vm, Assert, { + test => expression_to_ast(vm, test), + msg => py_msg + }) } ast::Statement::Delete { targets } => { - let node = create_node(vm, "Delete"); - let py_targets = vm .ctx .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); - vm.set_attr(&node, "targets", py_targets).unwrap(); - node + node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { - let node = create_node(vm, "Return"); - let py_value = if let Some(value) = value { expression_to_ast(vm, value) } else { vm.ctx.none() }; - vm.set_attr(&node, "value", py_value).unwrap(); - node - } - ast::Statement::If { test, body, orelse } => { - let node = create_node(vm, "If"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) - } else { - vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node + node!(vm, Return, { + value => py_value + }) } + ast::Statement::If { test, body, orelse } => node!(vm, If, { + test => expression_to_ast(vm, test), + body => statements_to_ast(vm, body), + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() + } else { + vm.ctx.none() + } + }), ast::Statement::For { target, iter, body, orelse, - } => { - let node = create_node(vm, "For"); - - let py_target = expression_to_ast(vm, target); - vm.set_attr(&node, "target", py_target).unwrap(); - - let py_iter = expression_to_ast(vm, iter); - vm.set_attr(&node, "iter", py_iter).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } => node!(vm, For, { + target => expression_to_ast(vm, target), + iter => expression_to_ast(vm, iter), + body => statements_to_ast(vm, body), + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() } else { vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node - } - ast::Statement::While { test, body, orelse } => { - let node = create_node(vm, "While"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } + }), + ast::Statement::While { test, body, orelse } => node!(vm, While, { + test => expression_to_ast(vm, test), + body => statements_to_ast(vm, body), + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() } else { vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node - } - ast::Statement::Expression { expression } => { - let node = create_node(vm, "Expr"); - - let value = expression_to_ast(vm, expression); - vm.set_attr(&node, "value", value).unwrap(); - - node - } + } + }), + ast::Statement::Expression { expression } => node!(vm, Expr, { + value => expression_to_ast(vm, expression) + }), x => { unimplemented!("{:?}", x); } @@ -245,35 +186,23 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P node } -fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyObjectRef { - let mut py_expression_nodes = vec![]; - for expression in expressions { - py_expression_nodes.push(expression_to_ast(vm, expression)); - } - vm.ctx.new_list(py_expression_nodes) +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { + let py_expression_nodes = expressions + .iter() + .map(|expression| expression_to_ast(vm, expression)) + .collect(); + vm.ctx.new_list(py_expression_nodes).downcast().unwrap() } fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { let node = match &expression { - ast::Expression::Call { function, args, .. } => { - let node = create_node(vm, "Call"); - - let py_func_ast = expression_to_ast(vm, function); - vm.set_attr(&node, "func", py_func_ast).unwrap(); - - let py_args = expressions_to_ast(vm, args); - vm.set_attr(&node, "args", py_args).unwrap(); - - node - } + ast::Expression::Call { function, args, .. } => node!(vm, Call, { + func => expression_to_ast(vm, function), + args => expressions_to_ast(vm, args), + }), ast::Expression::Binop { a, op, b } => { - let node = create_node(vm, "BinOp"); - - let py_a = expression_to_ast(vm, a); - vm.set_attr(&node, "left", py_a).unwrap(); - // Operator: - let str_op = match op { + let op = match op { ast::Operator::Add => "Add", ast::Operator::Sub => "Sub", ast::Operator::Mult => "Mult", @@ -288,53 +217,43 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Operator::BitAnd => "BitAnd", ast::Operator::FloorDiv => "FloorDiv", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - - let py_b = expression_to_ast(vm, b); - vm.set_attr(&node, "right", py_b).unwrap(); - node + node!(vm, BinOp, { + left => expression_to_ast(vm, a), + op => vm.ctx.new_str(op.to_string()), + right => expression_to_ast(vm, b), + }) } ast::Expression::Unop { op, a } => { - let node = create_node(vm, "UnaryOp"); - - let str_op = match op { + let op = match op { ast::UnaryOperator::Not => "Not", ast::UnaryOperator::Inv => "Invert", ast::UnaryOperator::Neg => "USub", ast::UnaryOperator::Pos => "UAdd", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - - let py_a = expression_to_ast(vm, a); - vm.set_attr(&node, "operand", py_a).unwrap(); - - node + node!(vm, UnaryOp, { + op => vm.ctx.new_str(op.to_string()), + operand => expression_to_ast(vm, a), + }) } ast::Expression::BoolOp { a, op, b } => { - let node = create_node(vm, "BoolOp"); - // Attach values: let py_a = expression_to_ast(vm, a); let py_b = expression_to_ast(vm, b); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); - vm.set_attr(&node, "values", py_values).unwrap(); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - node + node!(vm, BoolOp, { + op => py_op, + values => py_values, + }) } ast::Expression::Compare { vals, ops } => { - let node = create_node(vm, "Compare"); - let py_a = expression_to_ast(vm, &vals[0]); - vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -355,53 +274,31 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .collect(), ); - vm.set_attr(&node, "ops", py_ops).unwrap(); - let py_b = vm.ctx.new_list( vals.iter() .skip(1) .map(|x| expression_to_ast(vm, x)) .collect(), ); - vm.set_attr(&node, "comparators", py_b).unwrap(); - node - } - ast::Expression::Identifier { name } => { - let node = create_node(vm, "Identifier"); - - // Id: - let py_name = vm.ctx.new_str(name.clone()); - vm.set_attr(&node, "id", py_name).unwrap(); - node - } - ast::Expression::Lambda { args, body } => { - let node = create_node(vm, "Lambda"); - - vm.set_attr(&node, "args", parameters_to_ast(vm, args)) - .unwrap(); - - let py_body = expression_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - node - } - ast::Expression::IfExpression { test, body, orelse } => { - let node = create_node(vm, "IfExp"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = expression_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = expression_to_ast(vm, orelse); - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node + node!(vm, Compare, { + left => py_a, + ops => py_ops, + comparators => py_b, + }) } + ast::Expression::Identifier { name } => node!(vm, Identifier, { + id => vm.ctx.new_str(name.clone()) + }), + ast::Expression::Lambda { args, body } => node!(vm, Lambda, { + args => parameters_to_ast(vm, args), + body => expression_to_ast(vm, body), + }), + ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { + text => expression_to_ast(vm, test), + body => expression_to_ast(vm, body), + or_else => expression_to_ast(vm, orelse), + }), ast::Expression::Number { value } => { - let node = create_node(vm, "Num"); - let py_n = match value { ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), ast::Number::Float { value } => vm.ctx.new_float(*value), @@ -409,62 +306,30 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - vm.set_attr(&node, "n", py_n).unwrap(); - - node - } - ast::Expression::True => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.new_bool(true)).unwrap(); - - node - } - ast::Expression::False => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.new_bool(false)).unwrap(); - - node - } - ast::Expression::None => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.none()).unwrap(); - - node - } - ast::Expression::Ellipsis => create_node(vm, "Ellipsis"), - ast::Expression::List { elements } => { - let node = create_node(vm, "List"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node - } - ast::Expression::Tuple { elements } => { - let node = create_node(vm, "Tuple"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node - } - ast::Expression::Set { elements } => { - let node = create_node(vm, "Set"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node + node!(vm, Num, { + n => py_n + }) } + ast::Expression::True => node!(vm, NameConstant, { + value => vm.ctx.new_bool(true) + }), + ast::Expression::False => node!(vm, NameConstant, { + value => vm.ctx.new_bool(false) + }), + ast::Expression::None => node!(vm, NameConstant, { + value => vm.ctx.none() + }), + ast::Expression::Ellipsis => node!(vm, Ellipsis), + ast::Expression::List { elements } => node!(vm, List, { + elts => expressions_to_ast(vm, &elements) + }), + ast::Expression::Tuple { elements } => node!(vm, Tuple, { + elts => expressions_to_ast(vm, &elements) + }), + ast::Expression::Set { elements } => node!(vm, Set, { + elts => expressions_to_ast(vm, &elements) + }), ast::Expression::Dict { elements } => { - let node = create_node(vm, "Dict"); - let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { @@ -472,96 +337,65 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj values.push(expression_to_ast(vm, v)); } - let py_keys = vm.ctx.new_list(keys); - vm.set_attr(&node, "keys", py_keys).unwrap(); - - let py_values = vm.ctx.new_list(values); - vm.set_attr(&node, "values", py_values).unwrap(); - - node + node!(vm, Dict, { + keys => vm.ctx.new_list(keys), + values => vm.ctx.new_list(values), + }) } ast::Expression::Comprehension { kind, generators } => { - let node = match kind.deref() { - ast::ComprehensionKind::GeneratorExpression { .. } => { - create_node(vm, "GeneratorExp") - } - ast::ComprehensionKind::List { .. } => create_node(vm, "ListComp"), - ast::ComprehensionKind::Set { .. } => create_node(vm, "SetComp"), - ast::ComprehensionKind::Dict { .. } => create_node(vm, "DictComp"), - }; - let g = generators .iter() .map(|g| comprehension_to_ast(vm, g)) .collect(); let py_generators = vm.ctx.new_list(g); - vm.set_attr(&node, "generators", py_generators).unwrap(); - node + match kind.deref() { + ast::ComprehensionKind::GeneratorExpression { .. } => { + node!(vm, GeneratorExp, {generators => py_generators}) + } + ast::ComprehensionKind::List { .. } => { + node!(vm, ListComp, {generators => py_generators}) + } + ast::ComprehensionKind::Set { .. } => { + node!(vm, SetComp, {generators => py_generators}) + } + ast::ComprehensionKind::Dict { .. } => { + node!(vm, DictComp, {generators => py_generators}) + } + } } ast::Expression::Yield { value } => { - let node = create_node(vm, "Yield"); - let py_value = match value { Some(value) => expression_to_ast(vm, value), None => vm.ctx.none(), }; - vm.set_attr(&node, "value", py_value).unwrap(); - - node + node!(vm, Yield, { + value => py_value + }) } ast::Expression::YieldFrom { value } => { - let node = create_node(vm, "YieldFrom"); - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - node - } - ast::Expression::Subscript { a, b } => { - let node = create_node(vm, "Subscript"); - - let py_value = expression_to_ast(vm, a); - vm.set_attr(&node, "value", py_value).unwrap(); - - let py_slice = expression_to_ast(vm, b); - vm.set_attr(&node, "slice", py_slice).unwrap(); - - node - } - ast::Expression::Attribute { value, name } => { - let node = create_node(vm, "Attribute"); - - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - let py_attr = vm.ctx.new_str(name.to_string()); - vm.set_attr(&node, "attr", py_attr).unwrap(); - - node - } - ast::Expression::Starred { value } => { - let node = create_node(vm, "Starred"); - - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - node - } - ast::Expression::Slice { elements } => { - let node = create_node(vm, "Slice"); - - let py_value = expressions_to_ast(vm, elements); - vm.set_attr(&node, "bounds", py_value).unwrap(); - - node + node!(vm, YieldFrom, { + value => py_value + }) } + ast::Expression::Subscript { a, b } => node!(vm, Subscript, { + value => expression_to_ast(vm, a), + slice => expression_to_ast(vm, b), + }), + ast::Expression::Attribute { value, name } => node!(vm, Attribute, { + value => expression_to_ast(vm, value), + attr => vm.ctx.new_str(name.to_string()), + }), + ast::Expression::Starred { value } => node!(vm, Starred, { + value => expression_to_ast(vm, value) + }), + ast::Expression::Slice { elements } => node!(vm, Slice, { + bounds => expressions_to_ast(vm, elements) + }), ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { - let node = create_node(vm, "Bytes"); - vm.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())) - .unwrap(); - node + node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } }; @@ -573,74 +407,50 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { - let node = create_node(vm, "arguments"); + let args = vm + .ctx + .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()); - vm.set_attr( - &node, - "args", - vm.ctx - .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), - ) - .unwrap(); - - node + node!(vm, arguments, { args => args }) } fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { - let node = create_node(vm, "arg"); - - let py_arg = vm.ctx.new_str(parameter.arg.to_string()); - vm.set_attr(&node, "arg", py_arg).unwrap(); - let py_annotation = if let Some(annotation) = ¶meter.annotation { expression_to_ast(vm, annotation) } else { vm.ctx.none() }; - vm.set_attr(&node, "annotation", py_annotation).unwrap(); - node + node!(vm, arg, { + arg => vm.ctx.new_str(parameter.arg.to_string()), + annotation => py_annotation + }) } fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { - let node = create_node(vm, "comprehension"); - - let py_target = expression_to_ast(vm, &comprehension.target); - vm.set_attr(&node, "target", py_target).unwrap(); - - let py_iter = expression_to_ast(vm, &comprehension.iter); - vm.set_attr(&node, "iter", py_iter).unwrap(); - - let py_ifs = expressions_to_ast(vm, &comprehension.ifs); - vm.set_attr(&node, "ifs", py_ifs).unwrap(); - - node + node!(vm, comprehension, { + target => expression_to_ast(vm, &comprehension.target), + iter => expression_to_ast(vm, &comprehension.iter), + ifs => expressions_to_ast(vm, &comprehension.ifs), + }) } fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { match string { ast::StringGroup::Constant { value } => { - let node = create_node(vm, "Str"); - vm.set_attr(&node, "s", vm.ctx.new_str(value.clone())) - .unwrap(); - node + node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - let node = create_node(vm, "FormattedValue"); - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - node + node!(vm, FormattedValue, { value => expression_to_ast(vm, value) }) } ast::StringGroup::Joined { values } => { - let node = create_node(vm, "JoinedStr"); let py_values = vm.ctx.new_list( values .iter() .map(|value| string_to_ast(vm, value)) .collect(), ); - vm.set_attr(&node, "values", py_values).unwrap(); - node + node!(vm, JoinedStr, { values => py_values }) } } } @@ -659,11 +469,13 @@ fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), - "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}), - "AST" => py_class!(ctx, "_ast.AST", ctx.object(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), + "AST" => ast_base, }) } From d7e0ceec125f8101b9317f56781bd277878a3caa Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 09:34:12 +0100 Subject: [PATCH 230/658] New (PyRef) style types. --- vm/src/stdlib/ast.rs | 104 +++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 67c58a15d..518465fe6 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -9,11 +9,10 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; -use crate::function::PyFuncArgs; use crate::obj::objlist::PyListRef; -use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -26,28 +25,12 @@ impl PyValue for AstNode { } } -/* - * Idea: maybe we can create a sort of struct with some helper functions? -struct AstToPyAst { - ctx: &PyContext, -} - -impl AstToPyAst { - fn new(ctx: &PyContext) -> Self { - AstToPyAst { - ctx: ctx, - } - } - -} -*/ - macro_rules! node { ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { { let node = create_node($vm, stringify!($node_name)); $( - $vm.set_attr(&node, stringify!($attr_name), $attr_value).unwrap(); + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value).unwrap(); )* node } @@ -57,25 +40,27 @@ macro_rules! node { } } -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> AstNodeRef { let py_body = statements_to_ast(vm, &program.statements); node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, name: &str) -> PyObjectRef { - PyObject::new(AstNode, vm.class("ast", name), Some(vm.ctx.new_dict())) +fn create_node(vm: &VirtualMachine, name: &str) -> AstNodeRef { + AstNode + .into_ref_with_type(vm, vm.class("ast", name)) + .unwrap() } fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { let body = statements .iter() - .map(|statement| statement_to_ast(&vm, statement)) + .map(|statement| statement_to_ast(&vm, statement).into_object()) .collect(); vm.ctx.new_list(body).downcast().unwrap() } -fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { +fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> AstNodeRef { let node = match &statement.node { ast::Statement::ClassDef { name, @@ -95,7 +80,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P returns, } => { let py_returns = if let Some(hint) = returns { - expression_to_ast(vm, hint) + expression_to_ast(vm, hint).into_object() } else { vm.ctx.none() }; @@ -112,7 +97,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { let py_msg = match msg { - Some(msg) => expression_to_ast(vm, msg), + Some(msg) => expression_to_ast(vm, msg).into_object(), None => vm.ctx.none(), }; node!(vm, Assert, { @@ -121,15 +106,18 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P }) } ast::Statement::Delete { targets } => { - let py_targets = vm - .ctx - .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); + let py_targets = vm.ctx.new_tuple( + targets + .iter() + .map(|v| expression_to_ast(vm, v).into_object()) + .collect(), + ); node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { let py_value = if let Some(value) = value { - expression_to_ast(vm, value) + expression_to_ast(vm, value).into_object() } else { vm.ctx.none() }; @@ -181,7 +169,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); - vm.set_attr(&node, "lineno", lineno).unwrap(); + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); node } @@ -189,12 +177,12 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { let py_expression_nodes = expressions .iter() - .map(|expression| expression_to_ast(vm, expression)) + .map(|expression| expression_to_ast(vm, expression).into_object()) .collect(); vm.ctx.new_list(py_expression_nodes).downcast().unwrap() } -fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNodeRef { let node = match &expression { ast::Expression::Call { function, args, .. } => node!(vm, Call, { func => expression_to_ast(vm, function), @@ -237,8 +225,8 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } ast::Expression::BoolOp { a, op, b } => { // Attach values: - let py_a = expression_to_ast(vm, a); - let py_b = expression_to_ast(vm, b); + let py_a = expression_to_ast(vm, a).into_object(); + let py_b = expression_to_ast(vm, b).into_object(); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); let str_op = match op { @@ -277,7 +265,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let py_b = vm.ctx.new_list( vals.iter() .skip(1) - .map(|x| expression_to_ast(vm, x)) + .map(|x| expression_to_ast(vm, x).into_object()) .collect(), ); node!(vm, Compare, { @@ -333,8 +321,8 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k)); - values.push(expression_to_ast(vm, v)); + keys.push(expression_to_ast(vm, k).into_object()); + values.push(expression_to_ast(vm, v).into_object()); } node!(vm, Dict, { @@ -345,7 +333,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Expression::Comprehension { kind, generators } => { let g = generators .iter() - .map(|g| comprehension_to_ast(vm, g)) + .map(|g| comprehension_to_ast(vm, g).into_object()) .collect(); let py_generators = vm.ctx.new_list(g); @@ -366,7 +354,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } ast::Expression::Yield { value } => { let py_value = match value { - Some(value) => expression_to_ast(vm, value), + Some(value) => expression_to_ast(vm, value).into_object(), None => vm.ctx.none(), }; node!(vm, Yield, { @@ -401,22 +389,24 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); - vm.set_attr(&node, "lineno", lineno).unwrap(); - + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); node } -fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { - let args = vm - .ctx - .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()); +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> AstNodeRef { + let args = vm.ctx.new_list( + args.args + .iter() + .map(|a| parameter_to_ast(vm, a).into_object()) + .collect(), + ); node!(vm, arguments, { args => args }) } -fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeRef { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(vm, annotation) + expression_to_ast(vm, annotation).into_object() } else { vm.ctx.none() }; @@ -427,7 +417,7 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObject }) } -fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { +fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> AstNodeRef { node!(vm, comprehension, { target => expression_to_ast(vm, &comprehension.target), iter => expression_to_ast(vm, &comprehension.iter), @@ -435,7 +425,7 @@ fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) }) } -fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> AstNodeRef { match string { ast::StringGroup::Constant { value } => { node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) @@ -447,7 +437,7 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef let py_values = vm.ctx.new_list( values .iter() - .map(|value| string_to_ast(vm, value)) + .map(|value| string_to_ast(vm, value).into_object()) .collect(), ); node!(vm, JoinedStr, { values => py_values }) @@ -455,15 +445,11 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef } } -fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); - - let source_string = objstr::get_value(source); - let internal_ast = parser::parse_program(&source_string) +fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { + let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - let ast_node = program_to_ast(&vm, &internal_ast); - Ok(ast_node) + Ok(program_to_ast(&vm, &internal_ast)) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From 8a1466557531880e5999b176cef746f19e1bd1cd Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 14:49:24 +0100 Subject: [PATCH 231/658] Ast refactor: Everything can fail. --- vm/src/stdlib/ast.rs | 228 +++++++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 114 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 518465fe6..0612c0d80 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -28,11 +28,11 @@ impl PyValue for AstNode { macro_rules! node { ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { { - let node = create_node($vm, stringify!($node_name)); + let node = create_node($vm, stringify!($node_name))?; $( - $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value).unwrap(); + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value)?; )* - node + Ok(node) } }; ( $vm: expr, $node_name:ident) => { @@ -40,27 +40,31 @@ macro_rules! node { } } -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> AstNodeRef { - let py_body = statements_to_ast(vm, &program.statements); +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyResult { + let py_body = statements_to_ast(vm, &program.statements)?; node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, name: &str) -> AstNodeRef { - AstNode - .into_ref_with_type(vm, vm.class("ast", name)) - .unwrap() +fn create_node(vm: &VirtualMachine, name: &str) -> PyResult { + AstNode.into_ref_with_type(vm, vm.class("ast", name)) } -fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { - let body = statements +fn statements_to_ast( + vm: &VirtualMachine, + statements: &[ast::LocatedStatement], +) -> PyResult { + let body: PyResult> = statements .iter() - .map(|statement| statement_to_ast(&vm, statement).into_object()) + .map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object())) .collect(); - vm.ctx.new_list(body).downcast().unwrap() + Ok(vm.ctx.new_list(body?).downcast().unwrap()) } -fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> AstNodeRef { +fn statement_to_ast( + vm: &VirtualMachine, + statement: &ast::LocatedStatement, +) -> PyResult { let node = match &statement.node { ast::Statement::ClassDef { name, @@ -69,8 +73,8 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A .. } => node!(vm, ClassDef, { name => vm.ctx.new_str(name.to_string()), - body => statements_to_ast(vm, body), - decorator_list => expressions_to_ast(vm, decorator_list), + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, }), ast::Statement::FunctionDef { name, @@ -80,15 +84,15 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A returns, } => { let py_returns = if let Some(hint) = returns { - expression_to_ast(vm, hint).into_object() + expression_to_ast(vm, hint)?.into_object() } else { vm.ctx.none() }; node!(vm, FunctionDef, { name => vm.ctx.new_str(name.to_string()), - args => parameters_to_ast(vm, args), - body => statements_to_ast(vm, body), - decorator_list => expressions_to_ast(vm, decorator_list), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, returns => py_returns }) } @@ -97,27 +101,25 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { let py_msg = match msg { - Some(msg) => expression_to_ast(vm, msg).into_object(), + Some(msg) => expression_to_ast(vm, msg)?.into_object(), None => vm.ctx.none(), }; node!(vm, Assert, { - test => expression_to_ast(vm, test), + test => expression_to_ast(vm, test)?, msg => py_msg }) } ast::Statement::Delete { targets } => { - let py_targets = vm.ctx.new_tuple( - targets - .iter() - .map(|v| expression_to_ast(vm, v).into_object()) - .collect(), - ); - + let targets: PyResult<_> = targets + .iter() + .map(|v| Ok(expression_to_ast(vm, v)?.into_object())) + .collect(); + let py_targets = vm.ctx.new_tuple(targets?); node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { let py_value = if let Some(value) = value { - expression_to_ast(vm, value).into_object() + expression_to_ast(vm, value)?.into_object() } else { vm.ctx.none() }; @@ -127,10 +129,10 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A }) } ast::Statement::If { test, body, orelse } => node!(vm, If, { - test => expression_to_ast(vm, test), - body => statements_to_ast(vm, body), + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, orelse => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } @@ -141,52 +143,52 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A body, orelse, } => node!(vm, For, { - target => expression_to_ast(vm, target), - iter => expression_to_ast(vm, iter), - body => statements_to_ast(vm, body), + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, or_else => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } }), ast::Statement::While { test, body, orelse } => node!(vm, While, { - test => expression_to_ast(vm, test), - body => statements_to_ast(vm, body), + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, orelse => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } }), ast::Statement::Expression { expression } => node!(vm, Expr, { - value => expression_to_ast(vm, expression) + value => expression_to_ast(vm, expression)? }), x => { - unimplemented!("{:?}", x); + return Err(vm.new_type_error(format!("Ast not implemented: {:?}", x))); } - }; + }?; // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { - let py_expression_nodes = expressions +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyResult { + let py_expression_nodes: PyResult<_> = expressions .iter() - .map(|expression| expression_to_ast(vm, expression).into_object()) + .map(|expression| Ok(expression_to_ast(vm, expression)?.into_object())) .collect(); - vm.ctx.new_list(py_expression_nodes).downcast().unwrap() + Ok(vm.ctx.new_list(py_expression_nodes?).downcast().unwrap()) } -fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNodeRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult { let node = match &expression { ast::Expression::Call { function, args, .. } => node!(vm, Call, { - func => expression_to_ast(vm, function), - args => expressions_to_ast(vm, args), + func => expression_to_ast(vm, function)?, + args => expressions_to_ast(vm, args)?, }), ast::Expression::Binop { a, op, b } => { // Operator: @@ -206,9 +208,9 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo ast::Operator::FloorDiv => "FloorDiv", }; node!(vm, BinOp, { - left => expression_to_ast(vm, a), + left => expression_to_ast(vm, a)?, op => vm.ctx.new_str(op.to_string()), - right => expression_to_ast(vm, b), + right => expression_to_ast(vm, b)?, }) } ast::Expression::Unop { op, a } => { @@ -220,13 +222,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }; node!(vm, UnaryOp, { op => vm.ctx.new_str(op.to_string()), - operand => expression_to_ast(vm, a), + operand => expression_to_ast(vm, a)?, }) } ast::Expression::BoolOp { a, op, b } => { // Attach values: - let py_a = expression_to_ast(vm, a).into_object(); - let py_b = expression_to_ast(vm, b).into_object(); + let py_a = expression_to_ast(vm, a)?.into_object(); + let py_b = expression_to_ast(vm, b)?.into_object(); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); let str_op = match op { @@ -241,7 +243,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::Compare { vals, ops } => { - let py_a = expression_to_ast(vm, &vals[0]); + let left = expression_to_ast(vm, &vals[0])?; // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -256,35 +258,35 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = vm.ctx.new_list( + let ops = vm.ctx.new_list( ops.iter() .map(|x| vm.ctx.new_str(to_operator(x).to_string())) .collect(), ); - let py_b = vm.ctx.new_list( - vals.iter() - .skip(1) - .map(|x| expression_to_ast(vm, x).into_object()) - .collect(), - ); + let comparators: PyResult<_> = vals + .iter() + .skip(1) + .map(|x| Ok(expression_to_ast(vm, x)?.into_object())) + .collect(); + let comparators = vm.ctx.new_list(comparators?); node!(vm, Compare, { - left => py_a, - ops => py_ops, - comparators => py_b, + left => left, + ops => ops, + comparators => comparators, }) } ast::Expression::Identifier { name } => node!(vm, Identifier, { id => vm.ctx.new_str(name.clone()) }), ast::Expression::Lambda { args, body } => node!(vm, Lambda, { - args => parameters_to_ast(vm, args), - body => expression_to_ast(vm, body), + args => parameters_to_ast(vm, args)?, + body => expression_to_ast(vm, body)?, }), ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { - text => expression_to_ast(vm, test), - body => expression_to_ast(vm, body), - or_else => expression_to_ast(vm, orelse), + text => expression_to_ast(vm, test)?, + body => expression_to_ast(vm, body)?, + or_else => expression_to_ast(vm, orelse)?, }), ast::Expression::Number { value } => { let py_n = match value { @@ -309,20 +311,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }), ast::Expression::Ellipsis => node!(vm, Ellipsis), ast::Expression::List { elements } => node!(vm, List, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Tuple { elements } => node!(vm, Tuple, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Set { elements } => node!(vm, Set, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Dict { elements } => { let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k).into_object()); - values.push(expression_to_ast(vm, v).into_object()); + keys.push(expression_to_ast(vm, k)?.into_object()); + values.push(expression_to_ast(vm, v)?.into_object()); } node!(vm, Dict, { @@ -331,11 +333,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::Comprehension { kind, generators } => { - let g = generators - .iter() - .map(|g| comprehension_to_ast(vm, g).into_object()) - .collect(); - let py_generators = vm.ctx.new_list(g); + let py_generators = map_ast(comprehension_to_ast, vm, generators)?; match kind.deref() { ast::ComprehensionKind::GeneratorExpression { .. } => { @@ -354,7 +352,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo } ast::Expression::Yield { value } => { let py_value = match value { - Some(value) => expression_to_ast(vm, value).into_object(), + Some(value) => expression_to_ast(vm, value)?.into_object(), None => vm.ctx.none(), }; node!(vm, Yield, { @@ -362,51 +360,45 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::YieldFrom { value } => { - let py_value = expression_to_ast(vm, value); + let py_value = expression_to_ast(vm, value)?; node!(vm, YieldFrom, { value => py_value }) } ast::Expression::Subscript { a, b } => node!(vm, Subscript, { - value => expression_to_ast(vm, a), - slice => expression_to_ast(vm, b), + value => expression_to_ast(vm, a)?, + slice => expression_to_ast(vm, b)?, }), ast::Expression::Attribute { value, name } => node!(vm, Attribute, { - value => expression_to_ast(vm, value), + value => expression_to_ast(vm, value)?, attr => vm.ctx.new_str(name.to_string()), }), ast::Expression::Starred { value } => node!(vm, Starred, { - value => expression_to_ast(vm, value) + value => expression_to_ast(vm, value)? }), ast::Expression::Slice { elements } => node!(vm, Slice, { - bounds => expressions_to_ast(vm, elements) + bounds => expressions_to_ast(vm, elements)? }), ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } - }; + }?; // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> AstNodeRef { - let args = vm.ctx.new_list( - args.args - .iter() - .map(|a| parameter_to_ast(vm, a).into_object()) - .collect(), - ); - +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyResult { + let args = map_ast(parameter_to_ast, vm, &args.args)?; node!(vm, arguments, { args => args }) } -fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeRef { +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(vm, annotation).into_object() + expression_to_ast(vm, annotation)?.into_object() } else { vm.ctx.none() }; @@ -417,29 +409,37 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeR }) } -fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> AstNodeRef { +fn map_ast( + f: fn(vm: &VirtualMachine, &T) -> PyResult, + vm: &VirtualMachine, + items: &Vec, +) -> PyResult { + let list: PyResult> = + items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); + Ok(vm.ctx.new_list(list?)) +} + +fn comprehension_to_ast( + vm: &VirtualMachine, + comprehension: &ast::Comprehension, +) -> PyResult { node!(vm, comprehension, { - target => expression_to_ast(vm, &comprehension.target), - iter => expression_to_ast(vm, &comprehension.iter), - ifs => expressions_to_ast(vm, &comprehension.ifs), + target => expression_to_ast(vm, &comprehension.target)?, + iter => expression_to_ast(vm, &comprehension.iter)?, + ifs => expressions_to_ast(vm, &comprehension.ifs)?, }) } -fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> AstNodeRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyResult { match string { ast::StringGroup::Constant { value } => { node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - node!(vm, FormattedValue, { value => expression_to_ast(vm, value) }) + node!(vm, FormattedValue, { value => expression_to_ast(vm, value)? }) } ast::StringGroup::Joined { values } => { - let py_values = vm.ctx.new_list( - values - .iter() - .map(|value| string_to_ast(vm, value).into_object()) - .collect(), - ); + let py_values = map_ast(string_to_ast, vm, &values)?; node!(vm, JoinedStr, { values => py_values }) } } @@ -449,7 +449,7 @@ fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - Ok(program_to_ast(&vm, &internal_ast)) + program_to_ast(&vm, &internal_ast) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From 856914575148b0ec850be205a90ebcc82b36d0d2 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 15:03:39 +0100 Subject: [PATCH 232/658] Define classes for all the ast types. --- vm/src/stdlib/ast.rs | 52 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 0612c0d80..816882f31 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -458,10 +458,54 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), - "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "AST" => ast_base.clone(), + // TODO: There's got to be a better way! + "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), + "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), + "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), + "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), + "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), + "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), + "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), + "Bytes" => py_class!(ctx, "_ast.Bytes", ast_base.clone(), {}), "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), - "AST" => ast_base, + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Compare" => py_class!(ctx, "_ast.Compare", ast_base.clone(), {}), + "comprehension" => py_class!(ctx, "_ast.comprehension", ast_base.clone(), {}), + "Continue" => py_class!(ctx, "_ast.Continue", ast_base.clone(), {}), + "Delete" => py_class!(ctx, "_ast.Delete", ast_base.clone(), {}), + "Dict" => py_class!(ctx, "_ast.Dict", ast_base.clone(), {}), + "DictComp" => py_class!(ctx, "_ast.DictComp", ast_base.clone(), {}), + "Ellipsis" => py_class!(ctx, "_ast.Ellipsis", ast_base.clone(), {}), + "Expr" => py_class!(ctx, "_ast.Expr", ast_base.clone(), {}), + "For" => py_class!(ctx, "_ast.For", ast_base.clone(), {}), + "FormattedValue" => py_class!(ctx, "_ast.FormattedValue", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "GeneratorExp" => py_class!(ctx, "_ast.GeneratorExp", ast_base.clone(), {}), + "Identifier" => py_class!(ctx, "_ast.Identifier", ast_base.clone(), {}), + "If" => py_class!(ctx, "_ast.If", ast_base.clone(), {}), + "IfExp" => py_class!(ctx, "_ast.IfExp", ast_base.clone(), {}), + "JoinedStr" => py_class!(ctx, "_ast.JoinedStr", ast_base.clone(), {}), + "Lambda" => py_class!(ctx, "_ast.Lambda", ast_base.clone(), {}), + "List" => py_class!(ctx, "_ast.List", ast_base.clone(), {}), + "ListComp" => py_class!(ctx, "_ast.ListComp", ast_base.clone(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "Num" => py_class!(ctx, "_ast.Num", ast_base.clone(), {}), + "Pass" => py_class!(ctx, "_ast.Pass", ast_base.clone(), {}), + "Return" => py_class!(ctx, "_ast.Return", ast_base.clone(), {}), + "Set" => py_class!(ctx, "_ast.Set", ast_base.clone(), {}), + "SetComp" => py_class!(ctx, "_ast.SetComp", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Str" => py_class!(ctx, "_ast.Str", ast_base.clone(), {}), + "Subscript" => py_class!(ctx, "_ast.Subscript", ast_base.clone(), {}), + "Tuple" => py_class!(ctx, "_ast.Tuple", ast_base.clone(), {}), + "UnaryOp" => py_class!(ctx, "_ast.UnaryOp", ast_base.clone(), {}), + "While" => py_class!(ctx, "_ast.While", ast_base.clone(), {}), + "Yield" => py_class!(ctx, "_ast.Yield", ast_base.clone(), {}), + "YieldFrom" => py_class!(ctx, "_ast.YieldFrom", ast_base.clone(), {}), }) } From 72467069bea77b987ca4961e5cd19e25cca5ad4d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:10:31 +0300 Subject: [PATCH 233/658] Add os.getenv --- vm/src/stdlib/os.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 25937cd5c..217e4d381 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,11 +1,11 @@ -use std::fs; use std::fs::File; use std::fs::OpenOptions; use std::io::{ErrorKind, Read, Write}; +use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::PyFuncArgs; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; @@ -173,6 +173,13 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + match env::var(&key.value) { + Ok(val) => Ok(vm.new_str(val)), + Err(_) => Ok(default.into_option().unwrap_or(vm.get_none())), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -194,6 +201,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), + "getenv" => ctx.new_rustfunc(os_getenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 9b475a2c417902ea1bcc5f8f3bb87fc3241181cb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:15:02 +0300 Subject: [PATCH 234/658] Add os.putenv --- vm/src/stdlib/os.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 217e4d381..0b10ff2d9 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -180,6 +180,10 @@ fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMa } } +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { + env::set_var(&key.value, &value.value) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -202,6 +206,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), "getenv" => ctx.new_rustfunc(os_getenv), + "putenv" => ctx.new_rustfunc(os_putenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 951659a54d71efa99b939f919a278a664ff2864e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:20:58 +0300 Subject: [PATCH 235/658] Add os.unsetenv --- vm/src/stdlib/os.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0b10ff2d9..6a13003b8 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -184,6 +184,10 @@ fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { env::set_var(&key.value, &value.value) } +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { + env::remove_var(&key.value) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -207,6 +211,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "listdir" => ctx.new_rustfunc(os_listdir), "getenv" => ctx.new_rustfunc(os_getenv), "putenv" => ctx.new_rustfunc(os_putenv), + "unsetenv" => ctx.new_rustfunc(os_unsetenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From c33593efa9e97a4ca04cb40b9ea87e21656505f5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Tue, 9 Apr 2019 09:31:31 +0300 Subject: [PATCH 236/658] Remove getenv and add environ --- vm/src/stdlib/os.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6a13003b8..66698b3cb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -5,13 +5,14 @@ use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::PyFuncArgs; use crate::obj::objbytes::PyBytesRef; +use crate::obj::objdict::PyDictRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -173,13 +174,6 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { - match env::var(&key.value) { - Ok(val) => Ok(vm.new_str(val)), - Err(_) => Ok(default.into_option().unwrap_or(vm.get_none())), - } -} - fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { env::set_var(&key.value, &value.value) } @@ -188,6 +182,14 @@ fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { env::remove_var(&key.value) } +fn _os_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { + environ.set_item(&key, vm.new_str(value), vm).unwrap(); + } + environ +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -197,6 +199,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "posix".to_string() }; + let environ = _os_environ(vm); + py_module!(vm, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), @@ -209,9 +213,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), - "getenv" => ctx.new_rustfunc(os_getenv), "putenv" => ctx.new_rustfunc(os_putenv), "unsetenv" => ctx.new_rustfunc(os_unsetenv), + "environ" => environ, "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 4293b11429195905728cf0d5d1481837d1cea12a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 18:37:42 +0300 Subject: [PATCH 237/658] Add environ class --- Lib/os.py | 67 +++++++++++++++++++++++++++++++++++++ tests/snippets/stdlib_os.py | 14 ++++++++ vm/src/stdlib/mod.rs | 2 +- vm/src/stdlib/os.rs | 2 +- 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 Lib/os.py diff --git a/Lib/os.py b/Lib/os.py new file mode 100644 index 000000000..badb08ebb --- /dev/null +++ b/Lib/os.py @@ -0,0 +1,67 @@ +from _os import * + +class _Environ(): + def __init__(self, data, putenv, unsetenv): + self.putenv = putenv + self.unsetenv = unsetenv + self._data = data + + def __getitem__(self, key): + return self._data[key] + + def __setitem__(self, key, value): + self.putenv(key, value) + self._data[key] = value + + def __delitem__(self, key): + self.unsetenv(key) + del self._data[key] + + def __iter__(self): + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: + yield key + + def __len__(self): + return len(self._data) + + def __repr__(self): + return 'environ({{{}}})'.format(', '.join( + ('{}: {}'.format(key, value) + for key, value in self._data.items()))) + + def copy(self): + return dict(self) + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") + +def _createenviron(): + return _Environ(environ,_putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + + +def getenv(key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default. + key, default and the result are str.""" + try: + return environ[key] + except KeyError: + return default diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ba6033ff8..15fa176d8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -40,3 +40,17 @@ assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) assert os.O_RDONLY == 0 assert os.O_WRONLY == 1 assert os.O_RDWR == 2 + +ENV_KEY = "TEST_ENV_VAR" +ENV_VALUE = "value" + +assert os.getenv(ENV_KEY) == None +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY, 5) == 5 +os.environ[ENV_KEY] = ENV_VALUE +assert ENV_KEY in os.environ +assert os.getenv(ENV_KEY) == ENV_VALUE +del os.environ[ENV_KEY] +os.unsetenv(ENV_KEY) +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY) == None diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index a69a80d11..9b0d0b345 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -50,7 +50,7 @@ pub fn get_module_inits() -> HashMap { #[cfg(not(target_arch = "wasm32"))] { modules.insert("io".to_string(), Box::new(io::make_module)); - modules.insert("os".to_string(), Box::new(os::make_module)); + modules.insert("_os".to_string(), Box::new(os::make_module)); modules.insert("socket".to_string(), Box::new(socket::make_module)); } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 66698b3cb..26707d3ca 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -201,7 +201,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let environ = _os_environ(vm); - py_module!(vm, "os", { + py_module!(vm, "_os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), From 69e49e6cba58a43c6cb230ae8be2b733d46c9223 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 18:44:09 +0300 Subject: [PATCH 238/658] Simplify os.py --- Lib/os.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index badb08ebb..8b7876c7a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -39,23 +39,7 @@ class _Environ(): self[key] = value return self[key] -try: - _putenv = putenv -except NameError: - _putenv = lambda key, value: None - -try: - _unsetenv = unsetenv -except NameError: - _unsetenv = lambda key: _putenv(key, "") - -def _createenviron(): - return _Environ(environ,_putenv, _unsetenv) - -# unicode environ -environ = _createenviron() -del _createenviron - +environ = _Environ(environ, putenv, unsetenv) def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. From eb40e08d8a59c0e43b00f8c2206573bc7f8ad83f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:02:04 +0300 Subject: [PATCH 239/658] Add _collections_abc.py from CPython --- Lib/_collections_abc.py | 1013 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1013 insertions(+) create mode 100644 Lib/_collections_abc.py diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py new file mode 100644 index 000000000..c36398797 --- /dev/null +++ b/Lib/_collections_abc.py @@ -0,0 +1,1013 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +from abc import ABCMeta, abstractmethod +import sys + +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", + ] + +# This module has been renamed from collections.abc to _collections_abc to +# speed up interpreter startup. Some of the types such as MutableMapping are +# required early but collections module imports a lot of other modules. +# See issue #19218 +__name__ = "collections.abc" + +# Private list of types that we want to register with the various ABCs +# so that they will pass tests like: +# it = iter(somebytearray) +# assert isinstance(it, Iterable) +# Note: in other implementations, these types might not be distinct +# and they may have their own implementation specific types that +# are not included on this list. +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +longrange_iterator = type(iter(range(1 << 1000))) +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +mappingproxy = type(type.__dict__) +generator = type((lambda: (yield))()) +## coroutine ## +async def _coro(): pass +_coro = _coro() +coroutine = type(_coro) +_coro.close() # Prevent ResourceWarning +del _coro +## asynchronous generator ## +async def _ag(): yield +_ag = _ag() +async_generator = type(_ag) +del _ag + + +### ONE-TRICK PONIES ### + +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + return _check_methods(C, "__hash__") + return NotImplemented + + +class Awaitable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __await__(self): + yield + + @classmethod + def __subclasshook__(cls, C): + if cls is Awaitable: + return _check_methods(C, "__await__") + return NotImplemented + + +class Coroutine(Awaitable): + + __slots__ = () + + @abstractmethod + def send(self, value): + """Send a value into the coroutine. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the coroutine. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside coroutine. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("coroutine ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Coroutine: + return _check_methods(C, '__await__', 'send', 'throw', 'close') + return NotImplemented + + +Coroutine.register(coroutine) + + +class AsyncIterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __aiter__(self): + return AsyncIterator() + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterable: + return _check_methods(C, "__aiter__") + return NotImplemented + + +class AsyncIterator(AsyncIterable): + + __slots__ = () + + @abstractmethod + async def __anext__(self): + """Return the next item or raise StopAsyncIteration when exhausted.""" + raise StopAsyncIteration + + def __aiter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterator: + return _check_methods(C, "__anext__", "__aiter__") + return NotImplemented + + +class AsyncGenerator(AsyncIterator): + + __slots__ = () + + async def __anext__(self): + """Return the next item from the asynchronous generator. + When exhausted, raise StopAsyncIteration. + """ + return await self.asend(None) + + @abstractmethod + async def asend(self, value): + """Send a value into the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + raise StopAsyncIteration + + @abstractmethod + async def athrow(self, typ, val=None, tb=None): + """Raise an exception in the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + async def aclose(self): + """Raise GeneratorExit inside coroutine. + """ + try: + await self.athrow(GeneratorExit) + except (GeneratorExit, StopAsyncIteration): + pass + else: + raise RuntimeError("asynchronous generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncGenerator: + return _check_methods(C, '__aiter__', '__anext__', + 'asend', 'athrow', 'aclose') + return NotImplemented + + +AsyncGenerator.register(async_generator) + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + return _check_methods(C, "__iter__") + return NotImplemented + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + 'Return the next item from the iterator. When exhausted, raise StopIteration' + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + return _check_methods(C, '__iter__', '__next__') + return NotImplemented + +Iterator.register(bytes_iterator) +Iterator.register(bytearray_iterator) +#Iterator.register(callable_iterator) +Iterator.register(dict_keyiterator) +Iterator.register(dict_valueiterator) +Iterator.register(dict_itemiterator) +Iterator.register(list_iterator) +Iterator.register(list_reverseiterator) +Iterator.register(range_iterator) +Iterator.register(longrange_iterator) +Iterator.register(set_iterator) +Iterator.register(str_iterator) +Iterator.register(tuple_iterator) +Iterator.register(zip_iterator) + + +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + +class Generator(Iterator): + + __slots__ = () + + def __next__(self): + """Return the next item from the generator. + When exhausted, raise StopIteration. + """ + return self.send(None) + + @abstractmethod + def send(self, value): + """Send a value into the generator. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the generator. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside generator. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Generator: + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') + return NotImplemented + +Generator.register(generator) + + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + return _check_methods(C, "__len__") + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + return _check_methods(C, "__contains__") + return NotImplemented + +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + return _check_methods(C, "__call__") + return NotImplemented + + +### SETS ### + + +class Set(Collection): + + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), redefine __le__ and __ge__, + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) < len(other): + return False + for elem in other: + if elem not in self: + return False + return True + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + __rand__ = __and__ + + def isdisjoint(self, other): + 'Return True if two sets have a null intersection.' + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + __ror__ = __or__ + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __rsub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in other + if value not in self) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + __rxor__ = __xor__ + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + """A mutable set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__, __len__, + add(), and discard(). + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Collection): + + __slots__ = () + + """A Mapping is a generic container for associating key/value + pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __iter__, and __len__. + + """ + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return KeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return ItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + +Mapping.register(mappingproxy) + + +class MappingView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + +ItemsView.register(dict_items) + + +class ValuesView(MappingView, Collection): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + + __slots__ = () + + """A MutableMapping is a generic container for associating + key/value pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __setitem__, __delitem__, + __iter__, and __len__. + + """ + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + '''D.popitem() -> (k, v), remove and return some (key, value) pair + as a 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + 'D.clear() -> None. Remove all items from D.' + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. + If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v + In either case, this is followed by: for k, v in F.items(): D[k] = v + ''' + if not args: + raise TypeError("descriptor 'update' of 'MutableMapping' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Reversible, Collection): + + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + '''S.index(value, [start, [stop]]) -> integer -- return first index of value. + Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. + ''' + if start is not None and start < 0: + start = max(len(self) + start, 0) + if stop is not None and stop < 0: + stop += len(self) + + i = start + while stop is None or i < stop: + try: + v = self[i] + if v is value or v == value: + return i + except IndexError: + break + i += 1 + raise ValueError + + def count(self, value): + 'S.count(value) -> integer -- return number of occurrences of value' + return sum(1 for v in self if v is value or v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +Sequence.register(memoryview) + + +class ByteString(Sequence): + + """This unifies bytes and bytearray. + + XXX Should add all their methods. + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + + __slots__ = () + + """All the operations on a read-write sequence. + + Concrete subclasses must provide __new__ or __init__, + __getitem__, __setitem__, __delitem__, __len__, and insert(). + + """ + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + 'S.insert(index, value) -- insert value before index' + raise IndexError + + def append(self, value): + 'S.append(value) -- append value to the end of the sequence' + self.insert(len(self), value) + + def clear(self): + 'S.clear() -> None -- remove all items from S' + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + 'S.reverse() -- reverse *IN PLACE*' + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + '''S.pop([index]) -> item -- remove and return item at index (default last). + Raise IndexError if list is empty or index is out of range. + ''' + v = self[index] + del self[index] + return v + + def remove(self, value): + '''S.remove(value) -- remove first occurrence of value. + Raise ValueError if the value is not present. + ''' + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString From 127282979364ec780e164fa114a9544ce14a3386 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:03:08 +0300 Subject: [PATCH 240/658] Comment unsupported code from _collections_abc.py --- Lib/_collections_abc.py | 274 ++++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c36398797..3158a4f16 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -40,7 +40,7 @@ dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) list_iterator = type(iter([])) -list_reverseiterator = type(iter(reversed([]))) +# list_reverseiterator = type(iter(reversed([]))) range_iterator = type(iter(range(0))) longrange_iterator = type(iter(range(1 << 1000))) set_iterator = type(iter(set())) @@ -53,21 +53,21 @@ dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) -generator = type((lambda: (yield))()) +# generator = type((lambda: (yield))()) ## coroutine ## -async def _coro(): pass -_coro = _coro() -coroutine = type(_coro) -_coro.close() # Prevent ResourceWarning -del _coro -## asynchronous generator ## -async def _ag(): yield -_ag = _ag() -async_generator = type(_ag) -del _ag +# async def _coro(): pass +# _coro = _coro() +# coroutine = type(_coro) +# _coro.close() # Prevent ResourceWarning +# del _coro +# ## asynchronous generator ## +# async def _ag(): yield +# _ag = _ag() +# async_generator = type(_ag) +# del _ag -### ONE-TRICK PONIES ### +# ## ONE-TRICK PONIES ### def _check_methods(C, *methods): mro = C.__mro__ @@ -96,148 +96,148 @@ class Hashable(metaclass=ABCMeta): return NotImplemented -class Awaitable(metaclass=ABCMeta): +# class Awaitable(metaclass=ABCMeta): - __slots__ = () +# __slots__ = () - @abstractmethod - def __await__(self): - yield +# @abstractmethod +# def __await__(self): +# yield - @classmethod - def __subclasshook__(cls, C): - if cls is Awaitable: - return _check_methods(C, "__await__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Awaitable: +# return _check_methods(C, "__await__") +# return NotImplemented -class Coroutine(Awaitable): +# class Coroutine(Awaitable): - __slots__ = () +# __slots__ = () - @abstractmethod - def send(self, value): - """Send a value into the coroutine. - Return next yielded value or raise StopIteration. - """ - raise StopIteration +# @abstractmethod +# def send(self, value): +# """Send a value into the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# raise StopIteration - @abstractmethod - def throw(self, typ, val=None, tb=None): - """Raise an exception in the coroutine. - Return next yielded value or raise StopIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val +# @abstractmethod +# def throw(self, typ, val=None, tb=None): +# """Raise an exception in the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val - def close(self): - """Raise GeneratorExit inside coroutine. - """ - try: - self.throw(GeneratorExit) - except (GeneratorExit, StopIteration): - pass - else: - raise RuntimeError("coroutine ignored GeneratorExit") +# def close(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# self.throw(GeneratorExit) +# except (GeneratorExit, StopIteration): +# pass +# else: +# raise RuntimeError("coroutine ignored GeneratorExit") - @classmethod - def __subclasshook__(cls, C): - if cls is Coroutine: - return _check_methods(C, '__await__', 'send', 'throw', 'close') - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Coroutine: +# return _check_methods(C, '__await__', 'send', 'throw', 'close') +# return NotImplemented -Coroutine.register(coroutine) +# Coroutine.register(coroutine) -class AsyncIterable(metaclass=ABCMeta): +# class AsyncIterable(metaclass=ABCMeta): - __slots__ = () +# __slots__ = () - @abstractmethod - def __aiter__(self): - return AsyncIterator() +# @abstractmethod +# def __aiter__(self): +# return AsyncIterator() - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterable: - return _check_methods(C, "__aiter__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterable: +# return _check_methods(C, "__aiter__") +# return NotImplemented -class AsyncIterator(AsyncIterable): +# class AsyncIterator(AsyncIterable): - __slots__ = () +# __slots__ = () - @abstractmethod - async def __anext__(self): - """Return the next item or raise StopAsyncIteration when exhausted.""" - raise StopAsyncIteration +# @abstractmethod +# async def __anext__(self): +# """Return the next item or raise StopAsyncIteration when exhausted.""" +# raise StopAsyncIteration - def __aiter__(self): - return self +# def __aiter__(self): +# return self - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterator: - return _check_methods(C, "__anext__", "__aiter__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterator: +# return _check_methods(C, "__anext__", "__aiter__") +# return NotImplemented -class AsyncGenerator(AsyncIterator): +# class AsyncGenerator(AsyncIterator): - __slots__ = () +# __slots__ = () - async def __anext__(self): - """Return the next item from the asynchronous generator. - When exhausted, raise StopAsyncIteration. - """ - return await self.asend(None) +# async def __anext__(self): +# """Return the next item from the asynchronous generator. +# When exhausted, raise StopAsyncIteration. +# """ +# return await self.asend(None) - @abstractmethod - async def asend(self, value): - """Send a value into the asynchronous generator. - Return next yielded value or raise StopAsyncIteration. - """ - raise StopAsyncIteration +# @abstractmethod +# async def asend(self, value): +# """Send a value into the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# raise StopAsyncIteration - @abstractmethod - async def athrow(self, typ, val=None, tb=None): - """Raise an exception in the asynchronous generator. - Return next yielded value or raise StopAsyncIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val +# @abstractmethod +# async def athrow(self, typ, val=None, tb=None): +# """Raise an exception in the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val - async def aclose(self): - """Raise GeneratorExit inside coroutine. - """ - try: - await self.athrow(GeneratorExit) - except (GeneratorExit, StopAsyncIteration): - pass - else: - raise RuntimeError("asynchronous generator ignored GeneratorExit") +# async def aclose(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# await self.athrow(GeneratorExit) +# except (GeneratorExit, StopAsyncIteration): +# pass +# else: +# raise RuntimeError("asynchronous generator ignored GeneratorExit") - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncGenerator: - return _check_methods(C, '__aiter__', '__anext__', - 'asend', 'athrow', 'aclose') - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncGenerator: +# return _check_methods(C, '__aiter__', '__anext__', +# 'asend', 'athrow', 'aclose') +# return NotImplemented -AsyncGenerator.register(async_generator) +# AsyncGenerator.register(async_generator) class Iterable(metaclass=ABCMeta): @@ -274,20 +274,20 @@ class Iterator(Iterable): return _check_methods(C, '__iter__', '__next__') return NotImplemented -Iterator.register(bytes_iterator) -Iterator.register(bytearray_iterator) -#Iterator.register(callable_iterator) -Iterator.register(dict_keyiterator) -Iterator.register(dict_valueiterator) -Iterator.register(dict_itemiterator) -Iterator.register(list_iterator) -Iterator.register(list_reverseiterator) -Iterator.register(range_iterator) -Iterator.register(longrange_iterator) -Iterator.register(set_iterator) -Iterator.register(str_iterator) -Iterator.register(tuple_iterator) -Iterator.register(zip_iterator) +# Iterator.register(bytes_iterator) +# Iterator.register(bytearray_iterator) +# Iterator.register(callable_iterator) +# Iterator.register(dict_keyiterator) +# Iterator.register(dict_valueiterator) +# Iterator.register(dict_itemiterator) +# Iterator.register(list_iterator) +# Iterator.register(list_reverseiterator) +# Iterator.register(range_iterator) +# Iterator.register(longrange_iterator) +# Iterator.register(set_iterator) +# Iterator.register(str_iterator) +# Iterator.register(tuple_iterator) +# Iterator.register(zip_iterator) class Reversible(Iterable): @@ -353,7 +353,7 @@ class Generator(Iterator): 'send', 'throw', 'close') return NotImplemented -Generator.register(generator) +# Generator.register(generator) class Sized(metaclass=ABCMeta): From b293956258f45bdade3b97ae73df423d82678e9d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:15:37 +0300 Subject: [PATCH 241/658] Use original _Environ as much as possible --- Lib/os.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8b7876c7a..8c19a89c6 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,34 +1,56 @@ from _os import * -class _Environ(): - def __init__(self, data, putenv, unsetenv): +# Change environ to automatically call putenv(), unsetenv if they exist. +from _collections_abc import MutableMapping + +class _Environ(MutableMapping): + def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): + self.encodekey = encodekey + self.decodekey = decodekey + self.encodevalue = encodevalue + self.decodevalue = decodevalue self.putenv = putenv self.unsetenv = unsetenv self._data = data def __getitem__(self, key): - return self._data[key] + try: + value = self._data[self.encodekey(key)] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) + + return self.decodevalue(value) def __setitem__(self, key, value): + key = self.encodekey(key) + value = self.encodevalue(value) self.putenv(key, value) self._data[key] = value def __delitem__(self, key): - self.unsetenv(key) - del self._data[key] + encodedkey = self.encodekey(key) + self.unsetenv(encodedkey) + try: + del self._data[encodedkey] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) def __iter__(self): # list() from dict object is an atomic operation keys = list(self._data) for key in keys: - yield key + yield self.decodekey(key) def __len__(self): return len(self._data) def __repr__(self): return 'environ({{{}}})'.format(', '.join( - ('{}: {}'.format(key, value) + ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) for key, value in self._data.items()))) def copy(self): @@ -39,13 +61,62 @@ class _Environ(): self[key] = value return self[key] -environ = _Environ(environ, putenv, unsetenv) +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None +# else: +# if "putenv" not in __all__: +# __all__.append("putenv") + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") +# else: +# if "unsetenv" not in __all__: +# __all__.append("unsetenv") + +def _createenviron(): + # if name == 'nt': + # # Where Env Var Names Must Be UPPERCASE + # def check_str(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value + # encode = check_str + # decode = str + # def encodekey(key): + # return encode(key).upper() + # data = {} + # for key, value in environ.items(): + # data[encodekey(key)] = value + # else: + # # Where Env Var Names Can Be Mixed Case + # encoding = sys.getfilesystemencoding() + # def encode(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value.encode(encoding, 'surrogateescape') + # def decode(value): + # return value.decode(encoding, 'surrogateescape') + # encodekey = encode + decode = str + encode = str + encodekey = encode + data = environ + return _Environ(data, + encodekey, decode, + encode, decode, + _putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. The optional second argument can specify an alternate default. key, default and the result are str.""" - try: - return environ[key] - except KeyError: - return default + return environ.get(key, default) From 28cf4c001043490a14cc668efd2d5d8538b27430 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 11 Apr 2019 22:18:29 +0200 Subject: [PATCH 242/658] Symbol role scan fully covered. --- vm/src/compile.rs | 68 +++++++-- vm/src/symboltable.rs | 333 ++++++++++++++++++++++++++++++------------ 2 files changed, 298 insertions(+), 103 deletions(-) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index cc52661a4..2221fde7d 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -10,13 +10,14 @@ use crate::error::CompileError; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; -use crate::symboltable::SymbolTable; +use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope}; use crate::VirtualMachine; use num_complex::Complex64; use rustpython_parser::{ast, parser}; struct Compiler { code_object_stack: Vec, + scope_stack: Vec, nxt_label: usize, source_path: Option, current_source_location: ast::Location, @@ -38,17 +39,18 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - let mut symbol_table = SymbolTable::new(); - symbol_table.scan_program(&ast); - compiler.compile_program(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_table) } Mode::Eval => { let statement = parser::parse_statement(source).map_err(CompileError::Parse)?; - compiler.compile_statement_eval(&statement) + let symbol_table = statements_to_symbol_table(&statement); + compiler.compile_statement_eval(&statement, symbol_table) } Mode::Single => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program_single(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program_single(&ast, symbol_table) } }?; @@ -75,6 +77,7 @@ impl Compiler { fn new() -> Self { Compiler { code_object_stack: Vec::new(), + scope_stack: Vec::new(), nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), @@ -97,11 +100,17 @@ impl Compiler { } fn pop_code_object(&mut self) -> CodeObject { + // self.scope_stack.pop().unwrap(); self.code_object_stack.pop().unwrap() } - fn compile_program(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { let size_before = self.code_object_stack.len(); + self.scope_stack.push(symbol_scope); self.compile_statements(&program.statements)?; assert!(self.code_object_stack.len() == size_before); @@ -113,7 +122,12 @@ impl Compiler { Ok(()) } - fn compile_program_single(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program_single( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_scope); for statement in &program.statements { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; @@ -133,7 +147,9 @@ impl Compiler { fn compile_statement_eval( &mut self, statements: &[ast::LocatedStatement], + symbol_table: SymbolScope, ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_table); for statement in statements { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; @@ -464,6 +480,7 @@ impl Compiler { line_number, name.to_string(), )); + self.enter_scope(); let mut flags = bytecode::FunctionOpArg::empty(); if have_defaults { @@ -624,6 +641,7 @@ impl Compiler { }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); // Prepare type annotations: let mut num_annotations = 0; @@ -708,6 +726,7 @@ impl Compiler { line_number, name.to_string(), )); + self.enter_scope(); let (new_body, doc_str) = get_doc(body); @@ -718,6 +737,7 @@ impl Compiler { self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { @@ -1212,6 +1232,8 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { + self.lookup_name(name); + // TODO: if global, do something else! self.emit(Instruction::LoadName { name: name.to_string(), }); @@ -1223,6 +1245,7 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1569,6 +1592,30 @@ impl Compiler { Ok(()) } + // Scope helpers: + fn enter_scope(&mut self) { + // println!("Enter scope {:?}", self.scope_stack); + // Enter first subscope! + let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0); + self.scope_stack.push(scope); + } + + fn leave_scope(&mut self) { + // println!("Leave scope {:?}", self.scope_stack); + let scope = self.scope_stack.pop().unwrap(); + assert!(scope.sub_scopes.is_empty()); + } + + fn lookup_name(&self, name: &str) -> &SymbolRole { + // println!("Looking up {:?}", name); + for scope in self.scope_stack.iter().rev() { + if let Some(role) = scope.lookup(name) { + return role; + } + } + unreachable!(); + } + // Low level helper functions: fn emit(&mut self, instruction: Instruction) { let location = self.current_source_location.clone(); @@ -1624,13 +1671,13 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option< (body, None) } - #[cfg(test)] mod tests { use super::Compiler; use crate::bytecode::CodeObject; use crate::bytecode::Constant::*; use crate::bytecode::Instruction::*; + use crate::symboltable::make_symbol_table; use rustpython_parser::parser; fn compile_exec(source: &str) -> CodeObject { @@ -1638,7 +1685,8 @@ mod tests { compiler.source_path = Some("source_path".to_string()); compiler.push_new_code_object("".to_string()); let ast = parser::parse_program(&source.to_string()).unwrap(); - compiler.compile_program(&ast).unwrap(); + let symbol_scope = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() } diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 6525e71e1..78655da53 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -8,6 +8,23 @@ load and store instructions for names. use rustpython_parser::ast; use std::collections::HashMap; +pub fn make_symbol_table(program: &ast::Program) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_program(program).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +pub fn statements_to_symbol_table(statements: &[ast::LocatedStatement]) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_statements(statements).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +#[derive(Debug)] pub enum SymbolRole { Global, Nonlocal, @@ -15,42 +32,96 @@ pub enum SymbolRole { Assigned, } -pub struct SymbolTable { - // TODO: split-up into nested scopes. - symbols: HashMap, +/// Symbolscope captures all symbols in the current scope, and +/// has a list of subscopes in this scope. +pub struct SymbolScope { + /// A set of symbols present on this scope level. + pub symbols: HashMap, + + /// A list of subscopes in the order as found in the + /// AST nodes. + pub sub_scopes: Vec, } -impl SymbolTable { +type SymbolTableResult = Result<(), String>; + +impl SymbolScope { pub fn new() -> Self { - SymbolTable { + SymbolScope { symbols: HashMap::new(), + sub_scopes: vec![], } } pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { - return self.symbols.get(name); + self.symbols.get(name) + } +} + +impl std::fmt::Debug for SymbolScope { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "SymbolScope({:?} symbols, {:?} sub scopes)", + self.symbols.len(), + self.sub_scopes.len() + ) + } +} + +pub struct SymbolTableBuilder { + // Scope stack. + pub scopes: Vec, +} + +impl SymbolTableBuilder { + pub fn new() -> Self { + SymbolTableBuilder { scopes: vec![] } } - pub fn scan_program(&mut self, program: &ast::Program) { - self.scan_statements(&program.statements); + pub fn enter_scope(&mut self) { + // Create new scope and push into scope stack. + let scope = SymbolScope::new(); + self.scopes.push(scope); } - pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) { + pub fn leave_scope(&mut self) { + // Pop scope and add to subscopes of parent scope. + let scope = self.scopes.pop().unwrap(); + self.scopes.last_mut().unwrap().sub_scopes.push(scope); + } + + pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { + self.scan_statements(&program.statements)?; + Ok(()) + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult { for statement in statements { - self.scan_statement(statement) + self.scan_statement(statement)?; } + Ok(()) } - fn scan_statement(&mut self, statement: &ast::LocatedStatement) { + fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; + } + } + Ok(()) + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult { match &statement.node { ast::Statement::Global { names } => { for name in names { - self.register_name(name, SymbolRole::Global); + self.register_name(name, SymbolRole::Global)?; } } ast::Statement::Nonlocal { names } => { for name in names { - self.register_name(name, SymbolRole::Nonlocal); + self.register_name(name, SymbolRole::Nonlocal)?; } } ast::Statement::FunctionDef { @@ -60,22 +131,25 @@ impl SymbolTable { decorator_list, returns, } => { - self.scan_expressions(decorator_list); - self.register_name(name, SymbolRole::Assigned); - for parameter in &args.args {} - for parameter in &args.kwonlyargs {} + self.scan_expressions(decorator_list)?; + self.register_name(name, SymbolRole::Assigned)?; - self.scan_expressions(&args.defaults); + self.enter_scope(); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + + self.scan_expressions(&args.defaults)?; for kw_default in &args.kw_defaults { if let Some(expression) = kw_default { - self.scan_expression(&expression); + self.scan_expression(&expression)?; } } - self.scan_statements(body); + self.scan_statements(body)?; if let Some(expression) = returns { - self.scan_expression(expression); + self.scan_expression(expression)?; } + self.leave_scope(); } ast::Statement::ClassDef { name, @@ -84,20 +158,22 @@ impl SymbolTable { keywords, decorator_list, } => { - self.register_name(name, SymbolRole::Assigned); - self.scan_statements(body); - self.scan_expressions(bases); + self.register_name(name, SymbolRole::Assigned)?; + self.enter_scope(); + self.scan_statements(body)?; + self.leave_scope(); + self.scan_expressions(bases)?; for keyword in keywords { - self.scan_expression(&keyword.value); + self.scan_expression(&keyword.value)?; } - self.scan_expressions(decorator_list); + self.scan_expressions(decorator_list)?; } - ast::Statement::Expression { expression } => self.scan_expression(expression), + ast::Statement::Expression { expression } => self.scan_expression(expression)?, ast::Statement::If { test, body, orelse } => { - self.scan_expression(test); - self.scan_statements(body); + self.scan_expression(test)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::For { @@ -106,58 +182,66 @@ impl SymbolTable { body, orelse, } => { - self.scan_expression(target); - self.scan_expression(iter); - self.scan_statements(body); + self.scan_expression(target)?; + self.scan_expression(iter)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::While { test, body, orelse } => { - self.scan_expression(test); - self.scan_statements(body); + self.scan_expression(test)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { // No symbols here. } - ast::Statement::Import { import_parts } => for part in import_parts {}, + ast::Statement::Import { import_parts } => { + for part in import_parts { + if let Some(alias) = &part.alias { + self.register_name(alias, SymbolRole::Assigned)?; + } else { + if let Some(symbol) = &part.symbol { + self.register_name(symbol, SymbolRole::Assigned)?; + } else { + self.register_name(&part.module, SymbolRole::Assigned)?; + } + } + } + } ast::Statement::Return { value } => { if let Some(expression) = value { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Statement::Assert { test, msg } => { - self.scan_expression(test); + self.scan_expression(test)?; if let Some(expression) = msg { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Statement::Delete { targets } => { - self.scan_expressions(targets); + self.scan_expressions(targets)?; } ast::Statement::Assign { targets, value } => { - self.scan_expressions(targets); - self.scan_expression(value); + self.scan_expressions(targets)?; + self.scan_expression(value)?; } - ast::Statement::AugAssign { - target, - op: _, - value, - } => { - self.scan_expression(target); - self.scan_expression(value); + ast::Statement::AugAssign { target, value, .. } => { + self.scan_expression(target)?; + self.scan_expression(value)?; } ast::Statement::With { items, body } => { for item in items { - self.scan_expression(&item.context_expr); + self.scan_expression(&item.context_expr)?; if let Some(expression) = &item.optional_vars { - self.scan_expression(expression); + self.scan_expression(expression)?; } } - self.scan_statements(body); + self.scan_statements(body)?; } ast::Statement::Try { body, @@ -165,67 +249,78 @@ impl SymbolTable { orelse, finalbody, } => { - self.scan_statements(body); + self.scan_statements(body)?; + for handler in handlers { + if let Some(expression) = &handler.typ { + self.scan_expression(expression)?; + } + if let Some(name) = &handler.name { + self.register_name(name, SymbolRole::Assigned)?; + } + self.scan_statements(&handler.body)?; + } if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } if let Some(code) = finalbody { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::Raise { exception, cause } => { if let Some(expression) = exception { - self.scan_expression(expression); + self.scan_expression(expression)?; } if let Some(expression) = cause { - self.scan_expression(expression); + self.scan_expression(expression)?; } } } + Ok(()) } - fn scan_expressions(&mut self, expressions: &[ast::Expression]) { + fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult { for expression in expressions { - self.scan_expression(expression); + self.scan_expression(expression)?; } + Ok(()) } - fn scan_expression(&mut self, expression: &ast::Expression) { + fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { match expression { - ast::Expression::Binop { a, op: _, b } => { - self.scan_expression(a); - self.scan_expression(b); + ast::Expression::Binop { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::BoolOp { a, op: _, b } => { - self.scan_expression(a); - self.scan_expression(b); + ast::Expression::BoolOp { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::Compare { vals, ops: _ } => { - self.scan_expressions(vals); + ast::Expression::Compare { vals, .. } => { + self.scan_expressions(vals)?; } ast::Expression::Subscript { a, b } => { - self.scan_expression(a); - self.scan_expression(b); + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::Attribute { value, name: _ } => { - self.scan_expression(value); + ast::Expression::Attribute { value, .. } => { + self.scan_expression(value)?; } ast::Expression::Dict { elements } => { for (key, value) in elements { - self.scan_expression(key); - self.scan_expression(value); + self.scan_expression(key)?; + self.scan_expression(value)?; } } ast::Expression::Yield { value } => { if let Some(expression) = value { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Expression::YieldFrom { value } => { - self.scan_expression(value); + self.scan_expression(value)?; } - ast::Expression::Unop { op: _, a } => { - self.scan_expression(a); + ast::Expression::Unop { a, .. } => { + self.scan_expression(a)?; } ast::Expression::True | ast::Expression::False @@ -233,40 +328,92 @@ impl SymbolTable { | ast::Expression::Ellipsis => {} ast::Expression::Number { .. } => {} ast::Expression::Starred { value } => { - self.scan_expression(value); + self.scan_expression(value)?; } ast::Expression::Bytes { .. } => {} ast::Expression::Tuple { elements } | ast::Expression::Set { elements } | ast::Expression::List { elements } | ast::Expression::Slice { elements } => { - self.scan_expressions(elements); + self.scan_expressions(elements)?; + } + ast::Expression::Comprehension { kind, generators } => { + match **kind { + ast::ComprehensionKind::GeneratorExpression { ref element } + | ast::ComprehensionKind::List { ref element } + | ast::ComprehensionKind::Set { ref element } => { + self.scan_expression(element)?; + } + ast::ComprehensionKind::Dict { ref key, ref value } => { + self.scan_expression(&key)?; + self.scan_expression(&value)?; + } + } + + for generator in generators { + self.scan_expression(&generator.target)?; + self.scan_expression(&generator.iter)?; + for if_expr in &generator.ifs { + self.scan_expression(if_expr)?; + } + } } - ast::Expression::Comprehension { kind, generators } => {} ast::Expression::Call { function, args, keywords, } => { - self.scan_expression(function); - self.scan_expressions(args); + self.scan_expression(function)?; + self.scan_expressions(args)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + } + ast::Expression::String { value } => { + self.scan_string_group(value)?; } - ast::Expression::String { value } => {} ast::Expression::Identifier { name } => { - self.register_name(name, SymbolRole::Used); + self.register_name(name, SymbolRole::Used)?; } ast::Expression::Lambda { args, body } => { - self.scan_expression(body); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + self.enter_scope(); + self.scan_expression(body)?; + self.leave_scope(); } ast::Expression::IfExpression { test, body, orelse } => { - self.scan_expression(test); - self.scan_expression(body); - self.scan_expression(orelse); + self.scan_expression(test)?; + self.scan_expression(body)?; + self.scan_expression(orelse)?; } } + Ok(()) } - fn register_name(&mut self, name: &String, role: SymbolRole) { - self.symbols.insert(name.clone(), role); + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { + match group { + ast::StringGroup::Constant { .. } => {} + ast::StringGroup::FormattedValue { value, .. } => { + self.scan_expression(value)?; + } + ast::StringGroup::Joined { values } => { + for subgroup in values { + self.scan_string_group(subgroup)?; + } + } + } + Ok(()) + } + + fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { + let current_scope = self.scopes.last_mut().unwrap(); + if let Some(old_role) = current_scope.symbols.get(name) { + debug!("TODO: {:?}", old_role); + // Role already set.. + } else { + current_scope.symbols.insert(name.to_string(), role); + } + Ok(()) } } From 6c4b09264175e93f9334ab3048fe3d8ea8ce2da6 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 10:50:45 +0300 Subject: [PATCH 243/658] Support exception __cause__ --- tests/snippets/try_exceptions.py | 32 +++++++++++++++++++ vm/src/frame.rs | 54 +++++++++++++++++++------------- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 4e3b7269a..5d7509ba5 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -95,3 +95,35 @@ except TypeError as ex: l.append(3) print('boom', type(ex)) assert l == [1, 3] + +cause = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__cause__ == None + cause = ex + raise NameError from ex +except NameError as ex2: + assert ex2.__cause__ == cause + +try: + raise ZeroDivisionError from None +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError from 5 +except TypeError: + pass + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..a3dfd6032 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -677,31 +677,17 @@ impl Frame { } bytecode::Instruction::Raise { argc } => { + let cause = match argc { + 2 => self.get_exception(vm, true)?, + _ => vm.get_none(), + }; let exception = match argc { - 1 => self.pop_value(), - 0 | 2 | 3 => panic!("Not implemented!"), + 1 | 2 => self.get_exception(vm, false)?, + 0 | 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { - info!("Exception raised: {:?}", exception); - Err(exception) - } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { - if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { - let exception = vm.new_empty_exception(exception)?; - info!("Exception raised: {:?}", exception); - Err(exception) - } else { - let msg = format!( - "Can only raise BaseException derived types, not {}", - exception - ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) - } - } else { - Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) - } + vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + Err(exception) } bytecode::Instruction::Break => { @@ -1204,6 +1190,30 @@ impl Frame { let stack = self.stack.borrow_mut(); stack[stack.len() - depth - 1].clone() } + + fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { + let exception = self.pop_value(); + if none_allowed && vm.get_none().is(&exception) { + Ok(exception) + } else if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { + Ok(exception) + } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exception)?; + Ok(exception) + } else { + let msg = format!( + "Can only raise BaseException derived types, not {}", + exception + ); + let type_error_type = vm.ctx.exceptions.type_error.clone(); + let type_error = vm.new_exception(type_error_type, msg); + Err(type_error) + } + } else { + Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) + } + } } impl fmt::Debug for Frame { From 63643e1fdba7a621102553597aae8f20bbda5b6a Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 12 Apr 2019 20:30:37 +1200 Subject: [PATCH 244/658] Implement generator.throw --- tests/snippets/generators.py | 22 +++++++++++++ vm/src/frame.rs | 11 +++++++ vm/src/obj/objgenerator.rs | 64 ++++++++++++++++++++++++------------ vm/src/vm.rs | 15 +++++++-- 4 files changed, 89 insertions(+), 23 deletions(-) diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 65044c0da..3b7dc77c3 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + r = [] @@ -42,3 +44,23 @@ def g4(): r = list(g4()) assert r == [None, (2,)] + +def catch_exception(): + try: + yield 1 + except ValueError: + yield 2 + yield 3 + + +g = catch_exception() +assert next(g) == 1 + +assert g.throw(ValueError, ValueError(), None) == 2 +assert next(g) == 3 + +g = catch_exception() +assert next(g) == 1 + +with assertRaises(KeyError): + assert g.throw(KeyError, KeyError(), None) == 2 diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..17322f21f 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -283,6 +283,17 @@ impl Frame { } } + pub fn throw( + &self, + vm: &VirtualMachine, + exception: PyObjectRef, + ) -> Result { + match self.unwind_exception(vm, exception) { + None => self.run(vm), + Some(exception) => Err(exception), + } + } + pub fn fetch_instruction(&self) -> &bytecode::Instruction { let ins2 = &self.code.instructions[*self.lasti.borrow()]; *self.lasti.borrow_mut() += 1; diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 9718d29a4..2541263f1 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -3,12 +3,13 @@ */ use crate::frame::{ExecutionResult, FrameRef}; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objtype::{isinstance, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; +#[pyclass(name = "generator", __inside_vm)] #[derive(Debug)] pub struct PyGenerator { frame: FrameRef, @@ -20,38 +21,59 @@ impl PyValue for PyGenerator { } } -impl PyGeneratorRef { +#[pyimpl(__inside_vm)] +impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { frame }.into_ref(vm) } - fn iter(self, _vm: &VirtualMachine) -> PyGeneratorRef { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyGeneratorRef, _vm: &VirtualMachine) -> PyGeneratorRef { + zelf } - fn next(self, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { self.send(vm.get_none(), vm) } - fn send(self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.frame.push_value(value.clone()); - match vm.run_frame(self.frame.clone())? { - ExecutionResult::Yield(value) => Ok(value), - ExecutionResult::Return(_value) => { - // Stop iteration! - let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); - Err(vm.new_exception(stop_iteration, "End of generator".to_string())) - } + let result = vm.run_frame(self.frame.clone())?; + handle_execution_result(result, vm) + } + + #[pymethod] + fn throw( + &self, + _exc_type: PyObjectRef, + exc_val: PyObjectRef, + _exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // TODO what should we do with the other parameters? CPython normalises them with + // PyErr_NormalizeException, do we want to do the same. + if !isinstance(&exc_val, &vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error("Can't throw non exception".to_string())); + } + let result = vm.frame_throw(self.frame.clone(), exc_val)?; + handle_execution_result(result, vm) + } +} + +fn handle_execution_result(result: ExecutionResult, vm: &VirtualMachine) -> PyResult { + match result { + ExecutionResult::Yield(value) => Ok(value), + ExecutionResult::Return(_value) => { + // Stop iteration! + let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); + Err(vm.new_exception(stop_iteration, "End of generator".to_string())) } } } -pub fn init(context: &PyContext) { - let generator_type = &context.generator_type; - extend_class!(context, generator_type, { - "__iter__" => context.new_rustfunc(PyGeneratorRef::iter), - "__next__" => context.new_rustfunc(PyGeneratorRef::next), - "send" => context.new_rustfunc(PyGeneratorRef::send) - }); +pub fn init(ctx: &PyContext) { + PyGenerator::extend_class(ctx, &ctx.generator_type); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7be05023b..5ae5b0b50 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -21,7 +21,7 @@ use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; -use crate::obj::objgenerator::PyGeneratorRef; +use crate::obj::objgenerator::PyGenerator; use crate::obj::objiter; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; @@ -94,6 +94,17 @@ impl VirtualMachine { result } + pub fn frame_throw( + &self, + frame: FrameRef, + exception: PyObjectRef, + ) -> PyResult { + self.frames.borrow_mut().push(frame.clone()); + let result = frame.throw(self, exception); + self.frames.borrow_mut().pop(); + result + } + pub fn current_frame(&self) -> Option> { let frames = self.frames.borrow(); if frames.is_empty() { @@ -374,7 +385,7 @@ impl VirtualMachine { // If we have a generator, create a new generator if code.code.is_generator { - Ok(PyGeneratorRef::new(frame, self).into_object()) + Ok(PyGenerator::new(frame, self).into_object()) } else { self.run_frame_full(frame) } From 80ec464d84f6c5ad931fad41c7c84e64f9a82c45 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 12 Apr 2019 12:25:19 +0100 Subject: [PATCH 245/658] Use dict.update to combine class and object attrs in object_dir. --- vm/src/obj/objdict.rs | 13 +++++++++++++ vm/src/obj/objobject.rs | 27 ++++++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index be85c7b2f..efd38a7fd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -239,6 +239,19 @@ impl PyDictRef { attrs } + pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult { + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + + for (key, value) in attrs { + entries + .borrow_mut() + .insert(vm, &vm.ctx.new_str(key), value)?; + } + + Ok(PyDict { entries }.into_ref(vm)) + } + fn hash(self, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 8f6c90a36..f02bbabd1 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,6 +1,6 @@ use super::objdict::PyDictRef; use super::objlist::PyList; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; @@ -118,26 +118,19 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { } pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut attributes: PyAttributes = objtype::get_attributes(obj.class()); + let attributes: PyAttributes = objtype::get_attributes(obj.class()); + + let dict = PyDictRef::from_attributes(attributes, vm)?; // Get instance attributes: - if let Some(dict) = &obj.dict { - for (key, value) in dict { - if let Some(key_string) = key.payload::() { - attributes.insert(key_string.to_string(), value.clone()); - } else { - return Err(vm.new_type_error(format!( - "Attribute is not a string: {:?}", - vm.to_pystr(&key)? - ))); - } - } + if let Some(object_dict) = &obj.dict { + vm.invoke( + vm.get_attribute(dict.clone().into_object(), "update")?, + object_dict.clone().into_object(), + )?; } - let attributes: Vec = attributes - .keys() - .map(|k| vm.ctx.new_str(k.to_string())) - .collect(); + let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k.clone()).collect(); Ok(PyList::from(attributes)) } From 908f758cbedcb0bddb3f5b32c99aa4855da668eb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 15:50:17 +0300 Subject: [PATCH 246/658] Print cause on exception --- vm/src/exceptions.rs | 15 +++++++++++++-- vm/src/frame.rs | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index d81764e25..abd670a14 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -2,7 +2,7 @@ use crate::function::PyFuncArgs; use crate::obj::objsequence; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{create_type, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,8 +18,19 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -// Print exception including traceback: +// print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { + if !vm.get_none().is(&cause) { + print_exception(vm, &cause); + println!("\nThe above exception was the direct cause of the following exception:\n"); + } + } + print_exception_inner(vm, exc) +} + +// Print exception including traceback: +pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a3dfd6032..a1da85d3a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -686,6 +686,7 @@ impl Frame { 0 | 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; + info!("Exception raised: {:?} with cause: {:?}", exception, cause); vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; Err(exception) } From 76c597f2afc21edc1c6a5158120b619cb88e7f49 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Fri, 12 Apr 2019 19:24:23 +0200 Subject: [PATCH 247/658] Add global bytecode and testcase. --- tests/snippets/global_nonlocal.py | 13 ++++++ vm/src/bytecode.rs | 12 ++++- vm/src/compile.rs | 73 ++++++++++++++++++------------- vm/src/frame.rs | 60 ++++++++++++++++++------- vm/src/symboltable.rs | 59 +++++++++++++++++-------- 5 files changed, 148 insertions(+), 69 deletions(-) create mode 100644 tests/snippets/global_nonlocal.py diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py new file mode 100644 index 000000000..48d124210 --- /dev/null +++ b/tests/snippets/global_nonlocal.py @@ -0,0 +1,13 @@ + +# Test global and nonlocal funkyness + +a = 2 + +def b(): + global a + a = 4 + +assert a == 2 +b() +assert a == 4 + diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index b24801861..2725aa586 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -38,6 +38,12 @@ bitflags! { pub type Label = usize; +#[derive(Debug, Clone, PartialEq)] +pub enum NameScope { + Local, + Global, +} + /// A Single bytecode instruction. #[derive(Debug, Clone, PartialEq)] pub enum Instruction { @@ -50,9 +56,11 @@ pub enum Instruction { }, LoadName { name: String, + scope: NameScope, }, StoreName { name: String, + scope: NameScope, }, DeleteName { name: String, @@ -322,8 +330,8 @@ impl Instruction { match self { Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)), ImportStar { name } => w!(ImportStar, name), - LoadName { name } => w!(LoadName, name), - StoreName { name } => w!(StoreName, name), + LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)), + StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)), DeleteName { name } => w!(DeleteName, name), StoreSubscript => w!(StoreSubscript), DeleteSubscript => w!(DeleteSubscript), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 2221fde7d..31e43c04d 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -171,6 +171,31 @@ impl Compiler { Ok(()) } + fn scope_for_name(&self, name: &str) -> bytecode::NameScope { + let role = self.lookup_name(name); + match role { + SymbolRole::Global => bytecode::NameScope::Global, + _ => bytecode::NameScope::Local, + } + } + + fn load_name(&mut self, name: &str) { + // TODO: if global, do something else! + let scope = self.scope_for_name(name); + self.emit(Instruction::LoadName { + name: name.to_string(), + scope, + }); + } + + fn store_name(&mut self, name: &str) { + let scope = self.scope_for_name(name); + self.emit(Instruction::StoreName { + name: name.to_string(), + scope, + }); + } + fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> { trace!("Compiling {:?}", statement); self.set_source_location(&statement.location); @@ -194,15 +219,14 @@ impl Compiler { name: module.clone(), symbol: symbol.clone(), }); - self.emit(Instruction::StoreName { - name: match alias { - Some(alias) => alias.clone(), - None => match symbol { - Some(symbol) => symbol.clone(), - None => module.clone(), - }, + let name = match alias { + Some(alias) => alias.clone(), + None => match symbol { + Some(symbol) => symbol.clone(), + None => module.clone(), }, - }); + }; + self.store_name(&name); } } } @@ -337,6 +361,7 @@ impl Compiler { self.compile_test(test, Some(end_label), None, EvalContext::Statement)?; self.emit(Instruction::LoadName { name: String::from("AssertionError"), + scope: bytecode::NameScope::Local, }); match msg { Some(e) => { @@ -544,6 +569,7 @@ impl Compiler { // Check exception type: self.emit(Instruction::LoadName { name: String::from("isinstance"), + scope: bytecode::NameScope::Local, }); self.emit(Instruction::Rotate { amount: 2 }); self.compile_expression(exc_type)?; @@ -558,9 +584,7 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { - self.emit(Instruction::StoreName { - name: alias.clone(), - }); + self.store_name(alias); } else { // Drop exception from top of stack: self.emit(Instruction::Pop); @@ -695,9 +719,7 @@ impl Compiler { self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; @@ -796,9 +818,7 @@ impl Compiler { self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); self.in_loop = was_in_loop; Ok(()) } @@ -943,9 +963,7 @@ impl Compiler { fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); } ast::Expression::Subscript { a, b } => { self.compile_expression(a)?; @@ -1232,11 +1250,7 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { - self.lookup_name(name); - // TODO: if global, do something else! - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + self.load_name(name); } ast::Expression::Lambda { args, body } => { let name = "".to_string(); @@ -1453,6 +1467,7 @@ impl Compiler { // Load iterator onto stack (passed as first argument): self.emit(Instruction::LoadName { name: String::from(".0"), + scope: bytecode::NameScope::Local, }); } else { // Evaluate iterated item: @@ -1608,12 +1623,8 @@ impl Compiler { fn lookup_name(&self, name: &str) -> &SymbolRole { // println!("Looking up {:?}", name); - for scope in self.scope_stack.iter().rev() { - if let Some(role) = scope.lookup(name) { - return role; - } - } - unreachable!(); + let scope = self.scope_stack.last().unwrap(); + scope.lookup(name).unwrap() } // Low level helper functions: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2570c0934..a72ed6a76 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -315,8 +315,14 @@ impl Frame { ref symbol, } => self.import(vm, name, symbol), bytecode::Instruction::ImportStar { ref name } => self.import_star(vm, name), - bytecode::Instruction::LoadName { ref name } => self.load_name(vm, name), - bytecode::Instruction::StoreName { ref name } => self.store_name(vm, name), + bytecode::Instruction::LoadName { + ref name, + ref scope, + } => self.load_name(vm, name, scope), + bytecode::Instruction::StoreName { + ref name, + ref scope, + } => self.store_name(vm, name, scope), bytecode::Instruction::DeleteName { ref name } => self.delete_name(vm, name), bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), @@ -984,9 +990,21 @@ impl Frame { vm.call_method(context_manager, "__exit__", args) } - fn store_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { + fn store_name( + &self, + vm: &VirtualMachine, + name: &str, + scope: &bytecode::NameScope, + ) -> FrameResult { let obj = self.pop_value(); - self.scope.store_name(&vm, name, obj); + match scope { + bytecode::NameScope::Global => { + self.scope.globals.set_item(name, obj, vm)?; + } + bytecode::NameScope::Local => { + self.scope.store_name(&vm, name, obj); + } + } Ok(None) } @@ -995,19 +1013,27 @@ impl Frame { Ok(None) } - fn load_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - match self.scope.load_name(&vm, name) { - Some(value) => { - self.push_value(value); - Ok(None) - } - None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - Err(name_error) - } - } + fn load_name( + &self, + vm: &VirtualMachine, + name: &str, + scope: &bytecode::NameScope, + ) -> FrameResult { + let value = match scope { + bytecode::NameScope::Global => self.scope.globals.get_item(name, vm)?, + bytecode::NameScope::Local => match self.scope.load_name(&vm, name) { + Some(value) => value, + None => { + let name_error_type = vm.ctx.exceptions.name_error.clone(); + let msg = format!("name '{}' is not defined", name); + let name_error = vm.new_exception(name_error_type, msg); + return Err(name_error); + } + }, + }; + + self.push_value(value); + Ok(None) } fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 78655da53..2d6b49884 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -105,9 +105,14 @@ impl SymbolTableBuilder { fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { for parameter in parameters { - if let Some(annotation) = ¶meter.annotation { - self.scan_expression(&annotation)?; - } + self.scan_parameter(parameter)?; + } + Ok(()) + } + + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; } Ok(()) } @@ -134,16 +139,7 @@ impl SymbolTableBuilder { self.scan_expressions(decorator_list)?; self.register_name(name, SymbolRole::Assigned)?; - self.enter_scope(); - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - - self.scan_expressions(&args.defaults)?; - for kw_default in &args.kw_defaults { - if let Some(expression) = kw_default { - self.scan_expression(&expression)?; - } - } + self.enter_function(args)?; self.scan_statements(body)?; if let Some(expression) = returns { @@ -376,9 +372,7 @@ impl SymbolTableBuilder { self.register_name(name, SymbolRole::Used)?; } ast::Expression::Lambda { args, body } => { - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - self.enter_scope(); + self.enter_function(args)?; self.scan_expression(body)?; self.leave_scope(); } @@ -391,6 +385,27 @@ impl SymbolTableBuilder { Ok(()) } + fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { + // Evaulate eventual default parameters: + self.scan_expressions(&args.defaults)?; + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression)?; + } + } + + self.enter_scope(); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter(name)?; + } + Ok(()) + } + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { match group { ast::StringGroup::Constant { .. } => {} @@ -408,9 +423,15 @@ impl SymbolTableBuilder { fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { let current_scope = self.scopes.last_mut().unwrap(); - if let Some(old_role) = current_scope.symbols.get(name) { - debug!("TODO: {:?}", old_role); - // Role already set.. + if let Some(_old_role) = current_scope.symbols.get(name) { + // Role already set.. + // debug!("TODO: {:?}", old_role); + match role { + SymbolRole::Global => return Err("global must appear first".to_string()), + _ => { + // Ok? + } + } } else { current_scope.symbols.insert(name.to_string(), role); } From 585e6e5428c42e0615eb6990315ee0e39a6e5d40 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Fri, 12 Apr 2019 19:28:09 +0200 Subject: [PATCH 248/658] Drop IntoPyNativeFunc impl to fix Nightly/Beta breakages. --- vm/src/function.rs | 6 ------ wasm/lib/src/vm_class.rs | 37 ++++++++++++++++++------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index 305c4de5e..514fe4cb6 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -437,12 +437,6 @@ where } } -impl IntoPyNativeFunc for PyNativeFunc { - fn into_func(self) -> PyNativeFunc { - self - } -} - pub struct OwnedParam(std::marker::PhantomData); pub struct RefParam(std::marker::PhantomData); diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 5efa7c849..ff72dddbb 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -199,15 +199,15 @@ impl WASMVirtualMachine { fn error() -> JsValue { TypeError::new("Unknown stdout option, please pass a function or 'console'").into() } - let print_fn: Box PyResult> = - if let Some(s) = stdout.as_string() { - match s.as_str() { - "console" => Box::new(wasm_builtins::builtin_print_console), - _ => return Err(error()), - } - } else if stdout.is_function() { - let func = js_sys::Function::from(stdout); - Box::new(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { + let print_fn: PyObjectRef = if let Some(s) = stdout.as_string() { + match s.as_str() { + "console" => vm.ctx.new_rustfunc(wasm_builtins::builtin_print_console), + _ => return Err(error()), + } + } else if stdout.is_function() { + let func = js_sys::Function::from(stdout); + vm.ctx + .new_rustfunc(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { func.call1( &JsValue::UNDEFINED, &wasm_builtins::format_print_args(vm, args)?.into(), @@ -215,16 +215,15 @@ impl WASMVirtualMachine { .map_err(|err| convert::js_to_py(vm, err))?; Ok(vm.get_none()) }) - } else if stdout.is_undefined() || stdout.is_null() { - fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { - Ok(vm.get_none()) - } - Box::new(noop) - } else { - return Err(error()); - }; - vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn)) - .unwrap(); + } else if stdout.is_undefined() || stdout.is_null() { + fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) + } + vm.ctx.new_rustfunc(noop) + } else { + return Err(error()); + }; + vm.set_attr(&vm.builtins, "print", print_fn).unwrap(); Ok(()) })? } From a91cfd16bfb9b58057b2507aed14ee1cd3ebfb61 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 12 Apr 2019 14:28:26 -0400 Subject: [PATCH 249/658] Fix WASM import error --- wasm/demo/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index 9b58afe39..602e25241 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -1,6 +1,6 @@ import './style.css'; import 'codemirror/lib/codemirror.css'; -import 'xterm/dist/xterm.css'; +import 'xterm/lib/xterm.css'; // A dependency graph that contains any wasm must all be imported // asynchronously. This `index.js` file does the single async import, so From 20e0cc8bd771ccfea1fc0269db031070e03abf76 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 23:06:53 +0300 Subject: [PATCH 250/658] Use assertRaises --- tests/snippets/try_exceptions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 5d7509ba5..821d4e8c8 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -1,4 +1,4 @@ - +from testutils import assertRaises try: raise BaseException() @@ -117,10 +117,8 @@ try: except ZeroDivisionError as ex: assert ex.__cause__ == None -try: +with assertRaises(TypeError): raise ZeroDivisionError from 5 -except TypeError: - pass try: raise ZeroDivisionError from NameError From 256abe63b6ccdb5df6f86fe91f8e3ca2c7f3d211 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 23:08:26 +0300 Subject: [PATCH 251/658] Use vm.new_type_error --- vm/src/frame.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a1da85d3a..d67fd6b8d 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1207,9 +1207,7 @@ impl Frame { "Can only raise BaseException derived types, not {}", exception ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) + Err(vm.new_type_error(msg)) } } else { Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) From bd78f7e12bfe89a8a56185b167e599e8bb13b73c Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 13 Apr 2019 08:21:37 +0200 Subject: [PATCH 252/658] add capitalize --- tests/snippets/bytes.py | 6 +++++- vm/src/obj/objbyteinner.rs | 9 +++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 0ffbc29f8..1ad23ea18 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -109,11 +109,14 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower, capitalize l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytes().capitalize() == bytes() # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" @@ -127,3 +130,4 @@ try: bytes.fromhex("6Z2") except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 532504b02..9bedb0bf0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -305,6 +305,15 @@ impl PyByteInner { self.elements.to_ascii_uppercase() } + pub fn capitalize(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::new(); + if let Some((first, second)) = self.elements.split_first() { + new.push(first.to_ascii_uppercase()); + second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); + } + new + } + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { let bla = self .elements diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2c8edca6a..953e5b2bb 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -209,6 +209,11 @@ impl PyBytesRef { Ok(vm.ctx.new_bytes(self.inner.upper(vm))) } + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.capitalize(vm))) + } + #[pymethod(name = "hex")] fn hex(self, vm: &VirtualMachine) -> PyResult { self.inner.hex(vm) From 73dbcfbe0108f9a7cffb83ef9ab0d7540e11b8a3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Fri, 12 Apr 2019 16:30:57 +0200 Subject: [PATCH 253/658] Add bytes.center fix str.center add some tests introduce is_byte --- tests/snippets/bytes.py | 27 ++++++++++++++++++++++++ tests/snippets/strings.py | 23 +++++++++++++++++++++ vm/src/obj/objbyteinner.rs | 40 ++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 42 +++++++++++++++++++++++++++++++++++++- vm/src/obj/objstr.rs | 18 ++++++++++++++-- 5 files changed, 147 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 1ad23ea18..d2a60943c 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -131,3 +131,30 @@ try: except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" +# center +assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [b"kok".center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +b"kok".center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".center(2, "a") +with assertRaises(TypeError): + b"b".center(2, b"ba") +b"kok".center(5, bytearray(b"x")) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 31511db11..adcc106bb 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -55,6 +55,29 @@ assert b.rstrip() == ' hallo' c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' +assert ["koki".center(i, "|") for i in range(3, 10)] == [ + "koki", + "koki", + "|koki", + "|koki|", + "||koki|", + "||koki||", + "|||koki||", +] + + +assert ["kok".center(i, "|") for i in range(2, 10)] == [ + "kok", + "kok", + "kok|", + "|kok|", + "|kok||", + "||kok||", + "||kok|||", + "|||kok|||", +] + + # assert c.isascii() assert c.index('a') == 1 assert c.rindex('l') == 3 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 9bedb0bf0..64ac24a3d 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,4 +1,5 @@ use crate::pyobject::PyObjectRef; +use num_bigint::BigInt; use crate::function::OptionalArg; @@ -13,8 +14,12 @@ use std::hash::{Hash, Hasher}; use super::objint; use super::objsequence::PySliceableSequence; use crate::obj::objint::PyInt; +use num_integer::Integer; use num_traits::ToPrimitive; +use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; +use super::objbytes::PyBytes; + #[derive(Debug, Default, Clone)] pub struct PyByteInner { pub elements: Vec, @@ -355,4 +360,39 @@ impl PyByteInner { .map(|x| x.unwrap()) .collect::>()) } + + pub fn center(&self, width: &BigInt, fillbyte: u8, _vm: &VirtualMachine) -> Vec { + let width = width.to_usize().unwrap(); + + // adjust right et left side + if width <= self.len() { + return self.elements.clone(); + } + let diff: usize = width - self.len(); + let mut ln: usize = diff / 2; + let mut rn: usize = ln; + + if diff.is_odd() && self.len() % 2 == 0 { + ln += 1 + } + + if diff.is_odd() && self.len() % 2 != 0 { + rn += 1 + } + + // merge all + let mut res = vec![fillbyte; ln]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; rn][..]); + + res + } +} + +pub fn is_byte(obj: &PyObjectRef) -> Option> { + match_class!(obj.clone(), + + i @ PyBytes => Some(i.get_value().to_vec()), + j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), + _ => None) } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 953e5b2bb..f183738c3 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::PyByteInner; +use super::objbyteinner::{is_byte, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -34,6 +34,10 @@ impl PyBytes { inner: PyByteInner { elements }, } } + + pub fn get_value(&self) -> &[u8] { + &self.inner.elements + } } impl Deref for PyBytes { @@ -229,6 +233,42 @@ impl PyBytesRef { obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) ) } + + #[pymethod(name = "center")] + fn center( + self, + width: PyObjectRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let sym = if let OptionalArg::Present(v) = fillbyte { + match is_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + } else { + 32 // default is space + }; + + match_class!(width, + i @PyInt => Ok(vm.ctx.new_bytes(self.inner.center(i.as_bigint(), sym, vm))), + obj => {Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)))} + ) + } } #[derive(Debug)] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 62cc923b1..2d5f1f8e1 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -691,8 +691,22 @@ impl PyStringRef { ) -> PyResult { let value = &self.value; let rep_char = Self::get_fill_char(&rep, vm)?; - let left_buff: usize = (len - value.len()) / 2; - let right_buff = len - value.len() - left_buff; + let value_len = self.value.chars().count(); + + if len <= value_len { + return Ok(value.to_string()); + } + let diff: usize = len - value_len; + let mut left_buff: usize = diff / 2; + let mut right_buff: usize = left_buff; + + if diff % 2 != 0 && value_len % 2 == 0 { + left_buff += 1 + } + + if diff % 2 != 0 && value_len % 2 != 0 { + right_buff += 1 + } Ok(format!( "{}{}{}", rep_char.repeat(left_buff), From 83ea419c63515f1a05910bfcc99209dd4bbb0e66 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sat, 13 Apr 2019 11:55:10 +0200 Subject: [PATCH 254/658] Add load and store of global to name protocol. --- vm/src/frame.rs | 42 +++++++++++++++++++++++++++--------------- vm/src/symboltable.rs | 23 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a72ed6a76..e8276d1c8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -129,6 +129,8 @@ pub trait NameProtocol { fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn delete_name(&self, vm: &VirtualMachine, name: &str); fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); } impl NameProtocol for Scope { @@ -162,6 +164,14 @@ impl NameProtocol for Scope { fn delete_name(&self, vm: &VirtualMachine, key: &str) { self.get_locals().del_item(key, vm).unwrap(); } + + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { + self.globals.get_item_option(name, vm).unwrap() + } + + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.globals.set_item(name, value, vm).unwrap(); + } } #[derive(Clone, Debug)] @@ -994,12 +1004,12 @@ impl Frame { &self, vm: &VirtualMachine, name: &str, - scope: &bytecode::NameScope, + name_scope: &bytecode::NameScope, ) -> FrameResult { let obj = self.pop_value(); - match scope { + match name_scope { bytecode::NameScope::Global => { - self.scope.globals.set_item(name, obj, vm)?; + self.scope.store_global(vm, name, obj); } bytecode::NameScope::Local => { self.scope.store_name(&vm, name, obj); @@ -1017,19 +1027,21 @@ impl Frame { &self, vm: &VirtualMachine, name: &str, - scope: &bytecode::NameScope, + name_scope: &bytecode::NameScope, ) -> FrameResult { - let value = match scope { - bytecode::NameScope::Global => self.scope.globals.get_item(name, vm)?, - bytecode::NameScope::Local => match self.scope.load_name(&vm, name) { - Some(value) => value, - None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - return Err(name_error); - } - }, + let optional_value = match name_scope { + bytecode::NameScope::Global => self.scope.load_global(vm, name), + bytecode::NameScope::Local => self.scope.load_name(&vm, name), + }; + + let value = match optional_value { + Some(value) => value, + None => { + let name_error_type = vm.ctx.exceptions.name_error.clone(); + let msg = format!("name '{}' is not defined", name); + let name_error = vm.new_exception(name_error_type, msg); + return Err(name_error); + } }; self.push_value(value); diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 2d6b49884..d7f30c870 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -111,6 +111,17 @@ impl SymbolTableBuilder { } fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.register_name(¶meter.arg, SymbolRole::Assigned) + } + + fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter_annotation(parameter)?; + } + Ok(()) + } + + fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { if let Some(annotation) = ¶meter.annotation { self.scan_expression(&annotation)?; } @@ -394,7 +405,19 @@ impl SymbolTableBuilder { } } + // Annotations are scanned in outer scope: + self.scan_parameters_annotations(&args.args)?; + self.scan_parameters_annotations(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter_annotation(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter_annotation(name)?; + } + self.enter_scope(); + + // Fill scope with parameter names: self.scan_parameters(&args.args)?; self.scan_parameters(&args.kwonlyargs)?; if let ast::Varargs::Named(name) = &args.vararg { From 16644daffa2fc7e140e02f1737e8be32963e6fb0 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 12:04:33 -0700 Subject: [PATCH 255/658] Fix sub-crate compilation --- derive/src/pyclass.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 433d5911e..e072db32d 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -16,11 +16,11 @@ enum ClassItem { }, } -fn meta_to_vec(meta: Meta) -> Result, Meta> { +fn meta_to_vec(meta: Meta) -> Option> { match meta { - Meta::Word(_) => Ok(Vec::new()), - Meta::List(list) => Ok(list.nested.into_iter().collect()), - Meta::NameValue(_) => Err(meta), + Meta::Word(_) => Some(Vec::new()), + Meta::List(list) => Some(list.nested.into_iter().collect()), + Meta::NameValue(_) => None, } } From 883946102849401a5953c4758e0ccffda6e38b61 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 12:45:09 -0700 Subject: [PATCH 256/658] str: improve {is,}title impl and add tests --- vm/src/obj/objstr.rs | 126 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 20 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 0e16a8b3b..3e0bcccb6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -43,6 +43,14 @@ impl PyString { } } +impl From<&str> for PyString { + fn from(s: &str) -> PyString { + PyString { + value: s.to_string(), + } + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { @@ -396,9 +404,33 @@ impl PyString { } } + /// Return a titlecased version of the string where words start with an + /// uppercase character and the remaining characters are lowercase. #[pymethod] fn title(&self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + let mut title = String::new(); + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_lowercase() { + if !previous_is_cased { + title.extend(c.to_uppercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else if c.is_uppercase() { + if previous_is_cased { + title.extend(c.to_lowercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else { + previous_is_cased = false; + title.push(c); + } + } + title } #[pymethod] @@ -609,13 +641,34 @@ impl PyString { vm.ctx.new_tuple(new_tup) } + /// Return `true` if the sequence is ASCII titlecase and the sequence is not + /// empty, `false` otherwise. #[pymethod] fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { - false - } else { - self.value.split(' ').all(|word| word == make_title(word)) + return false; } + + let mut cased = false; + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_uppercase() { + if previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else if c.is_lowercase() { + if !previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else { + previous_is_cased = false; + } + } + cased } #[pymethod] @@ -967,22 +1020,55 @@ fn adjust_indices( } } -// helper function to title strings -fn make_title(s: &str) -> String { - let mut titled_str = String::new(); - let mut capitalize_char: bool = true; - for c in s.chars() { - if c.is_alphabetic() { - if !capitalize_char { - titled_str.push(c); - } else if capitalize_char { - titled_str.push(c.to_ascii_uppercase()); - capitalize_char = false; - } - } else { - titled_str.push(c); - capitalize_char = true; +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn str_title() { + let vm = VirtualMachine::new(); + + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyString::from(input).title(&vm), String::from(title)); + } + } + + #[test] + fn str_istitle() { + let vm = VirtualMachine::new(); + + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + ]; + + for s in pos { + assert!(PyString::from(s).istitle(&vm)); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyString::from(s).istitle(&vm)); } } - titled_str } From 1f8181b56160b4591f17da22467553750ae00f7e Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 09:40:59 +1200 Subject: [PATCH 257/658] Update formatting to be consistent with rustfmt included in 1.34 --- parser/src/lexer.rs | 6 +++++- vm/src/obj/objbytes.rs | 10 +++++----- vm/src/obj/objsuper.rs | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index e42e3b248..dfc9ecd51 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -826,7 +826,11 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + tok_end, + ))); } } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 279dbefe0..fd9bfd5bf 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -62,13 +62,13 @@ pub fn init(context: &PyContext) { PyBytesRef::extend_class(context, &context.bytes_type); let bytes_type = &context.bytes_type; extend_class!(context, bytes_type, { -"fromhex" => context.new_rustfunc(PyBytesRef::fromhex), -}); + "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), + }); let bytesiterator_type = &context.bytesiterator_type; extend_class!(context, bytesiterator_type, { -"__next__" => context.new_rustfunc(PyBytesIteratorRef::next), -"__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), -}); + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } #[pyimpl(__inside_vm)] diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 94fe0fc7b..194212a49 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -127,8 +127,10 @@ fn super_new( match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { From 6650ad895a3ca925556f802efc6c1d3b91ef7b52 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 10:07:50 +1200 Subject: [PATCH 258/658] Fix a bunch of clippy warnings --- vm/src/compile.rs | 4 ++-- vm/src/frame.rs | 6 +++--- vm/src/function.rs | 2 +- vm/src/obj/objbyteinner.rs | 3 +-- vm/src/obj/objdict.rs | 2 +- vm/src/obj/objint.rs | 2 +- vm/src/obj/objrange.rs | 4 ++-- vm/src/obj/objset.rs | 12 ++++++------ vm/src/obj/objslice.rs | 2 +- vm/src/obj/objtype.rs | 2 +- vm/src/pyobject.rs | 12 +++--------- vm/src/stdlib/os.rs | 4 ++-- 12 files changed, 24 insertions(+), 31 deletions(-) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index d5a4b48ec..163e2fcf1 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -871,8 +871,8 @@ impl Compiler { vals: &[ast::Expression], ops: &[ast::Comparison], ) -> Result<(), CompileError> { - assert!(ops.len() > 0); - assert!(vals.len() == ops.len() + 1); + assert!(!ops.is_empty()); + assert_eq!(vals.len(), ops.len() + 1); let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 14154016d..064951770 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1215,9 +1215,9 @@ impl Frame { fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { let exception = self.pop_value(); - if none_allowed && vm.get_none().is(&exception) { - Ok(exception) - } else if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { + if none_allowed && vm.get_none().is(&exception) + || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) + { Ok(exception) } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { diff --git a/vm/src/function.rs b/vm/src/function.rs index 6e421ac98..9ed237a50 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -286,7 +286,7 @@ pub struct Args(Vec); impl Args> { pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx - .new_tuple(self.0.into_iter().map(|obj| obj.into_object()).collect()) + .new_tuple(self.0.into_iter().map(PyRef::into_object).collect()) } } diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 64ac24a3d..d0a63633a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -356,8 +356,7 @@ impl PyByteInner { .collect::>() .chunks(2) .map(|x| x.to_vec().iter().collect::()) - .map(|x| u8::from_str_radix(&x, 16)) - .map(|x| x.unwrap()) + .map(|x| u8::from_str_radix(&x, 16).unwrap()) .collect::>()) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index efd38a7fd..fd77107ee 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -118,7 +118,7 @@ impl PyDictRef { } } } - return Ok(true); + Ok(true) } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 5be50f6c4..52f321adf 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -518,7 +518,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult Ok(f.to_f64().to_bigint().unwrap()), s @ PyString => { i32::from_str_radix(s.as_str(), base) - .map(|i| BigInt::from(i)) + .map(BigInt::from) .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 2820dd9b4..00b4e0e2e 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -144,8 +144,8 @@ impl PyRange { vm: &VirtualMachine, ) -> PyResult { PyRange { - start: start, - stop: stop, + start, + stop, step: step .into_option() .unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)), diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 5ec4fca46..b8f9f6d3b 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -262,7 +262,7 @@ impl PySetInner { perform_action_with_hash(vm, &mut self.elements, &item, &discard) } - fn clear(&mut self) -> () { + fn clear(&mut self) { self.elements.clear(); } @@ -449,11 +449,11 @@ impl PySetRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let inner = self.inner.borrow(); let s = if inner.len() == 0 { - format!("set()") + "set()".to_string() } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { inner.repr(vm)? } else { - format!("set(...)") + "set(...)".to_string() }; Ok(vm.new_str(s)) } @@ -470,7 +470,7 @@ impl PySetRef { self.inner.borrow_mut().discard(&item, vm) } - fn clear(self, _vm: &VirtualMachine) -> () { + fn clear(self, _vm: &VirtualMachine) { self.inner.borrow_mut().clear() } @@ -656,11 +656,11 @@ impl PyFrozenSetRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let inner = &self.inner; let s = if inner.len() == 0 { - format!("frozenset()") + "frozenset()".to_string() } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { format!("frozenset({})", inner.repr(vm)?) } else { - format!("frozenset(...)") + "frozenset(...)".to_string() }; Ok(vm.new_str(s)) } diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index c82a3ca6e..31fcfa52f 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -63,7 +63,7 @@ fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { step: step.map(|x| objint::get_value(x).clone()), } .into_ref_with_type(vm, cls.clone().downcast().unwrap()) - .map(|x| x.into_object()) + .map(PyRef::into_object) } fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index b1b50dfbd..819f85088 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -235,7 +235,7 @@ pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { let (typ, name, bases, dict) = args.bind(vm)?; - type_new_class(vm, typ, name, bases, dict).map(|x| x.into_object()) + type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { Err(vm.new_type_error(format!(": type_new: {:?}", args))) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index f91886996..3b21482ba 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -87,7 +87,6 @@ pub type PyAttributes = HashMap; impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { let type_name = self.class().name.clone(); // We don't have access to a vm, so just assume that if its parent's name @@ -698,7 +697,7 @@ impl PyContext { pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ: class, - dict: dict, + dict, payload: objobject::PyInstance, } .into_ref() @@ -1065,7 +1064,7 @@ where fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { if let Ok(method) = vm.get_method(obj.clone(), "__iter__") { Ok(PyIterable { - method: method, + method, _item: std::marker::PhantomData, }) } else if vm.get_method(obj.clone(), "__getitem__").is_ok() { @@ -1170,12 +1169,7 @@ where T: Sized + PyObjectPayload, { pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { - PyObject { - typ, - dict: dict, - payload, - } - .into_ref() + PyObject { typ, dict, payload }.into_ref() } // Move this object into a reference object, transferring ownership. diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 26707d3ca..f397b2d8a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -174,11 +174,11 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) { env::set_var(&key.value, &value.value) } -fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) { env::remove_var(&key.value) } From d24dd170e9b299839a3c98f03ab21ef45c1c8584 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 12:09:54 +1200 Subject: [PATCH 259/658] Changes to RustPython to support importing and running unittest --- vm/src/builtins.rs | 18 ++++++------------ vm/src/eval.rs | 7 +------ vm/src/import.rs | 11 ++++------- vm/src/obj/objclassmethod.rs | 2 ++ vm/src/obj/objfunction.rs | 6 ++++++ vm/src/obj/objobject.rs | 16 +++++++++++++++- vm/src/pyobject.rs | 4 +++- vm/src/stdlib/io.rs | 6 +++++- vm/src/stdlib/mod.rs | 2 ++ vm/src/stdlib/thread.rs | 30 ++++++++++++++++++++++++++++++ vm/src/stdlib/types.rs | 1 + vm/src/sysmodule.rs | 8 ++++++++ vm/src/vm.rs | 5 +++++ 13 files changed, 88 insertions(+), 28 deletions(-) create mode 100644 vm/src/stdlib/thread.rs diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 6a94b8362..8f7721609 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -102,10 +102,8 @@ fn builtin_compile( } }; - compile::compile(vm, &source, &mode, filename.value.to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }) + compile::compile(vm, &source, &mode, filename.value.to_string()) + .map_err(|err| vm.new_syntax_error(&err)) } fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { @@ -151,10 +149,8 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else { return Err(vm.new_type_error("code argument must be str or code object".to_string())); }; @@ -181,10 +177,8 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) { code_obj } else { diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 6c7b3ca54..21bc9dadf 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,7 +1,5 @@ extern crate rustpython_parser; -use std::error::Error; - use crate::compile; use crate::frame::Scope; use crate::pyobject::PyResult; @@ -13,10 +11,7 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) debug!("Code object: {:?}", bytecode); vm.run_code_obj(bytecode, scope) } - Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - Err(vm.new_exception(syntax_error, err.description().to_string())) - } + Err(err) => Err(vm.new_syntax_error(&err)), } } diff --git a/vm/src/import.rs b/vm/src/import.rs index 14c0a7566..041fab650 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -2,7 +2,6 @@ * Import mechanics */ -use std::error::Error; use std::path::PathBuf; use crate::compile; @@ -25,17 +24,14 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let file_path = find_source(vm, current_path, module) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), e.description().to_string()))?; + .map_err(|e| vm.new_exception(import_error.clone(), format!("{}", e)))?; let code_obj = compile::compile( vm, &source, &compile::Mode::Exec, file_path.to_str().unwrap().to_string(), ) - .map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.description().to_string()) - })?; + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); @@ -64,12 +60,13 @@ fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result paths.insert(0, current_path); + let rel_name = name.replace('.', "/"); let suffixes = [".py", "/__init__.py"]; let mut file_paths = vec![]; for path in paths { for suffix in suffixes.iter() { let mut file_path = path.clone(); - file_path.push(format!("{}{}", name, suffix)); + file_path.push(format!("{}{}", rel_name, suffix)); file_paths.push(file_path); } } diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index ffacfdc07..7a8d90d1c 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -9,6 +9,8 @@ pub struct PyClassMethod { pub type PyClassMethodRef = PyRef; impl PyValue for PyClassMethod { + const HAVE_DICT: bool = true; + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.classmethod_type() } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 01e4b586f..05b66c346 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,4 +1,5 @@ use crate::frame::Scope; +use crate::function::{Args, KwArgs}; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objtuple::PyTupleRef; @@ -40,6 +41,10 @@ impl PyValue for PyFunction { } impl PyFunctionRef { + fn call(self, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.into_object(), (&args, &kwargs)) + } + fn code(self, _vm: &VirtualMachine) -> PyCodeRef { self.code.clone() } @@ -76,6 +81,7 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), + "__call__" => context.new_rustfunc(PyFunctionRef::call), "__code__" => context.new_property(PyFunctionRef::code), "__defaults__" => context.new_property(PyFunctionRef::defaults), "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index f02bbabd1..47cfa008d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -167,7 +167,11 @@ pub fn init(context: &PyContext) { "__ge__" => context.new_rustfunc(object_ge), "__setattr__" => context.new_rustfunc(object_setattr), "__delattr__" => context.new_rustfunc(object_delattr), - "__dict__" => context.new_property(object_dict), + "__dict__" => + PropertyBuilder::new(context) + .add_getter(object_dict) + .add_setter(object_dict_setter) + .create(), "__dir__" => context.new_rustfunc(object_dir), "__hash__" => context.new_rustfunc(object_hash), "__str__" => context.new_rustfunc(object_str), @@ -203,6 +207,16 @@ fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult } } +fn object_dict_setter( + _instance: PyObjectRef, + _value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on am object isn't yet implemented".to_string(), + )) +} + fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_str.value; trace!("object.__getattribute__({:?}, {:?})", obj, name); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 3b21482ba..f369bab4a 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1191,6 +1191,8 @@ impl PyObject { } pub trait PyValue: fmt::Debug + Sized + 'static { + const HAVE_DICT: bool = false; + fn class(vm: &VirtualMachine) -> PyClassRef; fn into_ref(self, vm: &VirtualMachine) -> PyRef { @@ -1203,7 +1205,7 @@ pub trait PyValue: fmt::Debug + Sized + 'static { fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyClassRef) -> PyResult> { let class = Self::class(vm); if objtype::issubclass(&cls, &class) { - let dict = if cls.is(&class) { + let dict = if !Self::HAVE_DICT && cls.is(&class) { None } else { Some(vm.ctx.new_dict()) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index bd33eac75..d864519f5 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -95,6 +95,9 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +// TODO Check if closed, then if so raise ValueError +fn io_base_flush(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); vm.set_attr(buffered, "raw", raw.clone())?; @@ -369,7 +372,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), - "__exit__" => ctx.new_rustfunc(io_base_cm_exit) + "__exit__" => ctx.new_rustfunc(io_base_cm_exit), + "flush" => ctx.new_rustfunc(io_base_flush) }); // IOBase Subclasses diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 9b0d0b345..0933b7192 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -9,6 +9,7 @@ mod random; mod re; pub mod socket; mod string; +mod thread; mod time_module; mod tokenize; mod types; @@ -41,6 +42,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("random".to_string(), Box::new(random::make_module)); modules.insert("string".to_string(), Box::new(string::make_module)); modules.insert("struct".to_string(), Box::new(pystruct::make_module)); + modules.insert("_thread".to_string(), Box::new(thread::make_module)); modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); modules.insert("types".to_string(), Box::new(types::make_module)); diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs new file mode 100644 index 000000000..8de881893 --- /dev/null +++ b/vm/src/stdlib/thread.rs @@ -0,0 +1,30 @@ +/// Implementation of the _thread module, currently noop implementation as RustPython doesn't yet +/// support threading +use super::super::pyobject::PyObjectRef; +use crate::function::PyFuncArgs; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +fn rlock_acquire(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) +} + +fn rlock_release(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + +fn get_ident(_vm: &VirtualMachine) -> u32 { + 1 +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let rlock_type = py_class!(ctx, "_thread.RLock", ctx.object(), { + "acquire" => ctx.new_rustfunc(rlock_acquire), + "release" => ctx.new_rustfunc(rlock_release), + }); + + py_module!(vm, "_thread", { + "RLock" => rlock_type, + "get_ident" => ctx.new_rustfunc(get_ident) + }) +} diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs index ad76a80a4..e224a93c6 100644 --- a/vm/src/stdlib/types.rs +++ b/vm/src/stdlib/types.rs @@ -31,6 +31,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { py_module!(vm, "types", { "new_class" => ctx.new_rustfunc(types_new_class), "FunctionType" => ctx.function_type(), + "MethodType" => ctx.bound_method_type(), "LambdaType" => ctx.function_type(), "CodeType" => ctx.code_type(), "FrameType" => ctx.frame_type() diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 2f8690448..97de7a425 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,6 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -39,6 +40,11 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } +// TODO implement string interning, this will be key for performance +fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + value +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; @@ -130,6 +136,7 @@ settrace() -- set the global debug tracing function "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, "ps1" => ctx.new_str(">>>>> ".to_string()), @@ -137,6 +144,7 @@ settrace() -- set the global debug tracing function "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), "modules" => modules.clone(), + "warnoptions" => ctx.new_list(vec![]), }); modules.set_item("sys", module.clone(), vm).unwrap(); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 5ae5b0b50..e360b8e5e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -234,6 +234,11 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } + pub fn new_syntax_error(&self, msg: &T) -> PyObjectRef { + let syntax_error = self.ctx.exceptions.syntax_error.clone(); + self.new_exception(syntax_error, msg.to_string()) + } + pub fn get_none(&self) -> PyObjectRef { self.ctx.none() } From a708ce2eb2e9179f957feb87b917110fa33a0539 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 12:15:55 +1200 Subject: [PATCH 260/658] Remove needless format! --- vm/src/import.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/import.rs b/vm/src/import.rs index 041fab650..b59ce6426 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -24,7 +24,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let file_path = find_source(vm, current_path, module) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), format!("{}", e)))?; + .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; let code_obj = compile::compile( vm, &source, From 762c281dd48c5c762e95dc25f491ccca5367c4f9 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 19:07:15 -0700 Subject: [PATCH 261/658] address feedback --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 3e0bcccb6..5a133615c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1038,7 +1038,7 @@ mod tests { ("Greek Ωppercases ...", "greek ωppercases ..."), ]; for (title, input) in tests { - assert_eq!(PyString::from(input).title(&vm), String::from(title)); + assert_eq!(PyString::from(input).title(&vm).as_str(), title); } } From 307689a15ee8ee6699ec7f9ebe3e2e4659f01247 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 17:47:51 +1200 Subject: [PATCH 262/658] comment out lines that reference mappingproxy as type doesn't yet have a __dict__ --- Lib/_collections_abc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3158a4f16..c9f7daa16 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -52,7 +52,7 @@ dict_keys = type({}.keys()) dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## -mappingproxy = type(type.__dict__) +# mappingproxy = type(type.__dict__) # generator = type((lambda: (yield))()) ## coroutine ## # async def _coro(): pass @@ -688,7 +688,7 @@ class Mapping(Collection): __reversed__ = None -Mapping.register(mappingproxy) +# Mapping.register(mappingproxy) class MappingView(Sized): From 2c2925a68435e837af8aa9c8b6dc01bdf0e5666e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 17:22:52 +0300 Subject: [PATCH 263/658] Support reraise --- tests/snippets/try_exceptions.py | 12 ++++++++++ vm/src/bytecode.rs | 3 ++- vm/src/compile.rs | 40 ++++++++++++++++++++++---------- vm/src/frame.rs | 12 +++++++--- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 821d4e8c8..2a6ed7b34 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -125,3 +125,15 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError +try: + try: + raise NameError + except: + raise +except NameError: + pass + +try: + raise +except RuntimeError: + pass diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 2725aa586..10c667890 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -135,6 +135,7 @@ pub enum Instruction { PopBlock, Raise { argc: usize, + in_exc: bool, }, BuildString { size: usize, @@ -363,7 +364,7 @@ impl Instruction { SetupWith { end } => w!(SetupWith, end), CleanupWith { end } => w!(CleanupWith, end), PopBlock => w!(PopBlock), - Raise { argc } => w!(Raise, argc), + Raise { argc, in_exc } => w!(Raise, argc, in_exc), BuildString { size } => w!(BuildString, size), BuildTuple { size, unpack } => w!(BuildTuple, size, unpack), BuildList { size, unpack } => w!(BuildList, size, unpack), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index fe40962fe..9e8dab905 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -24,6 +24,7 @@ struct Compiler { current_qualified_path: Option, in_loop: bool, in_function_def: bool, + in_exc_handler: bool, } /// Compile a given sourcecode into a bytecode object. @@ -85,6 +86,7 @@ impl Compiler { current_qualified_path: None, in_loop: false, in_function_def: false, + in_exc_handler: false, } } @@ -325,15 +327,24 @@ impl Compiler { match cause { Some(cause) => { self.compile_expression(cause)?; - self.emit(Instruction::Raise { argc: 2 }); + self.emit(Instruction::Raise { + argc: 2, + in_exc: self.in_exc_handler, + }); } None => { - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); } } } None => { - self.emit(Instruction::Raise { argc: 0 }); + self.emit(Instruction::Raise { + argc: 0, + in_exc: self.in_exc_handler, + }); } }, ast::Statement::Try { @@ -378,7 +389,10 @@ impl Compiler { }); } } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); self.set_label(end_label); } ast::Statement::Break => { @@ -558,6 +572,7 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: + self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now handler_label = self.new_label(); @@ -586,15 +601,10 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { + // Duplicate exception for context: + self.emit(Instruction::Duplicate); self.store_name(alias); - } else { - // Drop exception from top of stack: - self.emit(Instruction::Pop); } - } else { - // Catch all! - // Drop exception from top of stack: - self.emit(Instruction::Pop); } // Handler code: @@ -611,6 +621,9 @@ impl Compiler { target: handler_label, }); self.set_label(handler_label); + // Drop exception from top of stack: + self.emit(Instruction::Pop); + self.in_exc_handler = false; // If code flows here, we have an unhandled exception, // emit finally code and raise again! // Duplicate finally code here: @@ -619,7 +632,10 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: false, + }); // We successfully ran the try block: // else: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 321ee2046..464578d6b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -715,14 +715,20 @@ impl Frame { Ok(None) } - bytecode::Instruction::Raise { argc } => { + bytecode::Instruction::Raise { argc, in_exc } => { + if argc.clone() == 0 && in_exc.clone() == false { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } let cause = match argc { 2 => self.get_exception(vm, true)?, _ => vm.get_none(), }; let exception = match argc { - 1 | 2 => self.get_exception(vm, false)?, - 0 | 3 => panic!("Not implemented!"), + 0 | 1 | 2 => self.get_exception(vm, false)?, + 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; info!("Exception raised: {:?} with cause: {:?}", exception, cause); From 3ec9b01928ddbd212f54e74d4afdcbb8b98a856b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:22:15 +0300 Subject: [PATCH 264/658] Add exception __context__ --- tests/snippets/try_exceptions.py | 5 +++++ vm/src/compile.rs | 12 ++++++------ vm/src/frame.rs | 30 ++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 2a6ed7b34..e75e73039 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -137,3 +137,8 @@ try: raise except RuntimeError: pass + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__context__ == None diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 9e8dab905..ba1f50e7b 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -572,6 +572,7 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: + let was_in_exc_handler = self.in_exc_handler; self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now @@ -609,6 +610,8 @@ impl Compiler { // Handler code: self.compile_statements(&handler.body)?; + // Drop exception from top of stack: + self.emit(Instruction::Pop); self.emit(Instruction::Jump { target: finally_label, }); @@ -621,9 +624,6 @@ impl Compiler { target: handler_label, }); self.set_label(handler_label); - // Drop exception from top of stack: - self.emit(Instruction::Pop); - self.in_exc_handler = false; // If code flows here, we have an unhandled exception, // emit finally code and raise again! // Duplicate finally code here: @@ -633,8 +633,8 @@ impl Compiler { self.compile_statements(statements)?; } self.emit(Instruction::Raise { - argc: 1, - in_exc: false, + argc: 0, + in_exc: true, }); // We successfully ran the try block: @@ -649,7 +649,7 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - + self.in_exc_handler = was_in_exc_handler; // unimplemented!(); Ok(()) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 464578d6b..2bf8215c1 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -716,23 +716,37 @@ impl Frame { } bytecode::Instruction::Raise { argc, in_exc } => { - if argc.clone() == 0 && in_exc.clone() == false { - return Err(vm.new_exception( - vm.ctx.exceptions.runtime_error.clone(), - "No active exception to reraise".to_string(), - )); - } let cause = match argc { 2 => self.get_exception(vm, true)?, _ => vm.get_none(), }; let exception = match argc { - 0 | 1 | 2 => self.get_exception(vm, false)?, + 0 => match in_exc { + true => self.get_exception(vm, false)?, + false => { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } + }, + 1 | 2 => self.get_exception(vm, false)?, 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - info!("Exception raised: {:?} with cause: {:?}", exception, cause); + let context = match argc { + 0 => vm.get_none(), // We have already got the exception, + _ => match in_exc { + true => self.get_exception(vm, false)?, + false => vm.get_none(), + }, + }; + info!( + "Exception raised: {:?} with cause: {:?} and context: {:?}", + exception, cause, context + ); vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + vm.set_attr(&exception, vm.new_str("__context__".to_string()), context)?; Err(exception) } From 188758929df8b82e5484e87f800ecff5994ae692 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:42:23 +0300 Subject: [PATCH 265/658] Add more tests to exception context --- tests/snippets/try_exceptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index e75e73039..1af2d9762 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -106,6 +106,7 @@ try: raise NameError from ex except NameError as ex2: assert ex2.__cause__ == cause + assert ex2.__context__ == cause try: raise ZeroDivisionError from None @@ -138,7 +139,25 @@ try: except RuntimeError: pass +context = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__context__ == None + context = ex + raise NameError +except NameError as ex2: + assert ex2.__context__ == context + assert type(ex2.__context__) == ZeroDivisionError + try: raise ZeroDivisionError except ZeroDivisionError as ex: assert ex.__context__ == None + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + assert ex.__context__ == None From aec7f84960add779f0e81d5cb5e4169cc4aeab2b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:52:01 +0300 Subject: [PATCH 266/658] Print exception context --- vm/src/exceptions.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index abd670a14..7642908ee 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -20,12 +20,22 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + let mut had_casue = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { if !vm.get_none().is(&cause) { + had_casue = true; print_exception(vm, &cause); println!("\nThe above exception was the direct cause of the following exception:\n"); } } + if !had_casue { + if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { + if !vm.get_none().is(&context) { + print_exception(vm, &context); + println!("\nDuring handling of the above exception, another exception occurred:\n"); + } + } + } print_exception_inner(vm, exc) } From b6ce4073917bcf1d4e581896773aa0abce9a2a0c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 19:05:28 +0300 Subject: [PATCH 267/658] Remove else and finally from in_exc_handler --- tests/snippets/try_exceptions.py | 10 ++++++++++ vm/src/compile.rs | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 1af2d9762..d7785424a 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -161,3 +161,13 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError assert ex.__context__ == None + +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + pass + finally: + raise NameError +except NameError as ex2: + assert ex2.__context__ == None diff --git a/vm/src/compile.rs b/vm/src/compile.rs index ba1f50e7b..d28c10feb 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -637,6 +637,8 @@ impl Compiler { in_exc: true, }); + self.in_exc_handler = was_in_exc_handler; + // We successfully ran the try block: // else: self.set_label(else_label); @@ -649,7 +651,6 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.in_exc_handler = was_in_exc_handler; // unimplemented!(); Ok(()) } From 4068a36ce8a576b3b5c7b19716a8cb0ee0a43421 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 20:38:34 +0300 Subject: [PATCH 268/658] Use assertRaises --- tests/snippets/try_exceptions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index d7785424a..83e00c0fe 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -126,18 +126,14 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError -try: +with assertRaises(NameError): try: raise NameError except: raise -except NameError: - pass -try: +with assertRaises(RuntimeError): raise -except RuntimeError: - pass context = None try: From 2ffd1c948447df991e927e6d7a6d88d42d2527a7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 20:39:28 +0300 Subject: [PATCH 269/658] Fix typo --- vm/src/exceptions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 7642908ee..5f577d04f 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -20,15 +20,15 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { - let mut had_casue = false; + let mut had_cause = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { if !vm.get_none().is(&cause) { - had_casue = true; + had_cause = true; print_exception(vm, &cause); println!("\nThe above exception was the direct cause of the following exception:\n"); } } - if !had_casue { + if !had_cause { if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { if !vm.get_none().is(&context) { print_exception(vm, &context); From 79623f4e7a3e9f3e3fdcbbe65f2dda923250b799 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 14 Apr 2019 12:23:09 -0700 Subject: [PATCH 270/658] str: proper titlecase support --- Cargo.lock | 7 +++++++ vm/Cargo.toml | 1 + vm/src/obj/objstr.rs | 9 ++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 864e90ce5..100f6a359 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -827,6 +827,7 @@ dependencies = [ "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1102,6 +1103,11 @@ dependencies = [ "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "unicode-casing" +version = "0.1.0" +source = "git+https://github.com/OddCoincidence/unicode-casing#90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" + [[package]] name = "unicode-normalization" version = "0.1.8" @@ -1431,6 +1437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" "checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" "checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +"checksum unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing)" = "" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ef1a9cbc3..1ba1986b7 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -23,6 +23,7 @@ regex = "1" rustc_version_runtime = "0.1.*" statrs = "0.10.0" caseless = "0.2.1" +unicode-casing = { git = "https://github.com/OddCoincidence/unicode-casing" } unicode-segmentation = "1.2.1" lazy_static = "^1.0.1" lexical = "2.0.0" diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 390e92d27..b5e7a29cf 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -5,6 +5,7 @@ use std::str::FromStr; use std::string::ToString; use num_traits::ToPrimitive; +use unicode_casing::CharExt; use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; @@ -413,12 +414,12 @@ impl PyString { for c in self.value.chars() { if c.is_lowercase() { if !previous_is_cased { - title.extend(c.to_uppercase()); + title.extend(c.to_titlecase()); } else { title.push(c); } previous_is_cased = true; - } else if c.is_uppercase() { + } else if c.is_uppercase() || c.is_titlecase() { if previous_is_cased { title.extend(c.to_lowercase()); } else { @@ -652,7 +653,7 @@ impl PyString { let mut cased = false; let mut previous_is_cased = false; for c in self.value.chars() { - if c.is_uppercase() { + if c.is_uppercase() || c.is_titlecase() { if previous_is_cased { return false; } @@ -1050,6 +1051,7 @@ mod tests { ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), ("Getint", "getInt"), ("Greek Ωppercases ...", "greek ωppercases ..."), + ("Greek ῼitlecases ...", "greek ῳitlecases ..."), ]; for (title, input) in tests { assert_eq!(PyString::from(input).title(&vm).as_str(), title); @@ -1066,6 +1068,7 @@ mod tests { "A\nTitlecased Line", "A Titlecased, Line", "Greek Ωppercases ...", + "Greek ῼitlecases ...", ]; for s in pos { From daaeb7883146a0cd3611f3c661f951a4e64bbd58 Mon Sep 17 00:00:00 2001 From: Darren Kaste Date: Sun, 14 Apr 2019 17:08:35 -0400 Subject: [PATCH 271/658] Prefix helper macro calls with `$crate` --- vm/src/macros.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 27d164718..b84643b5f 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -9,7 +9,7 @@ macro_rules! replace_expr { #[macro_export] macro_rules! count_tts { - ($($tts:tt)*) => {0usize $(+ replace_expr!($tts 1usize))*}; + ($($tts:tt)*) => {0usize $(+ $crate::replace_expr!($tts 1usize))*}; } #[macro_export] @@ -45,14 +45,14 @@ macro_rules! arg_check { } }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*] ) => { - arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); + $crate::arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*], optional=[$( ($optional_arg_name:ident, $optional_arg_type:expr) ),*] ) => { let mut arg_count = 0; // use macro magic to compile-time count number of required and optional arguments - let minimum_arg_count = count_tts!($($arg_name)*); - let maximum_arg_count = minimum_arg_count + count_tts!($($optional_arg_name)*); + let minimum_arg_count = $crate::count_tts!($($arg_name)*); + let maximum_arg_count = minimum_arg_count + $crate::count_tts!($($optional_arg_name)*); // verify that the number of given arguments is right if $args.args.len() < minimum_arg_count || $args.args.len() > maximum_arg_count { @@ -72,7 +72,7 @@ macro_rules! arg_check { // check if the type matches. If not, return with error // assign the arg to a variable $( - type_check!($vm, $args, arg_count, $arg_name, $arg_type); + $crate::type_check!($vm, $args, arg_count, $arg_name, $arg_type); let $arg_name = &$args.args[arg_count]; #[allow(unused_assignments)] { @@ -85,7 +85,7 @@ macro_rules! arg_check { // assign the arg to a variable $( let $optional_arg_name = if arg_count < $args.args.len() { - type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); + $crate::type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); let ret = Some(&$args.args[arg_count]); #[allow(unused_assignments)] { @@ -226,7 +226,7 @@ macro_rules! match_class { ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { match $obj.downcast::<$class>() { Ok($binding) => $expr, - Err(_obj) => match_class!(_obj, $($rest)*), + Err(_obj) => $crate::match_class!(_obj, $($rest)*), } }; @@ -236,7 +236,7 @@ macro_rules! match_class { if $obj.payload_is::<$class>() { $expr } else { - match_class!($obj, $($rest)*) + $crate::match_class!($obj, $($rest)*) } }; } From a8e064b1c0f32077998a913d00b08a29ebeaf9dc Mon Sep 17 00:00:00 2001 From: Darren Kaste Date: Sun, 14 Apr 2019 17:19:44 -0400 Subject: [PATCH 272/658] Import `TypeProtocol` before use in `type_check!` --- vm/src/macros.rs | 2 ++ vm/src/obj/objbool.rs | 4 +--- vm/src/obj/objellipsis.rs | 2 +- vm/src/obj/objlist.rs | 1 - vm/src/obj/objslice.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/keyword.rs | 2 +- vm/src/stdlib/math.rs | 2 +- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/pystruct.rs | 2 +- vm/src/stdlib/random.rs | 2 +- vm/src/stdlib/socket.rs | 2 +- vm/src/stdlib/time_module.rs | 2 +- vm/src/stdlib/tokenize.rs | 2 +- vm/src/sysmodule.rs | 2 +- 15 files changed, 15 insertions(+), 16 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index b84643b5f..333673985 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -20,6 +20,8 @@ macro_rules! type_check { let arg = &$args.args[$arg_count]; if !$crate::obj::objtype::isinstance(arg, &expected_type) { + use $crate::pyobject::TypeProtocol; + let arg_typ = arg.class(); let expected_type_name = $vm.to_pystr(&expected_type)?; let actual_type = $vm.to_pystr(&arg_typ)?; diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index d4ef4b0fb..724c8bc5a 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -1,9 +1,7 @@ use num_traits::Zero; use crate::function::PyFuncArgs; -use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; use super::objint::PyInt; diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 6591d8173..cb188e1f1 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -1,5 +1,5 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 276a8ea89..a21bd9190 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -9,7 +9,6 @@ use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 31fcfa52f..60d911a92 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,7 +1,7 @@ use num_bigint::BigInt; use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index d864519f5..b8e72cc15 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,7 +20,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; fn compute_c_flag(mode: &str) -> u16 { diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index 3143af21a..062f41df7 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index dc7981ab6..2ee584a79 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; // Helper macro: diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f397b2d8a..fae9d6adb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -12,7 +12,7 @@ use crate::obj::objint; use crate::obj::objint::PyIntRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; #[cfg(unix)] diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 2ff61d630..4a454b0c9 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -15,7 +15,7 @@ use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; #[derive(Debug)] diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index c6cc44a36..cf3c0ea18 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,7 +4,7 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index ab5072f7c..e893e0dab 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,7 +10,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objsequence::get_elements; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; use crate::obj::objtype::PyClassRef; diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index 93beabf8a..6c818689e 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 88a26881c..000dd5e1a 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 97de7a425..a5bf2a6f4 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -4,7 +4,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* From cc76bf7a1f4c7ecf9d72313107538bf4b20a600f Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 14 Apr 2019 20:09:15 -0500 Subject: [PATCH 273/658] Add weakproxy class --- vm/src/exceptions.rs | 3 +++ vm/src/obj/mod.rs | 1 + vm/src/obj/objweakproxy.rs | 50 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objweakref.rs | 2 +- vm/src/pyobject.rs | 9 +++++++ vm/src/stdlib/weakref.rs | 3 ++- 6 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 vm/src/obj/objweakproxy.rs diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 5f577d04f..58a0f929d 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -118,6 +118,7 @@ pub struct ExceptionZoo { pub os_error: PyClassRef, pub overflow_error: PyClassRef, pub permission_error: PyClassRef, + pub reference_error: PyClassRef, pub runtime_error: PyClassRef, pub stop_iteration: PyClassRef, pub syntax_error: PyClassRef, @@ -152,6 +153,7 @@ impl ExceptionZoo { let name_error = create_type("NameError", &type_type, &exception_type); let os_error = create_type("OSError", &type_type, &exception_type); let runtime_error = create_type("RuntimeError", &type_type, &exception_type); + let reference_error = create_type("ReferenceError", &type_type, &exception_type); let stop_iteration = create_type("StopIteration", &type_type, &exception_type); let syntax_error = create_type("SyntaxError", &type_type, &exception_type); let type_error = create_type("TypeError", &type_type, &exception_type); @@ -208,6 +210,7 @@ impl ExceptionZoo { syntax_warning, resource_warning, runtime_warning, + reference_error, user_warning, } } diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index c824e726d..bfe89fd15 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -34,5 +34,6 @@ pub mod objstr; pub mod objsuper; pub mod objtuple; pub mod objtype; +pub mod objweakproxy; pub mod objweakref; pub mod objzip; diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs new file mode 100644 index 000000000..735670f4e --- /dev/null +++ b/vm/src/obj/objweakproxy.rs @@ -0,0 +1,50 @@ +use super::objweakref::PyWeak; +use crate::function::OptionalArg; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[derive(Debug)] +pub struct PyWeakProxy { + weak: PyWeak, +} + +impl PyValue for PyWeakProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.weakproxy_type() + } +} + +pub type PyWeakProxyRef = PyRef; + +impl PyWeakProxyRef { + // TODO callbacks + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyWeakProxy { + weak: PyWeak::downgrade(&referent), + } + .into_ref_with_type(vm, cls) + } + + fn getattr(self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.get_attribute(obj, attr_name), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } +} + +pub fn init(context: &PyContext) { + extend_class!(context, &context.weakproxy_type, { + "__new__" => context.new_rustfunc(PyWeakProxyRef::create), + "__getattr__" => context.new_rustfunc(PyWeakProxyRef::getattr), + }); +} diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 05d6147ce..b3b7e03c9 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -1,9 +1,9 @@ +use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; use crate::pyobject::PyValue; use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; -use crate::function::OptionalArg; use std::rc::{Rc, Weak}; #[derive(Debug)] diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index f369bab4a..fdac31b83 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -51,6 +51,7 @@ use crate::obj::objstr; use crate::obj::objsuper; use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; +use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; @@ -158,6 +159,7 @@ pub struct PyContext { pub module_type: PyClassRef, pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, + pub weakproxy_type: PyClassRef, pub object: PyClassRef, pub exceptions: exceptions::ExceptionZoo, } @@ -255,6 +257,7 @@ impl PyContext { let readonly_property_type = create_type("readonly_property", &type_type, &object_type); let super_type = create_type("super", &type_type, &object_type); let weakref_type = create_type("ref", &type_type, &object_type); + let weakproxy_type = create_type("weakproxy", &type_type, &object_type); let generator_type = create_type("generator", &type_type, &object_type); let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); @@ -361,6 +364,7 @@ impl PyContext { module_type, bound_method_type, weakref_type, + weakproxy_type, type_type, exceptions, }; @@ -396,6 +400,7 @@ impl PyContext { objcode::init(&context); objframe::init(&context); objweakref::init(&context); + objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); exceptions::init(&context); @@ -554,6 +559,10 @@ impl PyContext { self.weakref_type.clone() } + pub fn weakproxy_type(&self) -> PyClassRef { + self.weakproxy_type.clone() + } + pub fn type_type(&self) -> PyClassRef { self.type_type.clone() } diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index b5f7b756a..d4529599e 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -12,6 +12,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "_weakref", { - "ref" => ctx.weakref_type() + "ref" => ctx.weakref_type(), + "proxy" => ctx.weakproxy_type(), }) } From 1612c954a17e38ed3c4f1fdee40096a9eabf468f Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 14 Apr 2019 20:14:54 -0500 Subject: [PATCH 274/658] Add weakproxy test --- tests/snippets/weakrefs.py | 17 ++++++++++++++++- vm/src/builtins.rs | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/snippets/weakrefs.py b/tests/snippets/weakrefs.py index ffc61f427..059ddd000 100644 --- a/tests/snippets/weakrefs.py +++ b/tests/snippets/weakrefs.py @@ -1,4 +1,5 @@ -from _weakref import ref +from _weakref import ref, proxy +from testutils import assert_raises class X: @@ -11,3 +12,17 @@ b = ref(a) assert callable(b) assert b() is a + +class G: + def __init__(self, h): + self.h = h + + +g = G(5) +p = proxy(g) + +assert p.h == 5 + +del g + +assert_raises(ReferenceError, lambda: p.h) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 8f7721609..a96df53e9 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -751,6 +751,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "NameError" => ctx.exceptions.name_error.clone(), "OverflowError" => ctx.exceptions.overflow_error.clone(), "RuntimeError" => ctx.exceptions.runtime_error.clone(), + "ReferenceError" => ctx.exceptions.reference_error.clone(), "NotImplementedError" => ctx.exceptions.not_implemented_error.clone(), "TypeError" => ctx.exceptions.type_error.clone(), "ValueError" => ctx.exceptions.value_error.clone(), From c9b479c43b62ef65bf632bce3d0e78ae9d57395f Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 09:19:23 +0100 Subject: [PATCH 275/658] Guard for changes in dictionary size during iteration. --- tests/snippets/dict.py | 12 ++++++++++++ vm/src/dictdatatype.rs | 29 +++++++++++++++++++++++------ vm/src/obj/objdict.rs | 27 +++++++++++++++++++-------- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 4c35af2f4..05cdd9928 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -69,6 +69,18 @@ assert (1, 2) == next(items) data[3] = "changed" assert (3, "changed") == next(items) +# But we can't add or delete items during iteration. +d = {} +a = iter(d.items()) +d['a'] = 2 +b = iter(d.items()) +assert ('a', 2) == next(b) +with assertRaises(RuntimeError): + next(a) +del d['a'] +with assertRaises(RuntimeError): + next(b) + # View isn't itself an iterator. with assertRaises(TypeError): next(data.keys()) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6dc49125b..1f4b4eecd 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -33,6 +33,12 @@ struct DictEntry { value: T, } +#[derive(Debug)] +pub struct DictSize { + size: usize, + entries_size: usize, +} + impl Dict { pub fn new() -> Self { Dict { @@ -120,17 +126,28 @@ impl Dict { self.len() == 0 } - pub fn next_entry(&self, mut position: usize) -> Option<(usize, &PyObjectRef, &T)> { - while position < self.entries.len() { - if let Some(DictEntry { key, value, .. }) = &self.entries[position] { - return Some((position + 1, key, value)); - } else { - position += 1; + pub fn size(&self) -> DictSize { + DictSize { + size: self.size, + entries_size: self.entries.len(), + } + } + + pub fn next_entry(&self, position: &mut usize) -> Option<(&PyObjectRef, &T)> { + while *position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { + *position += 1; + return Some((key, value)); } + *position += 1; } None } + pub fn has_changed_size(&self, position: &DictSize) -> bool { + position.size != self.size || self.entries.len() != position.entries_size + } + /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { let hash_value = calc_hash(vm, key)?; diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index fd77107ee..7b11dd6c1 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -260,6 +260,10 @@ impl PyDictRef { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().contains(vm, &key).unwrap() } + + pub fn size(&self) -> dictdatatype::DictSize { + self.entries.borrow().size() + } } impl ItemProtocol for PyDictRef { @@ -315,11 +319,8 @@ impl Iterator for DictIter { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { - match self.dict.entries.borrow().next_entry(self.position) { - Some((new_position, key, value)) => { - self.position = new_position; - Some((key.clone(), value.clone())) - } + match self.dict.entries.borrow().next_entry(&mut self.position) { + Some((key, value)) => Some((key.clone(), value.clone())), None => None, } } @@ -360,6 +361,7 @@ macro_rules! dict_iterator { #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, + pub size: dictdatatype::DictSize, pub position: Cell, } @@ -368,15 +370,24 @@ macro_rules! dict_iterator { fn new(dict: PyDictRef) -> Self { $iter_name { position: Cell::new(0), + size: dict.size(), dict, } } #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, value)) => { - self.position.set(new_position); + let mut position = self.position.get(); + let dict = self.dict.entries.borrow(); + if dict.has_changed_size(&self.size) { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "dictionary changed size during iteration".to_string(), + )); + } + match dict.next_entry(&mut position) { + Some((key, value)) => { + self.position.set(position); Ok($result_fn(vm, key, value)) } None => Err(objiter::new_stop_iteration(vm)), From dfbaf3cdf01936c5769467e250ced6365863935e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 15 Apr 2019 13:23:15 +0100 Subject: [PATCH 276/658] Remove pub from PyDict.entries (easy TODO) --- vm/src/obj/objdict.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7b11dd6c1..25df2ed14 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -19,8 +19,7 @@ pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { - // TODO: should be private - pub entries: RefCell, + entries: RefCell, } pub type PyDictRef = PyRef; From 08babef619763a3a69347c890cd448f3fe6cf2ea Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 15 Apr 2019 20:24:16 -0500 Subject: [PATCH 277/658] Remove the __inside_vm hack for procedural macros --- derive/src/from_args.rs | 27 +++++++++++---------------- derive/src/lib.rs | 37 +------------------------------------ derive/src/pyclass.rs | 17 ++++++----------- vm/src/builtins.rs | 1 - vm/src/lib.rs | 2 ++ vm/src/obj/objbytes.rs | 4 ++-- vm/src/obj/objdict.rs | 8 ++++---- vm/src/obj/objgenerator.rs | 4 ++-- vm/src/obj/objint.rs | 5 ++--- vm/src/obj/objstr.rs | 4 ++-- 10 files changed, 32 insertions(+), 77 deletions(-) diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs index 040ebf36c..d017a32e5 100644 --- a/derive/src/from_args.rs +++ b/derive/src/from_args.rs @@ -1,4 +1,3 @@ -use super::rustpython_path_derive; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; @@ -109,7 +108,7 @@ impl ArgAttribute { } } -fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { +fn generate_field(field: &Field) -> TokenStream2 { let mut pyarg_attrs = field .attrs .iter() @@ -132,7 +131,7 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { let name = &field.ident; let middle = quote! { - .map(|x| #rp_path::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? }; let ending = if let Some(default) = attr.default { quote! { @@ -140,16 +139,16 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { } } else if attr.optional { quote! { - .map(#rp_path::function::OptionalArg::Present) - .unwrap_or(#rp_path::function::OptionalArg::Missing) + .map(::rustpython_vm::function::OptionalArg::Present) + .unwrap_or(::rustpython_vm::function::OptionalArg::Missing) } } else { let err = match attr.kind { ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - #rp_path::function::ArgumentError::TooFewArgs + ::rustpython_vm::function::ArgumentError::TooFewArgs }, ParameterKind::KeywordOnly => quote! { - #rp_path::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) }, }; quote! { @@ -177,14 +176,10 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { } pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { - let rp_path = rustpython_path_derive(&input); let fields = match input.data { Data::Struct(ref data) => { match data.fields { - Fields::Named(ref fields) => fields - .named - .iter() - .map(|field| generate_field(field, &rp_path)), + Fields::Named(ref fields) => fields.named.iter().map(generate_field), Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message } } @@ -193,11 +188,11 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { let name = &input.ident; quote! { - impl #rp_path::function::FromArgs for #name { + impl ::rustpython_vm::function::FromArgs for #name { fn from_args( - vm: &#rp_path::VirtualMachine, - args: &mut #rp_path::function::PyFuncArgs - ) -> Result { + vm: &::rustpython_vm::VirtualMachine, + args: &mut ::rustpython_vm::function::PyFuncArgs + ) -> Result { Ok(#name { #(#fields)* }) } } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index adde078f0..26dce935a 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,47 +1,12 @@ extern crate proc_macro; use proc_macro::TokenStream; -use quote::quote; use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; mod from_args; mod pyclass; -fn rustpython_path(inside_vm: bool) -> syn::Path { - let path = if inside_vm { - quote!(crate) - } else { - quote!(::rustpython_vm) - }; - syn::parse2(path).unwrap() -} - -/// Does the item have the #[__inside_vm] attribute on it, signifying that the derive target is -/// being derived from inside the `rustpython_vm` crate. -fn rustpython_path_derive(input: &DeriveInput) -> syn::Path { - rustpython_path( - input - .attrs - .iter() - .any(|attr| attr.path.is_ident("__inside_vm")), - ) -} - -fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { - rustpython_path(attr.iter().any(|meta| { - if let syn::NestedMeta::Meta(meta) = meta { - if let syn::Meta::Word(ident) = meta { - ident == "__inside_vm" - } else { - false - } - } else { - false - } - })) -} - -#[proc_macro_derive(FromArgs, attributes(__inside_vm, pyarg))] +#[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index e072db32d..36ce9df1a 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,4 +1,3 @@ -use super::rustpython_path_attr; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use std::collections::HashMap; @@ -138,15 +137,13 @@ impl ClassItem { } } -pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); }; - let rp_path = rustpython_path_attr(&attr); - let items = imp .items .iter_mut() @@ -199,7 +196,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { class.set_str_attr( #name, - #rp_path::obj::objproperty::PropertyBuilder::new(ctx) + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) .add_getter(Self::#getter) #add_setter .create(), @@ -209,10 +206,10 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #imp - impl #rp_path::pyobject::PyClassImpl for #ty { + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { fn impl_extend_class( - ctx: &#rp_path::pyobject::PyContext, - class: &#rp_path::obj::objtype::PyClassRef, + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, ) { #(#methods)* #(#properties)* @@ -228,8 +225,6 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; - let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { @@ -273,7 +268,7 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #item - impl #rp_path::pyobject::PyClassDef for #ident { + impl ::rustpython_vm::pyobject::PyClassDef for #ident { const NAME: &'static str = #class_name; const DOC: Option<&'static str> = #doc; } diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a96df53e9..f2ca1e4c1 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -530,7 +530,6 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } #[derive(Debug, FromArgs)] -#[__inside_vm] pub struct PrintOptions { #[pyarg(keyword_only, default = "None")] sep: Option, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 578f1d99a..8b14454fe 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -32,6 +32,8 @@ extern crate rustpython_parser; #[macro_use] extern crate rustpython_derive; +extern crate self as rustpython_vm; + pub use rustpython_derive::*; //extern crate eval; use eval::eval::*; diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index fd9bfd5bf..0556de053 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -21,7 +21,7 @@ use super::objtype::PyClassRef; /// - a text string encoded using the specified encoding\n \ /// - any object implementing the buffer API.\n \ /// - an integer"; -#[pyclass(name = "bytes", __inside_vm)] +#[pyclass(name = "bytes")] #[derive(Clone, Debug)] pub struct PyBytes { inner: PyByteInner, @@ -71,7 +71,7 @@ pub fn init(context: &PyContext) { }); } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 25df2ed14..e5419e932 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -327,13 +327,13 @@ impl Iterator for DictIter { macro_rules! dict_iterator { ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { - #[pyclass(name = $class_name, __inside_vm)] + #[pyclass(name = $class_name)] #[derive(Debug)] struct $name { pub dict: PyDictRef, } - #[pyimpl(__inside_vm)] + #[pyimpl] impl $name { fn new(dict: PyDictRef) -> Self { $name { dict: dict } @@ -356,7 +356,7 @@ macro_rules! dict_iterator { } } - #[pyclass(name = $iter_class_name, __inside_vm)] + #[pyclass(name = $iter_class_name)] #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, @@ -364,7 +364,7 @@ macro_rules! dict_iterator { pub position: Cell, } - #[pyimpl(__inside_vm)] + #[pyimpl] impl $iter_name { fn new(dict: PyDictRef) -> Self { $iter_name { diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 2541263f1..2c51778fd 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -9,7 +9,7 @@ use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; -#[pyclass(name = "generator", __inside_vm)] +#[pyclass(name = "generator")] #[derive(Debug)] pub struct PyGenerator { frame: FrameRef, @@ -21,7 +21,7 @@ impl PyValue for PyGenerator { } } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { frame }.into_ref(vm) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 52f321adf..20f5de4ca 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -32,7 +32,7 @@ use crate::obj::objtype::PyClassRef; /// Base 0 means to interpret the base from the string as an integer literal. /// >>> int('0b100', base=0) /// 4 -#[pyclass(__inside_vm)] +#[pyclass] #[derive(Debug)] pub struct PyInt { value: BigInt, @@ -111,7 +111,7 @@ impl_try_from_object_int!( (u64, to_u64), ); -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyInt { #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { @@ -476,7 +476,6 @@ impl PyInt { } #[derive(FromArgs)] -#[__inside_vm] struct IntOptions { #[pyarg(positional_only, optional = true)] val_options: OptionalArg, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 390e92d27..1638a712c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -30,7 +30,7 @@ use super::objtype::{self, PyClassRef}; /// or repr(object). /// encoding defaults to sys.getdefaultencoding(). /// errors defaults to 'strict'." -#[pyclass(name = "str", __inside_vm)] +#[pyclass(name = "str")] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public @@ -74,7 +74,7 @@ impl TryIntoRef for &str { } } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyString { // TODO: should with following format // class str(object='') From dc63fc8ce7e02ad1728184e446d278a69016e1f4 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 15 Apr 2019 22:14:12 -0500 Subject: [PATCH 278/658] Switch to span-based errors for pyclass and pyimpl --- derive/src/error.rs | 158 ++++++++++++++++++++++++++++++++++++ derive/src/lib.rs | 22 +++-- derive/src/pyclass.rs | 184 +++++++++++++++++++++++++++--------------- 3 files changed, 292 insertions(+), 72 deletions(-) create mode 100644 derive/src/error.rs diff --git a/derive/src/error.rs b/derive/src/error.rs new file mode 100644 index 000000000..8754b4a51 --- /dev/null +++ b/derive/src/error.rs @@ -0,0 +1,158 @@ +// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs +// +// Copyright (c) 2014 Alex Crichton +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#![allow(dead_code)] + +use proc_macro2::*; +use quote::{ToTokens, TokenStreamExt}; +use syn::parse::Error; + +macro_rules! err_span { + ($span:expr, $($msg:tt)*) => ( + $crate::Diagnostic::spanned_error(&$span, format!($($msg)*)) + ) +} + +macro_rules! bail_span { + ($($t:tt)*) => ( + return Err(err_span!($($t)*).into()) + ) +} + +macro_rules! push_err_span { + ($diagnostics:expr, $($t:tt)*) => { + $diagnostics.push(err_span!($($t)*)) + }; +} + +#[derive(Debug)] +pub struct Diagnostic { + inner: Repr, +} + +#[derive(Debug)] +enum Repr { + Single { + text: String, + span: Option<(Span, Span)>, + }, + SynError(Error), + Multi { + diagnostics: Vec, + }, +} + +impl Diagnostic { + pub fn error>(text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: None, + }, + } + } + + pub fn span_error>(span: Span, text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: Some((span, span)), + }, + } + } + + pub fn spanned_error>(node: &ToTokens, text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: extract_spans(node), + }, + } + } + + pub fn from_vec(diagnostics: Vec) -> Result<(), Diagnostic> { + if diagnostics.len() == 0 { + Ok(()) + } else { + Err(Diagnostic { + inner: Repr::Multi { diagnostics }, + }) + } + } + + #[allow(unconditional_recursion)] + pub fn panic(&self) -> ! { + match &self.inner { + Repr::Single { text, .. } => panic!("{}", text), + Repr::SynError(error) => panic!("{}", error), + Repr::Multi { diagnostics } => diagnostics[0].panic(), + } + } +} + +impl From for Diagnostic { + fn from(err: Error) -> Diagnostic { + Diagnostic { + inner: Repr::SynError(err), + } + } +} + +fn extract_spans(node: &ToTokens) -> Option<(Span, Span)> { + let mut t = TokenStream::new(); + node.to_tokens(&mut t); + let mut tokens = t.into_iter(); + let start = tokens.next().map(|t| t.span()); + let end = tokens.last().map(|t| t.span()); + start.map(|start| (start, end.unwrap_or(start))) +} + +impl ToTokens for Diagnostic { + fn to_tokens(&self, dst: &mut TokenStream) { + match &self.inner { + Repr::Single { text, span } => { + let cs2 = (Span::call_site(), Span::call_site()); + let (start, end) = span.unwrap_or(cs2); + dst.append(Ident::new("compile_error", start)); + dst.append(Punct::new('!', Spacing::Alone)); + let mut message = TokenStream::new(); + message.append(Literal::string(text)); + let mut group = Group::new(Delimiter::Brace, message); + group.set_span(end); + dst.append(group); + } + Repr::Multi { diagnostics } => { + for diagnostic in diagnostics { + diagnostic.to_tokens(dst); + } + } + Repr::SynError(err) => { + err.to_compile_error().to_tokens(dst); + } + } + } +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 26dce935a..72ab44d4c 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,11 +1,23 @@ extern crate proc_macro; -use proc_macro::TokenStream; -use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; - +#[macro_use] +mod error; mod from_args; mod pyclass; +use error::Diagnostic; +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use quote::ToTokens; +use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; + +fn result_to_tokens(result: Result) -> TokenStream { + match result { + Ok(tokens) => tokens.into(), + Err(diagnostic) => diagnostic.into_token_stream().into(), + } +} + #[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); @@ -17,12 +29,12 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - pyclass::impl_pyclass(attr, item).into() + result_to_tokens(pyclass::impl_pyclass(attr, item)) } #[proc_macro_attribute] pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - pyclass::impl_pyimpl(attr, item).into() + result_to_tokens(pyclass::impl_pyimpl(attr, item)) } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 36ce9df1a..a2cf57b4a 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,3 +1,4 @@ +use super::Diagnostic; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use std::collections::HashMap; @@ -15,19 +16,21 @@ enum ClassItem { }, } -fn meta_to_vec(meta: Meta) -> Option> { +fn meta_to_vec(meta: Meta) -> Result, Meta> { match meta { - Meta::Word(_) => Some(Vec::new()), - Meta::List(list) => Some(list.nested.into_iter().collect()), - Meta::NameValue(_) => None, + Meta::Word(_) => Ok(Vec::new()), + Meta::List(list) => Ok(list.nested.into_iter().collect()), + Meta::NameValue(_) => Err(meta), } } impl ClassItem { - fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + fn extract_from_syn( + attrs: &mut Vec, + sig: &MethodSig, + ) -> Result, Diagnostic> { let mut item = None; let mut attr_idx = None; - // TODO: better error handling throughout this for (i, meta) in attrs .iter() .filter_map(|attr| attr.parse_meta().ok()) @@ -36,12 +39,18 @@ impl ClassItem { let name = meta.name(); if name == "pymethod" { if item.is_some() { - panic!("You can only have one #[py*] attribute on an impl item") + bail_span!( + sig.ident, + "You can only have one #[py*] attribute on an impl item" + ) } - let nesteds = meta_to_vec(meta).expect( - "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ - #[pyproperty(name = \"...\")]", - ); + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ) + })?; let mut py_name = None; for meta in nesteds { let meta = match meta { @@ -54,7 +63,10 @@ impl ClassItem { if let Lit::Str(s) = &name_value.lit { py_name = Some(s.value()); } else { - panic!("#[pymethod(name = ...)] must be a string"); + bail_span!( + &sig.ident, + "#[pymethod(name = ...)] must be a string" + ); } } } @@ -68,12 +80,18 @@ impl ClassItem { attr_idx = Some(i); } else if name == "pyproperty" { if item.is_some() { - panic!("You can only have one #[py*] attribute on an impl item") + bail_span!( + sig.ident, + "You can only have one #[py*] attribute on an impl item" + ) } - let nesteds = meta_to_vec(meta).expect( - "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ - #[pyproperty(name = \"...\")]", - ); + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]" + ) + })?; let mut setter = false; let mut py_name = None; for meta in nesteds { @@ -87,7 +105,10 @@ impl ClassItem { if let Lit::Str(s) = &name_value.lit { py_name = Some(s.value()); } else { - panic!("#[pyproperty(name = ...)] must be a string"); + bail_span!( + &sig.ident, + "#[pyproperty(name = ...)] must be a string" + ); } } } @@ -99,29 +120,34 @@ impl ClassItem { _ => {} } } - let py_name = py_name.unwrap_or_else(|| { - let item_ident = sig.ident.to_string(); - if setter { - if item_ident.starts_with("set_") { - let name = &item_ident["set_".len()..]; - if name.is_empty() { - panic!( - "A #[pyproperty(setter)] fn with a set_* name have something \ - after \"set_\"" - ) + let py_name = match py_name { + Some(py_name) => py_name, + None => { + let item_ident = sig.ident.to_string(); + if setter { + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; + if name.is_empty() { + bail_span!( + &sig.ident, + "A #[pyproperty(setter)] fn with a set_* name must \ + have something after \"set_\"" + ) + } else { + name.to_string() + } } else { - name.to_string() + bail_span!( + &sig.ident, + "A #[pyproperty(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of \"set_*\"" + ) } } else { - panic!( - "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ - fn name along the lines of \"set_*\"" - ) + item_ident } - } else { - item_ident } - }); + }; item = Some(ClassItem::Property { py_name, item_ident: sig.ident.clone(), @@ -133,23 +159,27 @@ impl ClassItem { if let Some(attr_idx) = attr_idx { attrs.remove(attr_idx); } - item + Ok(item) } } -pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result { let mut imp = if let Item::Impl(imp) = item { imp } else { - return quote!(#item); + return Ok(quote!(#item)); }; + let mut diagnostics: Vec = Vec::new(); + let items = imp .items .iter_mut() .filter_map(|item| { if let ImplItem::Method(meth) = item { ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig) + .map_err(|err| diagnostics.push(err)) + .unwrap_or_default() } else { None } @@ -160,14 +190,18 @@ pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { for item in items.iter() { match item { ClassItem::Property { - item_ident, - py_name, + ref item_ident, + ref py_name, setter, } => { let entry = properties.entry(py_name).or_default(); let func = if *setter { &mut entry.1 } else { &mut entry.0 }; if func.is_some() { - panic!("Multiple property accessors with name {:?}", py_name) + bail_span!( + item_ident, + "Multiple property accessors with name {:?}", + py_name + ) } *func = Some(item_ident); } @@ -187,24 +221,37 @@ pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { None } }); - let properties = properties.iter().map(|(name, prop)| { - let getter = match prop.0 { - Some(getter) => getter, - None => panic!("Property {:?} is missing a getter", name), - }; - let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); - quote! { - class.set_str_attr( - #name, - ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) - .add_getter(Self::#getter) - #add_setter - .create(), - ); - } - }); + let properties = properties + .iter() + .map(|(name, prop)| { + let getter = match prop.0 { + Some(getter) => getter, + None => { + push_err_span!( + diagnostics, + prop.1.unwrap(), + "Property {:?} is missing a getter", + name + ); + return TokenStream2::new(); + } + }; + let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); + quote! { + class.set_str_attr( + #name, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(Self::#getter) + #add_setter + .create(), + ); + } + }) + .collect::>(); - quote! { + Diagnostic::from_vec(diagnostics)?; + + let ret = quote! { #imp impl ::rustpython_vm::pyobject::PyClassImpl for #ty { fn impl_extend_class( @@ -215,14 +262,18 @@ pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { #(#properties)* } } - } + }; + Ok(ret) } -pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result { let (item, ident, attrs) = match item { Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), - _ => panic!("#[pyclass] can only be on a struct or enum declaration"), + other => bail_span!( + other, + "#[pyclass] can only be on a struct or enum declaration" + ), }; let mut class_name = None; @@ -233,7 +284,7 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { if let Lit::Str(s) = name_value.lit { class_name = Some(s.value()); } else { - panic!("#[pyclass(name = ...)] must be a string"); + bail_span!(name_value.lit, "#[pyclass(name = ...)] must be a string"); } } } @@ -252,8 +303,6 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { Some(ref mut doc) => doc.push(val), None => doc = Some(vec![val]), } - } else { - panic!("expected #[doc = ...] to be a string") } } } @@ -266,11 +315,12 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { None => quote!(None), }; - quote! { + let ret = quote! { #item impl ::rustpython_vm::pyobject::PyClassDef for #ident { const NAME: &'static str = #class_name; const DOC: Option<&'static str> = #doc; } - } + }; + Ok(ret) } From 224863583b5cbba1e17ea9223829fa73a503ecb7 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 16 Apr 2019 19:13:39 +1200 Subject: [PATCH 279/658] Use new_syntax error in main.rs --- src/main.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 750faf98a..2a673736e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,11 +65,8 @@ fn main() { } fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult { - let code_obj = - compile::compile(vm, source, &compile::Mode::Exec, source_path).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })?; + let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path) + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj.borrow()); let vars = vm.ctx.new_scope(); // Keep track of local variables vm.run_code_obj(code_obj, vars) @@ -120,8 +117,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com // Don't inject syntax errors for line continuation Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err), Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - let exc = vm.new_exception(syntax_error, format!("{}", err)); + let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); Err(err) } From 6abf1511e99e0eeb543445f0b7d88eb57ea8a7f5 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Tue, 16 Apr 2019 17:19:57 +0200 Subject: [PATCH 280/658] Add nonlocal support. --- tests/snippets/global_nonlocal.py | 10 ++++++++++ vm/src/bytecode.rs | 1 + vm/src/compile.rs | 2 +- vm/src/frame.rs | 17 ++++++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py index 48d124210..fd5a61e1c 100644 --- a/tests/snippets/global_nonlocal.py +++ b/tests/snippets/global_nonlocal.py @@ -11,3 +11,13 @@ assert a == 2 b() assert a == 4 +def x(): + def y(): + nonlocal b + b = 3 + b = 2 + y() + return b + +res = x() +assert res == 3, str(res) diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 10c667890..73d69dc18 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -41,6 +41,7 @@ pub type Label = usize; #[derive(Debug, Clone, PartialEq)] pub enum NameScope { Local, + NonLocal, Global, } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index d28c10feb..8635e21ac 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -179,12 +179,12 @@ impl Compiler { let role = self.lookup_name(name); match role { SymbolRole::Global => bytecode::NameScope::Global, + SymbolRole::Nonlocal => bytecode::NameScope::NonLocal, _ => bytecode::NameScope::Local, } } fn load_name(&mut self, name: &str) { - // TODO: if global, do something else! let scope = self.scope_for_name(name); self.emit(Instruction::LoadName { name: name.to_string(), diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2bf8215c1..4425a8a7d 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -129,6 +129,7 @@ pub trait NameProtocol { fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn delete_name(&self, vm: &VirtualMachine, name: &str); fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); } @@ -157,6 +158,16 @@ impl NameProtocol for Scope { None } + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.locals + .iter() + .skip(1) + .next() + .unwrap() + .set_item(name, value, vm) + .unwrap(); + } + fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { self.get_locals().set_item(key, value, vm).unwrap(); } @@ -1037,8 +1048,11 @@ impl Frame { bytecode::NameScope::Global => { self.scope.store_global(vm, name, obj); } + bytecode::NameScope::NonLocal => { + self.scope.store_cell(vm, name, obj); + } bytecode::NameScope::Local => { - self.scope.store_name(&vm, name, obj); + self.scope.store_name(vm, name, obj); } } Ok(None) @@ -1057,6 +1071,7 @@ impl Frame { ) -> FrameResult { let optional_value = match name_scope { bytecode::NameScope::Global => self.scope.load_global(vm, name), + bytecode::NameScope::NonLocal => self.scope.load_cell(vm, name), bytecode::NameScope::Local => self.scope.load_name(&vm, name), }; From 96d6c518e902dec2146baf189edc626316769b20 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Tue, 16 Apr 2019 19:40:01 -0500 Subject: [PATCH 281/658] Fix os.unsetenv test for windows --- tests/snippets/stdlib_os.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 15fa176d8..4ee13b274 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -51,6 +51,11 @@ os.environ[ENV_KEY] = ENV_VALUE assert ENV_KEY in os.environ assert os.getenv(ENV_KEY) == ENV_VALUE del os.environ[ENV_KEY] -os.unsetenv(ENV_KEY) assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY) == None + +if os.name == "posix": + os.environ[ENV_KEY] = ENV_VALUE + os.unsetenv(ENV_KEY) + assert ENV_KEY not in os.environ + assert os.getenv(ENV_KEY) == None From 11bf84e8c17ea1a61da3bb68babec8a8842eb914 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Tue, 16 Apr 2019 21:59:34 -0500 Subject: [PATCH 282/658] Fix test again --- tests/snippets/stdlib_os.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 4ee13b274..7b9981bdf 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -55,7 +55,6 @@ assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY) == None if os.name == "posix": - os.environ[ENV_KEY] = ENV_VALUE + os.putenv(ENV_KEY, ENV_VALUE) os.unsetenv(ENV_KEY) - assert ENV_KEY not in os.environ assert os.getenv(ENV_KEY) == None From 9b71424d4efe2b5de49a9f087fac7efb2a70ddce Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 17 Apr 2019 20:02:08 +1200 Subject: [PATCH 283/658] Use slice.xxx_index() methods in setslice and delslice --- vm/src/obj/objlist.rs | 14 +++++++------- vm/src/obj/objslice.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index a21bd9190..bfc763d8c 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -210,12 +210,12 @@ impl PyListRef { } fn setslice(self, slice: PySliceRef, sec: PyIterable, vm: &VirtualMachine) -> PyResult { - let step = slice.step.clone().unwrap_or_else(BigInt::one); + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); if step.is_zero() { Err(vm.new_value_error("slice step cannot be zero".to_string())) } else if step.is_positive() { - let range = self.get_slice_range(&slice.start, &slice.stop); + let range = self.get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); if range.start < range.end { match step.to_i32() { Some(1) => self._set_slice(range, sec, vm), @@ -237,14 +237,14 @@ impl PyListRef { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = &slice.start.as_ref().map(|x| { + let start = &slice.start_index(vm)?.as_ref().map(|x| { if *x == (-1).to_bigint().unwrap() { self.get_len() + BigInt::one() //.to_bigint().unwrap() } else { x + 1 } }); - let stop = &slice.stop.as_ref().map(|x| { + let stop = &slice.stop_index(vm)?.as_ref().map(|x| { if *x == (-1).to_bigint().unwrap() { self.get_len().to_bigint().unwrap() } else { @@ -552,9 +552,9 @@ impl PyListRef { } fn delslice(self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { - let start = &slice.start; - let stop = &slice.stop; - let step = slice.step.clone().unwrap_or_else(BigInt::one); + let start = slice.start_index(vm)?; + let stop = slice.stop_index(vm)?; + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); if step.is_zero() { Err(vm.new_value_error("slice step cannot be zero".to_string())) diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index cebfee330..82e4cef8c 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,5 +1,5 @@ use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; use crate::obj::objint::PyInt; From 06563a36f677b07aa7328cc98a4acb76c3b3d32d Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 12:33:09 +0200 Subject: [PATCH 284/658] Support mutual importing modules. --- tests/snippets/import.py | 1 + tests/snippets/import_mutual1.py | 4 ++++ tests/snippets/import_mutual2.py | 3 +++ vm/src/import.rs | 26 ++++++++++++++++++-------- 4 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 tests/snippets/import_mutual1.py create mode 100644 tests/snippets/import_mutual2.py diff --git a/tests/snippets/import.py b/tests/snippets/import.py index a481e3f1a..4f1164043 100644 --- a/tests/snippets/import.py +++ b/tests/snippets/import.py @@ -3,6 +3,7 @@ from import_target import func, other_func from import_target import func as aliased_func, other_func as aliased_other_func from import_star import * +import import_mutual1 assert import_target.X == import_target.func() assert import_target.X == func() diff --git a/tests/snippets/import_mutual1.py b/tests/snippets/import_mutual1.py new file mode 100644 index 000000000..0dca4a34e --- /dev/null +++ b/tests/snippets/import_mutual1.py @@ -0,0 +1,4 @@ + +# Mutual recursive import: +import import_mutual2 + diff --git a/tests/snippets/import_mutual2.py b/tests/snippets/import_mutual2.py new file mode 100644 index 000000000..388ce2521 --- /dev/null +++ b/tests/snippets/import_mutual2.py @@ -0,0 +1,3 @@ + +# Mutual recursive import: +import import_mutual1 diff --git a/vm/src/import.rs b/vm/src/import.rs index b59ce6426..a5925a8ca 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -11,9 +11,13 @@ use crate::pyobject::{ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; -fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &str) -> PyResult { +fn import_uncached_module( + vm: &VirtualMachine, + current_path: PathBuf, + module_name: &str, +) -> PyResult { // Check for Rust-native modules - if let Some(module) = vm.stdlib_inits.borrow().get(module) { + if let Some(module) = vm.stdlib_inits.borrow().get(module_name) { return Ok(module(vm).clone()); } @@ -21,7 +25,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let import_error = vm.context().exceptions.import_error.clone(); // Time to search for module in any place: - let file_path = find_source(vm, current_path, module) + let file_path = find_source(vm, current_path, module_name) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; @@ -35,19 +39,25 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item("__name__", vm.new_str(module.to_string()), vm)?; - vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; - Ok(vm.ctx.new_module(module, attrs)) + attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; + let module = vm.ctx.new_module(module_name, attrs.clone()); + + // Store module in cache to prevent infinite loop with mutual importing libs: + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + sys_modules.set_item(module_name, module.clone(), vm)?; + + // Execute main code in module: + vm.run_code_obj(code_obj, Scope::new(None, attrs))?; + Ok(module) } pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { // First, see if we already loaded the module: - let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { return Ok(module); } let module = import_uncached_module(vm, current_path, module_name)?; - sys_modules.set_item(module_name, module.clone(), vm)?; Ok(module) } From 6d1b807c77a5c1d1e5e7afc78729cde9a6cfdef6 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 12:59:56 +0200 Subject: [PATCH 285/658] Improve error message of unwrap operation on nonlocal scope. --- vm/src/frame.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 4425a8a7d..9d3e28e1d 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -163,7 +163,7 @@ impl NameProtocol for Scope { .iter() .skip(1) .next() - .unwrap() + .expect("no outer scope for non-local") .set_item(name, value, vm) .unwrap(); } From 2a8b5866226115a3092c662768f0eba7564f7d19 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 14:03:18 +0200 Subject: [PATCH 286/658] Fix caching of rust modules. --- vm/src/import.rs | 81 ++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/vm/src/import.rs b/vm/src/import.rs index a5925a8ca..45c7cd86a 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -11,54 +11,47 @@ use crate::pyobject::{ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; -fn import_uncached_module( - vm: &VirtualMachine, - current_path: PathBuf, - module_name: &str, -) -> PyResult { - // Check for Rust-native modules - if let Some(module) = vm.stdlib_inits.borrow().get(module_name) { - return Ok(module(vm).clone()); - } - - let notfound_error = vm.context().exceptions.module_not_found_error.clone(); - let import_error = vm.context().exceptions.import_error.clone(); - - // Time to search for module in any place: - let file_path = find_source(vm, current_path, module_name) - .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; - let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; - let code_obj = compile::compile( - vm, - &source, - &compile::Mode::Exec, - file_path.to_str().unwrap().to_string(), - ) - .map_err(|err| vm.new_syntax_error(&err))?; - // trace!("Code object: {:?}", code_obj); - - let attrs = vm.ctx.new_dict(); - attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; - let module = vm.ctx.new_module(module_name, attrs.clone()); - - // Store module in cache to prevent infinite loop with mutual importing libs: - let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); - sys_modules.set_item(module_name, module.clone(), vm)?; - - // Execute main code in module: - vm.run_code_obj(code_obj, Scope::new(None, attrs))?; - Ok(module) -} - pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { - // First, see if we already loaded the module: + // Cached modules: let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + + // First, see if we already loaded the module: if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { - return Ok(module); + Ok(module) + } else if let Some(make_module_func) = vm.stdlib_inits.borrow().get(module_name) { + let module = make_module_func(vm); + sys_modules.set_item(module_name, module.clone(), vm)?; + Ok(module) + } else { + let notfound_error = vm.context().exceptions.module_not_found_error.clone(); + let import_error = vm.context().exceptions.import_error.clone(); + + // Time to search for module in any place: + let file_path = find_source(vm, current_path, module_name) + .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; + let source = util::read_file(file_path.as_path()) + .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; + let code_obj = compile::compile( + vm, + &source, + &compile::Mode::Exec, + file_path.to_str().unwrap().to_string(), + ) + .map_err(|err| vm.new_syntax_error(&err))?; + // trace!("Code object: {:?}", code_obj); + + let attrs = vm.ctx.new_dict(); + attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; + let module = vm.ctx.new_module(module_name, attrs.clone()); + + // Store module in cache to prevent infinite loop with mutual importing libs: + sys_modules.set_item(module_name, module.clone(), vm)?; + + // Execute main code in module: + vm.run_code_obj(code_obj, Scope::new(None, attrs))?; + + Ok(module) } - let module = import_uncached_module(vm, current_path, module_name)?; - Ok(module) } fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result { From 84457e6e92985109b9456bf1900293d0d9b8270d Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 15:04:50 +0200 Subject: [PATCH 287/658] Add some benchmark scripts. --- benchmarks/README.md | 19 +++++ benchmarks/benchmarks/mandelbrot.py | 31 ++++++++ benchmarks/benchmarks/nbody.py | 110 ++++++++++++++++++++++++++++ benchmarks/test_benchmarks.py | 31 ++++++++ vm/src/compile.rs | 2 +- 5 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/benchmarks/mandelbrot.py create mode 100644 benchmarks/benchmarks/nbody.py create mode 100644 benchmarks/test_benchmarks.py diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..464dc65f2 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,19 @@ + +# Benchmarking + +These are some files to determine performance of rustpython. + +# Usage + +Install pytest and pytest-benchmark: + + $ pip install pytest-benchmark + +Then run: + + $ pytest + +# Benchmark source + +- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html + diff --git a/benchmarks/benchmarks/mandelbrot.py b/benchmarks/benchmarks/mandelbrot.py new file mode 100644 index 000000000..f0106cd74 --- /dev/null +++ b/benchmarks/benchmarks/mandelbrot.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +w = 50.0 +h = 50.0 + +y = 0.0 +while y < h: + x = 0.0 + while x < w: + Zr, Zi, Tr, Ti = 0.0, 0.0, 0.0, 0.0 + Cr = 2*x/w - 1.5 + Ci = 2*y/h - 1.0 + + i = 0 + while i < 50 and Tr+Ti <= 4: + Zi = 2*Zr*Zi + Ci + Zr = Tr - Ti + Cr + Tr = Zr * Zr + Ti = Zi * Zi + i = i+1 + + if Tr+Ti <= 4: + print('*', end='') + else: + print('·', end='') + + x = x+1 + + print() + y = y+1 diff --git a/benchmarks/benchmarks/nbody.py b/benchmarks/benchmarks/nbody.py new file mode 100644 index 000000000..2f36bdf0e --- /dev/null +++ b/benchmarks/benchmarks/nbody.py @@ -0,0 +1,110 @@ + +# The Computer Language Benchmarks Game +# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ +# +# originally by Kevin Carson +# modified by Tupteq, Fredrik Johansson, and Daniel Nanz +# modified by Maciej Fijalkowski +# 2to3 +# modified by Andriy Misyura + +from math import sqrt + +def combinations(l): + result = [] + for x in range(len(l) - 1): + ls = l[x+1:] + for y in ls: + result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2])) + return result + +PI = 3.14159265358979323 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 + +BODIES = { + 'sun': ([0.0, 0.0, 0.0], [0.0, 0.0, 0.0], SOLAR_MASS), + + 'jupiter': ([4.84143144246472090e+00, + -1.16032004402742839e+00, + -1.03622044471123109e-01], + [1.66007664274403694e-03 * DAYS_PER_YEAR, + 7.69901118419740425e-03 * DAYS_PER_YEAR, + -6.90460016972063023e-05 * DAYS_PER_YEAR], + 9.54791938424326609e-04 * SOLAR_MASS), + + 'saturn': ([8.34336671824457987e+00, + 4.12479856412430479e+00, + -4.03523417114321381e-01], + [-2.76742510726862411e-03 * DAYS_PER_YEAR, + 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.30417297573763929e-05 * DAYS_PER_YEAR], + 2.85885980666130812e-04 * SOLAR_MASS), + + 'uranus': ([1.28943695621391310e+01, + -1.51111514016986312e+01, + -2.23307578892655734e-01], + [2.96460137564761618e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR], + 4.36624404335156298e-05 * SOLAR_MASS), + + 'neptune': ([1.53796971148509165e+01, + -2.59193146099879641e+01, + 1.79258772950371181e-01], + [2.68067772490389322e-03 * DAYS_PER_YEAR, + 1.62824170038242295e-03 * DAYS_PER_YEAR, + -9.51592254519715870e-05 * DAYS_PER_YEAR], + 5.15138902046611451e-05 * SOLAR_MASS) } + +SYSTEM = tuple(BODIES.values()) +PAIRS = tuple(combinations(SYSTEM)) + +def advance(dt, n, bodies=SYSTEM, pairs=PAIRS): + for i in range(n): + for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs: + dx = x1 - x2 + dy = y1 - y2 + dz = z1 - z2 + dist = sqrt(dx * dx + dy * dy + dz * dz); + mag = dt / (dist*dist*dist) + b1m = m1 * mag + b2m = m2 * mag + v1[0] -= dx * b2m + v1[1] -= dy * b2m + v1[2] -= dz * b2m + v2[2] += dz * b1m + v2[1] += dy * b1m + v2[0] += dx * b1m + for (r, [vx, vy, vz], m) in bodies: + r[0] += dt * vx + r[1] += dt * vy + r[2] += dt * vz + +def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0): + for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs: + dx = x1 - x2 + dy = y1 - y2 + dz = z1 - z2 + e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5) + for (r, [vx, vy, vz], m) in bodies: + e += m * (vx * vx + vy * vy + vz * vz) / 2. + print(f"{e}") + +def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0): + for (r, [vx, vy, vz], m) in bodies: + px -= vx * m + py -= vy * m + pz -= vz * m + (r, v, m) = ref + v[0] = px / m + v[1] = py / m + v[2] = pz / m + +def main(n, ref='sun'): + offset_momentum(BODIES[ref]) + report_energy() + advance(0.01, n) + report_energy() + +main(500) diff --git a/benchmarks/test_benchmarks.py b/benchmarks/test_benchmarks.py new file mode 100644 index 000000000..571785e60 --- /dev/null +++ b/benchmarks/test_benchmarks.py @@ -0,0 +1,31 @@ + +import time +import sys + +import pytest +import subprocess + +from benchmarks import nbody + +# Interpreters: +rustpython_exe = '../target/release/rustpython' +cpython_exe = sys.executable +pythons = [ + cpython_exe, + rustpython_exe +] + +# Benchmark scripts: +benchmarks = [ + ['benchmarks/nbody.py'], + ['benchmarks/mandelbrot.py'], +] + +@pytest.mark.parametrize('exe', pythons) +@pytest.mark.parametrize('args', benchmarks) +def test_bench(exe, args, benchmark): + def bench(): + subprocess.run([exe] + args) + + benchmark(bench) + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 8635e21ac..b34b36e23 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -1014,7 +1014,7 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Tuple { elements } => { + ast::Expression::List { elements } | ast::Expression::Tuple { elements } => { let mut seen_star = false; // Scan for star args: From 9c57ae4046c588633bc2c7bcede52822c6cc8804 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 15:28:14 +0200 Subject: [PATCH 288/658] support bytes creation from hex and ascii --- parser/src/lexer.rs | 104 +++++++++++++++++++++++++++++++++++++++- tests/snippets/bytes.py | 4 ++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index dfc9ecd51..b9e39e7df 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -542,7 +542,7 @@ where let tok = if is_bytes { if string_content.is_ascii() { Tok::Bytes { - value: string_content.as_bytes().to_vec(), + value: self.lex_byte(string_content)?, } } else { return Err(LexicalError::StringError); @@ -1105,6 +1105,84 @@ where let tok_end = self.get_pos(); Ok((tok_start, ty, tok_end)) } + + fn lex_byte(&self, s: String) -> Result, LexicalError> { + let mut res = vec![]; + let mut escape = false; //flag if previous was \ + let mut hex_on = false; // hex mode on or off + let mut hex_value = String::new(); + + for c in s.chars() { + match c { + '\\' => { + if escape { + res.push(92); + escape = false; + } else { + escape = true; + } + } + + 'x' => { + if escape { + hex_on = true; + } else { + res.push(120); + } + escape = false; + } + 't' => { + if escape { + res.push(9); + } else { + res.push(116); + } + escape = false; + } + 'n' => { + if escape { + res.push(10); + } else { + res.push(110) + } + escape = false; + } + 'r' => { + if escape { + res.push(13); + } else { + res.push(114) + } + escape = false; + } + x => { + if hex_on { + if x.is_ascii_hexdigit() { + if hex_value.is_empty() { + hex_value.push(x); + continue; + } else { + hex_value.push(x); + res.push(u8::from_str_radix(&hex_value, 16).unwrap()); + hex_on = false; + hex_value.clear(); + } + } else { + return Err(LexicalError::StringError); + } + } else { + if escape { + res.push(92); + } + res.push(x as u8); + } + escape = false; + } + } + } + + Ok(res) + } } /* Implement iterator pattern for the get_tok function. @@ -1520,4 +1598,28 @@ mod tests { test_string_continuation_mac_eol: MAC_EOL, test_string_continuation_unix_eol: UNIX_EOL, } + + #[test] + fn test_byte() { + // single quote + let all = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = (0..=255).collect::>(); + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + + // double quote + let all = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = (0..=255).collect::>(); + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + + // backslash doesnt escape + let all = r##"b"omkmok\Xaa""##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97]; + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + } } diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index d2a60943c..c496bfeff 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -10,6 +10,10 @@ assert bytes("bla", "utf8") with assertRaises(TypeError): bytes("bla") +assert b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" == bytes(range(0,256)) +assert b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff' == bytes(range(0,256)) +assert b"omkmok\Xaa" == bytes([111, 109, 107, 109, 111, 107, 92, 88, 97, 97]) + a = b"abcd" b = b"ab" From eb16f1656647e3707364bb422da1ed493cae49d8 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 20:02:24 +0200 Subject: [PATCH 289/658] Improve syntax error with line information. --- src/main.rs | 6 ++-- tests/snippets/invalid_syntax.py | 16 +++++++++++ vm/src/builtins.rs | 1 + vm/src/compile.rs | 48 ++++++++++++++++++++++++-------- vm/src/error.rs | 43 ++++++++++++++++++++-------- vm/src/exceptions.rs | 5 +++- vm/src/vm.rs | 10 +++++-- wasm/lib/src/vm_class.rs | 4 ++- 8 files changed, 101 insertions(+), 32 deletions(-) create mode 100644 tests/snippets/invalid_syntax.py diff --git a/src/main.rs b/src/main.rs index 2a673736e..326e0bc3e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ extern crate rustyline; use clap::{App, Arg}; use rustpython_parser::error::ParseError; use rustpython_vm::{ - compile, error::CompileError, frame::Scope, import, obj::objstr, print_exception, + compile, error::CompileError, error::CompileErrorType, frame::Scope, import, obj::objstr, print_exception, pyobject::PyResult, util, VirtualMachine, }; use rustyline::{error::ReadlineError, Editor}; @@ -115,7 +115,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com Ok(()) } // Don't inject syntax errors for line continuation - Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err), + Err(err @ CompileError{ error: CompileErrorType::Parse(ParseError::EOF(_)), .. }) => Err(err), Err(err) => { let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); @@ -188,7 +188,7 @@ fn run_shell(vm: &VirtualMachine) -> PyResult { } match shell_exec(vm, &input, vars.clone()) { - Err(CompileError::Parse(ParseError::EOF(_))) => { + Err(CompileError { error: CompileErrorType::Parse(ParseError::EOF(_)), .. }) => { continuing = true; continue; } diff --git a/tests/snippets/invalid_syntax.py b/tests/snippets/invalid_syntax.py new file mode 100644 index 000000000..bbbd08a41 --- /dev/null +++ b/tests/snippets/invalid_syntax.py @@ -0,0 +1,16 @@ + + +src = """ +def valid_func(): + pass + +yield 2 +""" + +try: + compile(src, 'test.py', 'exec') +except SyntaxError as ex: + assert ex.lineno == 5 +else: + raise AssertionError("Must throw syntax error") + diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index f2ca1e4c1..d054c34fc 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -751,6 +751,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "OverflowError" => ctx.exceptions.overflow_error.clone(), "RuntimeError" => ctx.exceptions.runtime_error.clone(), "ReferenceError" => ctx.exceptions.reference_error.clone(), + "SyntaxError" => ctx.exceptions.syntax_error.clone(), "NotImplementedError" => ctx.exceptions.not_implemented_error.clone(), "TypeError" => ctx.exceptions.type_error.clone(), "ValueError" => ctx.exceptions.value_error.clone(), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 8635e21ac..9042bf576 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -6,7 +6,7 @@ //! https://github.com/micropython/micropython/blob/master/py/compile.c use crate::bytecode::{self, CallType, CodeObject, Instruction, Varargs}; -use crate::error::CompileError; +use crate::error::{CompileError, CompileErrorType}; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; @@ -40,17 +40,17 @@ pub fn compile( match mode { Mode::Exec => { - let ast = parser::parse_program(source).map_err(CompileError::Parse)?; + let ast = parser::parse_program(source)?; let symbol_table = make_symbol_table(&ast); compiler.compile_program(&ast, symbol_table) } Mode::Eval => { - let statement = parser::parse_statement(source).map_err(CompileError::Parse)?; + let statement = parser::parse_statement(source)?; let symbol_table = statements_to_symbol_table(&statement); compiler.compile_statement_eval(&statement, symbol_table) } Mode::Single => { - let ast = parser::parse_program(source).map_err(CompileError::Parse)?; + let ast = parser::parse_program(source)?; let symbol_table = make_symbol_table(&ast); compiler.compile_program_single(&ast, symbol_table) } @@ -158,7 +158,10 @@ impl Compiler { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; } else { - return Err(CompileError::ExpectExpr); + return Err(CompileError { + error: CompileErrorType::ExpectExpr, + location: statement.location.clone(), + }); } } self.emit(Instruction::ReturnValue); @@ -397,19 +400,28 @@ impl Compiler { } ast::Statement::Break => { if !self.in_loop { - return Err(CompileError::InvalidBreak); + return Err(CompileError { + error: CompileErrorType::InvalidBreak, + location: statement.location.clone(), + }); } self.emit(Instruction::Break); } ast::Statement::Continue => { if !self.in_loop { - return Err(CompileError::InvalidContinue); + return Err(CompileError { + error: CompileErrorType::InvalidContinue, + location: statement.location.clone(), + }); } self.emit(Instruction::Continue); } ast::Statement::Return { value } => { if !self.in_function_def { - return Err(CompileError::InvalidReturn); + return Err(CompileError { + error: CompileErrorType::InvalidReturn, + location: statement.location.clone(), + }); } match value { Some(v) => { @@ -462,7 +474,10 @@ impl Compiler { self.emit(Instruction::DeleteSubscript); } _ => { - return Err(CompileError::Delete(target.name())); + return Err(CompileError { + error: CompileErrorType::Delete(target.name()), + location: self.current_source_location.clone(), + }); } } } @@ -1021,7 +1036,10 @@ impl Compiler { for (i, element) in elements.iter().enumerate() { if let ast::Expression::Starred { .. } = element { if seen_star { - return Err(CompileError::StarArgs); + return Err(CompileError { + error: CompileErrorType::StarArgs, + location: self.current_source_location.clone(), + }); } else { seen_star = true; self.emit(Instruction::UnpackEx { @@ -1047,7 +1065,10 @@ impl Compiler { } } _ => { - return Err(CompileError::Assign(target.name())); + return Err(CompileError { + error: CompileErrorType::Assign(target.name()), + location: self.current_source_location.clone(), + }); } } @@ -1237,7 +1258,10 @@ impl Compiler { } ast::Expression::Yield { value } => { if !self.in_function_def { - return Err(CompileError::InvalidYield); + return Err(CompileError { + error: CompileErrorType::InvalidYield, + location: self.current_source_location.clone(), + }); } self.mark_generator(); match value { diff --git a/vm/src/error.rs b/vm/src/error.rs index 06eb387e2..81836fa44 100644 --- a/vm/src/error.rs +++ b/vm/src/error.rs @@ -1,10 +1,26 @@ use rustpython_parser::error::ParseError; +use rustpython_parser::lexer::Location; use std::error::Error; use std::fmt; #[derive(Debug)] -pub enum CompileError { +pub struct CompileError { + pub error: CompileErrorType, + pub location: Location, +} + +impl From for CompileError { + fn from(error: ParseError) -> Self { + CompileError { + error: CompileErrorType::Parse(error), + location: Default::default(), // TODO: extract location from parse error! + } + } +} + +#[derive(Debug)] +pub enum CompileErrorType { /// Invalid assignment, cannot store value in target. Assign(&'static str), /// Invalid delete @@ -25,17 +41,20 @@ pub enum CompileError { impl fmt::Display for CompileError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - CompileError::Assign(target) => write!(f, "can't assign to {}", target), - CompileError::Delete(target) => write!(f, "can't delete {}", target), - CompileError::ExpectExpr => write!(f, "Expecting expression, got statement"), - CompileError::Parse(err) => write!(f, "{}", err), - CompileError::StarArgs => write!(f, "Two starred expressions in assignment"), - CompileError::InvalidBreak => write!(f, "'break' outside loop"), - CompileError::InvalidContinue => write!(f, "'continue' outside loop"), - CompileError::InvalidReturn => write!(f, "'return' outside function"), - CompileError::InvalidYield => write!(f, "'yield' outside function"), - } + match &self.error { + CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target), + CompileErrorType::Delete(target) => write!(f, "can't delete {}", target), + CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"), + CompileErrorType::Parse(err) => write!(f, "{}", err), + CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"), + CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"), + CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"), + CompileErrorType::InvalidReturn => write!(f, "'return' outside function"), + CompileErrorType::InvalidYield => write!(f, "'yield' outside function"), + }?; + + // Print line number: + write!(f, " at line {:?}", self.location.get_row()) } } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 58a0f929d..a8b753e1a 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -67,7 +67,10 @@ pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { "".to_string() }; - println!(" File {}, line {}, in {}", filename, lineno, obj_name); + println!( + r##" File "{}", line {}, in {}"##, + filename, lineno, obj_name + ); } else { println!(" File ??"); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index e360b8e5e..6e7710516 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -14,6 +14,7 @@ use std::sync::{Mutex, MutexGuard}; use crate::builtins; use crate::bytecode; +use crate::error::CompileError; use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; use crate::function::PyFuncArgs; use crate::obj::objbool; @@ -234,9 +235,12 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } - pub fn new_syntax_error(&self, msg: &T) -> PyObjectRef { - let syntax_error = self.ctx.exceptions.syntax_error.clone(); - self.new_exception(syntax_error, msg.to_string()) + pub fn new_syntax_error(&self, error: &CompileError) -> PyObjectRef { + let syntax_error_type = self.ctx.exceptions.syntax_error.clone(); + let syntax_error = self.new_exception(syntax_error_type, error.to_string()); + let lineno = self.new_int(error.location.get_row()); + self.set_attr(&syntax_error, "lineno", lineno).unwrap(); + syntax_error } pub fn get_none(&self) -> PyObjectRef { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index ff72dddbb..2c330e6ee 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -266,7 +266,9 @@ impl WASMVirtualMachine { let code = compile::compile(vm, &source, &mode, "".to_string()); let code = code.map_err(|err| { let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); - if let rustpython_vm::error::CompileError::Parse(ref parse_error) = err { + if let rustpython_vm::error::CompileErrorType::Parse(ref parse_error) = + err.error + { use rustpython_parser::error::ParseError; if let ParseError::EOF(Some(ref loc)) | ParseError::ExtraToken((ref loc, ..)) From b34784e9efdcdc1449299a67f84cc3b07634b156 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 17 Apr 2019 20:52:53 +0200 Subject: [PATCH 290/658] Format main.rs --- src/main.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 326e0bc3e..24b46821b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,8 +10,8 @@ extern crate rustyline; use clap::{App, Arg}; use rustpython_parser::error::ParseError; use rustpython_vm::{ - compile, error::CompileError, error::CompileErrorType, frame::Scope, import, obj::objstr, print_exception, - pyobject::PyResult, util, VirtualMachine, + compile, error::CompileError, error::CompileErrorType, frame::Scope, import, obj::objstr, + print_exception, pyobject::PyResult, util, VirtualMachine, }; use rustyline::{error::ReadlineError, Editor}; use std::path::{Path, PathBuf}; @@ -115,7 +115,12 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com Ok(()) } // Don't inject syntax errors for line continuation - Err(err @ CompileError{ error: CompileErrorType::Parse(ParseError::EOF(_)), .. }) => Err(err), + Err( + err @ CompileError { + error: CompileErrorType::Parse(ParseError::EOF(_)), + .. + }, + ) => Err(err), Err(err) => { let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); @@ -188,7 +193,10 @@ fn run_shell(vm: &VirtualMachine) -> PyResult { } match shell_exec(vm, &input, vars.clone()) { - Err(CompileError { error: CompileErrorType::Parse(ParseError::EOF(_)), .. }) => { + Err(CompileError { + error: CompileErrorType::Parse(ParseError::EOF(_)), + .. + }) => { continuing = true; continue; } From eb2d0b01eec8a5ce1d73ce31dda798715d3dedd5 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 08:06:06 +0200 Subject: [PATCH 291/658] refactor lex byte --- parser/src/lexer.rs | 128 +++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 79 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index b9e39e7df..11d30cbee 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -542,7 +542,7 @@ where let tok = if is_bytes { if string_content.is_ascii() { Tok::Bytes { - value: self.lex_byte(string_content)?, + value: lex_byte(string_content)?, } } else { return Err(LexicalError::StringError); @@ -1105,84 +1105,6 @@ where let tok_end = self.get_pos(); Ok((tok_start, ty, tok_end)) } - - fn lex_byte(&self, s: String) -> Result, LexicalError> { - let mut res = vec![]; - let mut escape = false; //flag if previous was \ - let mut hex_on = false; // hex mode on or off - let mut hex_value = String::new(); - - for c in s.chars() { - match c { - '\\' => { - if escape { - res.push(92); - escape = false; - } else { - escape = true; - } - } - - 'x' => { - if escape { - hex_on = true; - } else { - res.push(120); - } - escape = false; - } - 't' => { - if escape { - res.push(9); - } else { - res.push(116); - } - escape = false; - } - 'n' => { - if escape { - res.push(10); - } else { - res.push(110) - } - escape = false; - } - 'r' => { - if escape { - res.push(13); - } else { - res.push(114) - } - escape = false; - } - x => { - if hex_on { - if x.is_ascii_hexdigit() { - if hex_value.is_empty() { - hex_value.push(x); - continue; - } else { - hex_value.push(x); - res.push(u8::from_str_radix(&hex_value, 16).unwrap()); - hex_on = false; - hex_value.clear(); - } - } else { - return Err(LexicalError::StringError); - } - } else { - if escape { - res.push(92); - } - res.push(x as u8); - } - escape = false; - } - } - } - - Ok(res) - } } /* Implement iterator pattern for the get_tok function. @@ -1211,6 +1133,54 @@ where } } +fn lex_byte(s: String) -> Result, LexicalError> { + let mut res = vec![]; + let mut escape = false; //flag if previous was \ + let mut hex_on = false; // hex mode on or off + let mut hex_value = String::new(); + + for c in s.chars() { + if hex_on { + if c.is_ascii_hexdigit() { + if hex_value.is_empty() { + hex_value.push(c); + continue; + } else { + hex_value.push(c); + res.push(u8::from_str_radix(&hex_value, 16).unwrap()); + hex_on = false; + hex_value.clear(); + } + } else { + return Err(LexicalError::StringError); + } + } else { + match (c, escape) { + ('\\', true) => res.push(b'\\'), + ('\\', false) => { + escape = true; + continue; + } + ('x', true) => hex_on = true, + ('x', false) => res.push(b'x'), + ('t', true) => res.push(b'\t'), + ('t', false) => res.push(b't'), + ('n', true) => res.push(b'\n'), + ('n', false) => res.push(b'n'), + ('r', true) => res.push(b'\r'), + ('r', false) => res.push(b'r'), + (x, true) => { + res.push(b'\\'); + res.push(x as u8); + } + (x, false) => res.push(x as u8), + } + escape = false; + } + } + Ok(res) +} + #[cfg(test)] mod tests { use super::{make_tokenizer, NewlineHandler, Tok}; From c550360be5f931763f7a44f4f1d35de95377a557 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 13 Apr 2019 08:39:29 +0200 Subject: [PATCH 292/658] add is_bytes_like add bytes.count wthout slice --- tests/snippets/bytes.py | 8 ++++++++ vm/src/obj/objbyteinner.rs | 27 +++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 12 +++++++++++- vm/src/obj/objmemory.rs | 7 +++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index c496bfeff..64cc003d3 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -162,3 +162,11 @@ with assertRaises(TypeError): with assertRaises(TypeError): b"b".center(2, b"ba") b"kok".center(5, bytearray(b"x")) + +# count +assert b"azeazerazeazopia".count(b"aze") == 3 +assert b"azeazerazeazopia".count(b"az") == 4 +assert b"azeazerazeazopia".count(b"a") == 5 +assert b"123456789".count(b"") == 10 +assert b"azeazerazeazopia".count(bytearray(b"aze")) == 3 +assert b"azeazerazeazopia".count(memoryview(b"aze")) == 3 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index d0a63633a..73808f533 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -19,6 +19,7 @@ use num_traits::ToPrimitive; use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; use super::objbytes::PyBytes; +use super::objmemory::PyMemoryView; #[derive(Debug, Default, Clone)] pub struct PyByteInner { @@ -386,6 +387,23 @@ impl PyByteInner { res } + + pub fn count(&self, sub: Vec) -> usize { + if sub.is_empty() { + return self.len() + 1; + } + + let mut total: usize = 0; + for (n, i) in self.elements.iter().enumerate() { + if n + sub.len() <= self.len() + && *i == sub[0] + && &self.elements[n..n + sub.len()] == sub.as_slice() + { + total += 1; + } + } + total + } } pub fn is_byte(obj: &PyObjectRef) -> Option> { @@ -395,3 +413,12 @@ pub fn is_byte(obj: &PyObjectRef) -> Option> { j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), _ => None) } + +pub fn is_bytes_like(obj: &PyObjectRef) -> Option> { + match_class!(obj.clone(), + + i @ PyBytes => Some(i.get_value().to_vec()), + j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), + k @ PyMemoryView => Some(k.get_obj_value().unwrap()), + _ => None) +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0556de053..fd3fad758 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{is_byte, PyByteInner}; +use super::objbyteinner::{is_byte, is_bytes_like, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -269,6 +269,16 @@ impl PyBytesRef { obj => {Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)))} ) } + + #[pymethod(name = "count")] + fn count(self, width: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match is_bytes_like(&width) { + Some(value) => Ok(vm.new_int(self.inner.count(value))), + None => { + Err(vm.new_type_error(format!("a bytes-like object is required, not {}", width))) + } + } + } } #[derive(Debug)] diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index cfe44dea3..a0e0613f9 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,3 +1,4 @@ +use crate::obj::objbyteinner::is_byte; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; @@ -9,6 +10,12 @@ pub struct PyMemoryView { obj: PyObjectRef, } +impl PyMemoryView { + pub fn get_obj_value(&self) -> Option> { + is_byte(&self.obj) + } +} + impl PyValue for PyMemoryView { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.memoryview_type() From 00cf5248147deb8df6fd0ddc0950270141790e1e Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 14 Apr 2019 21:33:53 +0200 Subject: [PATCH 293/658] add slice capability to bytes. add trait IsByte --- tests/snippets/bytes.py | 11 ++++++++ vm/src/obj/objbyteinner.rs | 57 +++++++++++++++++++++++++++++++++----- vm/src/obj/objbytes.rs | 21 +++++++++----- 3 files changed, 75 insertions(+), 14 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 64cc003d3..10de0d900 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -170,3 +170,14 @@ assert b"azeazerazeazopia".count(b"a") == 5 assert b"123456789".count(b"") == 10 assert b"azeazerazeazopia".count(bytearray(b"aze")) == 3 assert b"azeazerazeazopia".count(memoryview(b"aze")) == 3 +assert b"azeazerazeazopia".count(memoryview(b"aze"),1,9 ) == 1 +assert b"azeazerazeazopia".count(b"aze", None, None) == 3 +assert b"azeazerazeazopia".count(b"aze", 2, None) == 2 +assert b"azeazerazeazopia".count(b"aze", 2) == 2 +assert b"azeazerazeazopia".count(b"aze", None, 7) == 2 +assert b"azeazerazeazopia".count(b"aze", None, 7) == 2 +assert b"azeazerazeazopia".count(b"aze", 2, 7) == 1 +assert b"azeazerazeazopia".count(b"aze", -13, -10) == 1 +assert b"azeazerazeazopia".count(b"aze", 1, 10000) == 2 +with assertRaises(ValueError): + b"ilj".count(355) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 73808f533..1d72bf209 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -20,6 +20,7 @@ use num_traits::ToPrimitive; use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; use super::objbytes::PyBytes; use super::objmemory::PyMemoryView; +use super::objnone::PyNone; #[derive(Debug, Default, Clone)] pub struct PyByteInner { @@ -388,21 +389,52 @@ impl PyByteInner { res } - pub fn count(&self, sub: Vec) -> usize { + pub fn count( + &self, + sub: Vec, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> Result { + let start = if let OptionalArg::Present(st) = start { + match_class!(st, + i @ PyInt => {Some(i.as_bigint().clone())}, + _obj @ PyNone => None, + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + ) + } else { + None + }; + let end = if let OptionalArg::Present(e) = end { + match_class!(e, + i @ PyInt => {Some(i.as_bigint().clone())}, + _obj @ PyNone => None, + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + ) + } else { + None + }; + + let range = self.elements.get_slice_range(&start, &end); + if sub.is_empty() { - return self.len() + 1; + return Ok(self.len() + 1); } let mut total: usize = 0; - for (n, i) in self.elements.iter().enumerate() { - if n + sub.len() <= self.len() - && *i == sub[0] - && &self.elements[n..n + sub.len()] == sub.as_slice() + let mut i_start = range.start; + let i_end = range.end; + + for i in self.elements.do_slice(range) { + if i_start + sub.len() <= i_end + && i == sub[0] + && &self.elements[i_start..(i_start + sub.len())] == sub.as_slice() { total += 1; } + i_start += 1; } - total + Ok(total) } } @@ -422,3 +454,14 @@ pub fn is_bytes_like(obj: &PyObjectRef) -> Option> { k @ PyMemoryView => Some(k.get_obj_value().unwrap()), _ => None) } + +pub trait IsByte: ToPrimitive { + fn is_byte(&self, vm: &VirtualMachine) -> Result { + match self.to_u8() { + Some(value) => Ok(value), + None => Err(vm.new_value_error("byte must be in range(0, 256)".to_string())), + } + } +} + +impl IsByte for BigInt {} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index fd3fad758..338e51cbb 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{is_byte, is_bytes_like, PyByteInner}; +use super::objbyteinner::{is_byte, is_bytes_like, IsByte, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -271,12 +271,19 @@ impl PyBytesRef { } #[pymethod(name = "count")] - fn count(self, width: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match is_bytes_like(&width) { - Some(value) => Ok(vm.new_int(self.inner.count(value))), - None => { - Err(vm.new_type_error(format!("a bytes-like object is required, not {}", width))) - } + fn count( + self, + sub: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + match is_bytes_like(&sub) { + Some(value) => Ok(vm.new_int(self.inner.count(value, start, end, vm)?)), + None => match_class!(sub, + i @ PyInt => { + Ok(vm.new_int(self.inner.count(vec![i.as_bigint().is_byte(vm)?], start, end, vm)?))}, + obj => {Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))}), } } } From e6d6b77698ab94e7417d5d2085d37e767f48ef4e Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 14 Apr 2019 21:43:54 +0200 Subject: [PATCH 294/658] fix error msg --- vm/src/obj/objbytes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 338e51cbb..e8fdf0e56 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -283,7 +283,7 @@ impl PyBytesRef { None => match_class!(sub, i @ PyInt => { Ok(vm.new_int(self.inner.count(vec![i.as_bigint().is_byte(vm)?], start, end, vm)?))}, - obj => {Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))}), + obj => {Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)))}), } } } From 5f5d5c4278ef992728a5c71d2d8efaa6836f4613 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 14 Apr 2019 22:52:27 +0200 Subject: [PATCH 295/658] add bytes.join --- tests/snippets/bytes.py | 12 ++++++++++-- vm/src/obj/objbyteinner.rs | 21 ++++++++++++++++++++- vm/src/obj/objbytes.rs | 8 ++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 10de0d900..14fe9c241 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -170,7 +170,7 @@ assert b"azeazerazeazopia".count(b"a") == 5 assert b"123456789".count(b"") == 10 assert b"azeazerazeazopia".count(bytearray(b"aze")) == 3 assert b"azeazerazeazopia".count(memoryview(b"aze")) == 3 -assert b"azeazerazeazopia".count(memoryview(b"aze"),1,9 ) == 1 +assert b"azeazerazeazopia".count(memoryview(b"aze"), 1, 9) == 1 assert b"azeazerazeazopia".count(b"aze", None, None) == 3 assert b"azeazerazeazopia".count(b"aze", 2, None) == 2 assert b"azeazerazeazopia".count(b"aze", 2) == 2 @@ -180,4 +180,12 @@ assert b"azeazerazeazopia".count(b"aze", 2, 7) == 1 assert b"azeazerazeazopia".count(b"aze", -13, -10) == 1 assert b"azeazerazeazopia".count(b"aze", 1, 10000) == 2 with assertRaises(ValueError): - b"ilj".count(355) + b"ilj".count(3550) + +# join +assert ( + b"".join((b"jiljl", bytearray(b"kmoomk"), memoryview(b"aaaa"))) + == b"jiljlkmoomkaaaa" +) +with assertRaises(TypeError): + b"".join((b"km", "kl")) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 1d72bf209..377e93ec5 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,4 +1,4 @@ -use crate::pyobject::PyObjectRef; +use crate::pyobject::{PyIterable, PyObjectRef}; use num_bigint::BigInt; use crate::function::OptionalArg; @@ -436,6 +436,25 @@ impl PyByteInner { } Ok(total) } + + pub fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut refs = vec![]; + for (index, v) in iter.iter(vm)?.enumerate() { + let v = v?; + match is_bytes_like(&v) { + None => { + return Err(vm.new_type_error(format!( + "sequence item {}: expected a bytes-like object, {} found", + index, + &v.class().name, + ))); + } + Some(value) => refs.extend(value), + } + } + + Ok(vm.ctx.new_bytes(refs)) + } } pub fn is_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index e8fdf0e56..e34b4ee5e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -5,13 +5,12 @@ use core::cell::Cell; use std::ops::Deref; use crate::function::OptionalArg; -use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::{is_byte, is_bytes_like, IsByte, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; - /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -286,6 +285,11 @@ impl PyBytesRef { obj => {Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)))}), } } + + #[pymethod(name = "join")] + fn join(self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.join(iter, vm) + } } #[derive(Debug)] From 14658b6236cf0d34be781a3b3664bfa87091edc0 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 08:48:20 +0200 Subject: [PATCH 296/658] move is_byte to try_as_byte move trait IsByte to ByteOr refactor center refactor count fix center with negative value add test for count(integer) --- tests/snippets/bytes.py | 2 ++ vm/src/obj/objbyteinner.rs | 64 +++++++++++++++++++++++++++++++------- vm/src/obj/objbytes.rs | 40 +++--------------------- vm/src/obj/objmemory.rs | 4 +-- 4 files changed, 60 insertions(+), 50 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 14fe9c241..ec8001213 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -162,6 +162,7 @@ with assertRaises(TypeError): with assertRaises(TypeError): b"b".center(2, b"ba") b"kok".center(5, bytearray(b"x")) +b"kok".center(-5,) # count assert b"azeazerazeazopia".count(b"aze") == 3 @@ -181,6 +182,7 @@ assert b"azeazerazeazopia".count(b"aze", -13, -10) == 1 assert b"azeazerazeazopia".count(b"aze", 1, 10000) == 2 with assertRaises(ValueError): b"ilj".count(3550) +assert b"azeazerazeazopia".count(97) == 5 # join assert ( diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 377e93ec5..26f33c2e6 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -362,12 +362,45 @@ impl PyByteInner { .collect::>()) } - pub fn center(&self, width: &BigInt, fillbyte: u8, _vm: &VirtualMachine) -> Vec { - let width = width.to_usize().unwrap(); + pub fn center(&self, width_a: PyObjectRef, fillbyte: OptionalArg, vm: &VirtualMachine) -> PyResult> { + let fillbyte = if let OptionalArg::Present(v) = fillbyte { + match try_as_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + } else { + 32 // default is space + }; + + let width_b = match_class!(width_a, + i @PyInt => i, + obj => {return Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)));} + ); + + + // <0 = no change + let width = if let Some(x) = width_b.as_bigint().to_usize() { + x + } else {return Ok(self.elements.clone());}; + // adjust right et left side if width <= self.len() { - return self.elements.clone(); + return Ok(self.elements.clone()); } let diff: usize = width - self.len(); let mut ln: usize = diff / 2; @@ -386,16 +419,23 @@ impl PyByteInner { res.extend_from_slice(&self.elements[..]); res.extend_from_slice(&vec![fillbyte; rn][..]); - res + Ok(res) } pub fn count( &self, - sub: Vec, + sub: PyObjectRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, - ) -> Result { + ) -> PyResult { + let sub = match try_as_bytes_like(&sub) { + Some(value) => value, + None => match_class!(sub, + i @ PyInt => + vec![i.as_bigint().byte_or(vm)?], + obj => {return Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)));}), + }; let start = if let OptionalArg::Present(st) = start { match_class!(st, i @ PyInt => {Some(i.as_bigint().clone())}, @@ -441,7 +481,7 @@ impl PyByteInner { let mut refs = vec![]; for (index, v) in iter.iter(vm)?.enumerate() { let v = v?; - match is_bytes_like(&v) { + match try_as_bytes_like(&v) { None => { return Err(vm.new_type_error(format!( "sequence item {}: expected a bytes-like object, {} found", @@ -457,7 +497,7 @@ impl PyByteInner { } } -pub fn is_byte(obj: &PyObjectRef) -> Option> { +pub fn try_as_byte(obj: &PyObjectRef) -> Option> { match_class!(obj.clone(), i @ PyBytes => Some(i.get_value().to_vec()), @@ -465,7 +505,7 @@ pub fn is_byte(obj: &PyObjectRef) -> Option> { _ => None) } -pub fn is_bytes_like(obj: &PyObjectRef) -> Option> { +pub fn try_as_bytes_like(obj: &PyObjectRef) -> Option> { match_class!(obj.clone(), i @ PyBytes => Some(i.get_value().to_vec()), @@ -474,8 +514,8 @@ pub fn is_bytes_like(obj: &PyObjectRef) -> Option> { _ => None) } -pub trait IsByte: ToPrimitive { - fn is_byte(&self, vm: &VirtualMachine) -> Result { +pub trait ByteOr: ToPrimitive { + fn byte_or(&self, vm: &VirtualMachine) -> Result { match self.to_u8() { Some(value) => Ok(value), None => Err(vm.new_value_error("byte must be in range(0, 256)".to_string())), @@ -483,4 +523,4 @@ pub trait IsByte: ToPrimitive { } } -impl IsByte for BigInt {} +impl ByteOr for BigInt {} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index e34b4ee5e..851c8d8ab 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{is_byte, is_bytes_like, IsByte, PyByteInner}; +use super::objbyteinner::PyByteInner; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -240,33 +240,7 @@ impl PyBytesRef { fillbyte: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let sym = if let OptionalArg::Present(v) = fillbyte { - match is_byte(&v) { - Some(x) => { - if x.len() == 1 { - x[0] - } else { - return Err(vm.new_type_error(format!( - "center() argument 2 must be a byte string of length 1, not {}", - &v - ))); - } - } - None => { - return Err(vm.new_type_error(format!( - "center() argument 2 must be a byte string of length 1, not {}", - &v - ))); - } - } - } else { - 32 // default is space - }; - - match_class!(width, - i @PyInt => Ok(vm.ctx.new_bytes(self.inner.center(i.as_bigint(), sym, vm))), - obj => {Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)))} - ) + Ok(vm.ctx.new_bytes(self.inner.center(width, fillbyte, vm)?)) } #[pymethod(name = "count")] @@ -276,14 +250,8 @@ impl PyBytesRef { start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { - match is_bytes_like(&sub) { - Some(value) => Ok(vm.new_int(self.inner.count(value, start, end, vm)?)), - None => match_class!(sub, - i @ PyInt => { - Ok(vm.new_int(self.inner.count(vec![i.as_bigint().is_byte(vm)?], start, end, vm)?))}, - obj => {Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)))}), - } + ) -> PyResult { + self.inner.count(sub, start, end, vm) } #[pymethod(name = "join")] diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index a0e0613f9..88040ae14 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,4 +1,4 @@ -use crate::obj::objbyteinner::is_byte; +use crate::obj::objbyteinner::try_as_byte; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; @@ -12,7 +12,7 @@ pub struct PyMemoryView { impl PyMemoryView { pub fn get_obj_value(&self) -> Option> { - is_byte(&self.obj) + try_as_byte(&self.obj) } } From 8a7c46da7b7a26d9ff2e522a551a1267c8be8dbc Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 15:55:37 +0200 Subject: [PATCH 297/658] add endswith startswith --- tests/snippets/bytes.py | 18 +++++++++++- vm/src/obj/objbyteinner.rs | 59 +++++++++++++++++++++++++++++++++++++- vm/src/obj/objbytes.rs | 22 ++++++++++++++ vm/src/obj/objsequence.rs | 19 ++++++++++++ 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index ec8001213..755707a76 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -162,7 +162,7 @@ with assertRaises(TypeError): with assertRaises(TypeError): b"b".center(2, b"ba") b"kok".center(5, bytearray(b"x")) -b"kok".center(-5,) +b"kok".center(-5) # count assert b"azeazerazeazopia".count(b"aze") == 3 @@ -191,3 +191,19 @@ assert ( ) with assertRaises(TypeError): b"".join((b"km", "kl")) + + +# endswith startswith +assert b"abcde".endswith(b"de") +assert b"abcde".endswith(b"") +assert not b"abcde".endswith(b"zx") +assert b"abcde".endswith(b"bc", 0, 3) +assert not b"abcde".endswith(b"bc", 2, 3) +assert b"abcde".endswith((b"c", b"de")) + +assert b"abcde".startswith(b"ab") +assert b"abcde".startswith(b"") +assert not b"abcde".startswith(b"zx") +assert b"abcde".startswith(b"cd", 2) +assert not b"abcde".startswith(b"cd", 1, 4) +assert b"abcde".startswith((b"a", b"bc")) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 26f33c2e6..cc4111a8b 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -12,7 +12,8 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use super::objint; -use super::objsequence::PySliceableSequence; +use super::objtype; +use super::objsequence::{PySliceableSequence, is_valid_slice_arg}; use crate::obj::objint::PyInt; use num_integer::Integer; use num_traits::ToPrimitive; @@ -21,6 +22,7 @@ use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; use super::objbytes::PyBytes; use super::objmemory::PyMemoryView; use super::objnone::PyNone; +use super::objsequence; #[derive(Debug, Default, Clone)] pub struct PyByteInner { @@ -495,6 +497,61 @@ impl PyByteInner { Ok(vm.ctx.new_bytes(refs)) } + + pub fn startsendswith( + &self, + arg: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + endswith: bool, // true for endswith, false for startswith + vm: &VirtualMachine, + ) -> PyResult { + let suff = if objtype::isinstance(&arg, &vm.ctx.tuple_type()) { + let mut flatten = vec![]; + for v in objsequence::get_elements(&arg).to_vec() { + match try_as_bytes_like(&v) { + None => { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + &v.class().name, + ))); + } + Some(value) => flatten.extend(value), + } + } + flatten + } else { + match try_as_bytes_like(&arg) { + Some(value) => value, + None => { + return Err(vm.new_type_error(format!( + "endswith first arg must be bytes or a tuple of bytes, not {}", + arg + ))); + } + } + }; + + if suff.is_empty() { + return Ok(vm.new_bool(true)); + } + let range = self.elements.get_slice_range( + &is_valid_slice_arg(start, vm)?, + &is_valid_slice_arg(end, vm)?, + ); + + if range.end - range.start < suff.len() { + return Ok(vm.new_bool(false)); + } + + let offset = if endswith { + (range.end - suff.len())..range.end + } else { + 0..suff.len() + }; + + Ok(vm.new_bool(suff.as_slice() == &self.elements.do_slice(range)[offset])) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 851c8d8ab..0c6e5c4b9 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -258,6 +258,28 @@ impl PyBytesRef { fn join(self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.join(iter, vm) } + + #[pymethod(name = "endswith")] + fn endswith( + self, + suffix: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.startsendswith(suffix, start, end, true, vm) + } + + #[pymethod(name = "startswith")] + fn startswith( + self, + suffix: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.startsendswith(suffix, start, end, false, vm) + } } #[derive(Debug)] diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 5594ac858..af2034370 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -1,3 +1,5 @@ +use crate::function::OptionalArg; +use crate::obj::objnone::PyNone; use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; @@ -371,3 +373,20 @@ pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut, + vm: &VirtualMachine, +) -> Result, PyObjectRef> { + if let OptionalArg::Present(value) = arg { + match_class!(value, + i @ PyInt => Ok(Some(i.as_bigint().clone())), + _obj @ PyNone => Ok(None), + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + // TODO: check for an __index__ method + ) + } else { + Ok(None) + } +} From bd2166789df931ea393fae0fe666eadb3e9473ed Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 16:12:44 +0200 Subject: [PATCH 298/658] add bytes index and find --- tests/snippets/bytes.py | 25 +++++++++++++++++++++++++ vm/src/obj/objbyteinner.rs | 38 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 755707a76..dc9a439bc 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -207,3 +207,28 @@ assert not b"abcde".startswith(b"zx") assert b"abcde".startswith(b"cd", 2) assert not b"abcde".startswith(b"cd", 1, 4) assert b"abcde".startswith((b"a", b"bc")) + + +# index find +assert b"abcd".index(b"cd") == 2 +assert b"abcd".index(b"cd", 0) == 2 +assert b"abcd".index(b"cd", 1) == 2 +assert b"abcd".index(99) == 2 +with assertRaises(ValueError): + b"abcde".index(b"c", 3, 1) +with assertRaises(ValueError): + b"abcd".index(b"cdaaaaa") +with assertRaises(ValueError): + b"abcd".index(b"b", 3, 4) +with assertRaises(ValueError): + b"abcd".index(1) + + +assert b"abcd".find(b"cd") == 2 +assert b"abcd".find(b"cd", 0) == 2 +assert b"abcd".find(b"cd", 1) == 2 +assert b"abcde".find(b"c", 3, 1) == -1 +assert b"abcd".find(b"cdaaaaa") == -1 +assert b"abcd".find(b"b", 3, 4) == -1 +assert b"abcd".find(1) == -1 +assert b"abcd".find(99) == 2 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index cc4111a8b..5c5e97c25 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -552,6 +552,44 @@ impl PyByteInner { Ok(vm.new_bool(suff.as_slice() == &self.elements.do_slice(range)[offset])) } + + pub fn find( + &self, + sub: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> Result { + let sub = match try_as_bytes_like(&sub) { + Some(value) => value, + None => match_class!(sub, + i @ PyInt => + vec![i.as_bigint().byte_or(vm)?], + obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}), + }; + + let range = self.elements.get_slice_range( + &is_valid_slice_arg(start, vm)?, + &is_valid_slice_arg(end, vm)?, + ); + + // not allowed for this method + if range.end < range.start { + return Ok(-1isize); + } + + let start = range.start; + + let slice = &self.elements[range]; + for (n, _) in slice.iter().enumerate() { + if n + sub.len() <= slice.len() + && &slice[n..n + sub.len()] == sub.as_slice() + { + return Ok((start+n) as isize); + } + } + Ok(-1isize) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0c6e5c4b9..a53b371d0 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -280,6 +280,32 @@ impl PyBytesRef { ) -> PyResult { self.inner.startsendswith(suffix, start, end, false, vm) } + + #[pymethod(name = "find")] + fn find( + self, + sub: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm.new_int(self.inner.find(sub, start, end, vm)?)) + } + + #[pymethod(name = "index")] + fn index( + self, + sub: PyObjectRef, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let res = self.inner.find(sub, start, end, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(vm.new_int(res)) + } } #[derive(Debug)] From ced6f15a59155f42f2278c69dd652fc4c0b2f2be Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 16:16:43 +0200 Subject: [PATCH 299/658] add bytes.makertans --- tests/snippets/bytes.py | 9 +++++++++ vm/src/obj/objbyteinner.rs | 24 ++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 2 ++ 3 files changed, 35 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index dc9a439bc..ebd3eb877 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -232,3 +232,12 @@ assert b"abcd".find(b"cdaaaaa") == -1 assert b"abcd".find(b"b", 3, 4) == -1 assert b"abcd".find(1) == -1 assert b"abcd".find(99) == 2 + + +# make trans +# fmt: off +assert ( + bytes.maketrans(memoryview(b"abc"), bytearray(b"zzz")) + == bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 122, 122, 122, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]) +) +# fmt: on diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 5c5e97c25..c5fda16c8 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -590,6 +590,30 @@ impl PyByteInner { } Ok(-1isize) } + + pub fn maketrans(from: PyObjectRef, to: PyObjectRef, vm :&VirtualMachine) -> PyResult{ + let mut res = vec![]; + + let from = match try_as_bytes_like(&from) { + Some(value) => value, + None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", from)));}, + }; + + let to = match try_as_bytes_like(&to) { + Some(value) => value, + None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", to)));}, + }; + + for i in 0..=255 { + res.push(if let Some(position) = from.iter().position(|&x| x ==i) { + to[position] + } else { + i + }); + } + + Ok(vm.ctx.new_bytes(res)) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index a53b371d0..da675547b 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -62,6 +62,8 @@ pub fn init(context: &PyContext) { let bytes_type = &context.bytes_type; extend_class!(context, bytes_type, { "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), + "maketrans" => context.new_rustfunc(PyByteInner::maketrans), + }); let bytesiterator_type = &context.bytesiterator_type; extend_class!(context, bytesiterator_type, { From 0b283102a6c69022a2d79219c953a3d098456658 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 17 Apr 2019 17:52:45 +0200 Subject: [PATCH 300/658] add translate --- tests/snippets/bytes.py | 5 +++++ vm/src/obj/objbyteinner.rs | 30 ++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 10 ++++++++++ 3 files changed, 45 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index ebd3eb877..602ff9000 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -241,3 +241,8 @@ assert ( == bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 122, 122, 122, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]) ) # fmt: on + +# translate +assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"h") == b'btuybyubuyb' +assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"a") == b'abatuybyubuyb' +assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab")) == b'abatuybyubuyb' diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index c5fda16c8..aa35ffaa0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -612,6 +612,36 @@ impl PyByteInner { }); } + Ok(vm.ctx.new_bytes(res)) + } + pub fn translate(&self, table: PyObjectRef, delete: OptionalArg, vm :&VirtualMachine) -> PyResult{ + let table = match try_as_bytes_like(&table) { + Some(value) => value, + None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", table)));}, + }; + + if table.len() != 256 { + return Err(vm.new_value_error("translation table must be 256 characters long".to_string())); + } + + + + let delete = if let OptionalArg::Present(value) = delete { + match try_as_bytes_like(&value) { + Some(value) => value, + None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", value)));} + } + } else {vec![]}; + + + let mut res = vec![]; + + for i in self.elements.iter() { + if !delete.contains(&i) { + res.push(table[*i as usize]); + } + } + Ok(vm.ctx.new_bytes(res)) } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index da675547b..90f7adc07 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -308,6 +308,16 @@ impl PyBytesRef { } Ok(vm.new_int(res)) } + + #[pymethod(name = "translate")] + fn translate( + self, + table: PyObjectRef, + delete: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.translate(table, delete, vm) + } } #[derive(Debug)] From 8693adcd611f218189de5ef44dbe88dfcfbdc24c Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 08:41:44 +0200 Subject: [PATCH 301/658] handle None case for bytes.translate --- tests/snippets/bytes.py | 2 ++ vm/src/obj/objbyteinner.rs | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 602ff9000..a4a32273a 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -246,3 +246,5 @@ assert ( assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"h") == b'btuybyubuyb' assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"a") == b'abatuybyubuyb' assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab")) == b'abatuybyubuyb' +assert b'hjhtuyfjtyhuhjuyj'.translate(None, b"ht") == b'juyfjyujuyj' + diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index aa35ffaa0..33a16a6c3 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -617,7 +617,13 @@ impl PyByteInner { pub fn translate(&self, table: PyObjectRef, delete: OptionalArg, vm :&VirtualMachine) -> PyResult{ let table = match try_as_bytes_like(&table) { Some(value) => value, - None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", table)));}, + None => { + match_class!(table, + + _n @ PyNone => (0..=255).collect::>(), + obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, + ) + } }; if table.len() != 256 { From 046bd9b6d84d1112faaaaa5d6d1e8d6e8898f0e4 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 14:01:12 +0200 Subject: [PATCH 302/658] reactor bytes.center --- vm/src/obj/objbyteinner.rs | 122 +++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 45 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 33a16a6c3..f666affc5 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -12,8 +12,8 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use super::objint; +use super::objsequence::{is_valid_slice_arg, PySliceableSequence}; use super::objtype; -use super::objsequence::{PySliceableSequence, is_valid_slice_arg}; use crate::obj::objint::PyInt; use num_integer::Integer; use num_traits::ToPrimitive; @@ -364,7 +364,13 @@ impl PyByteInner { .collect::>()) } - pub fn center(&self, width_a: PyObjectRef, fillbyte: OptionalArg, vm: &VirtualMachine) -> PyResult> { + fn get_center_args( + &self, + width_a: PyObjectRef, + fillbyte: OptionalArg, + fn_name: String, + vm: &VirtualMachine, + ) -> PyResult<(u8, usize)> { let fillbyte = if let OptionalArg::Present(v) = fillbyte { match try_as_byte(&v) { Some(x) => { @@ -372,39 +378,52 @@ impl PyByteInner { x[0] } else { return Err(vm.new_type_error(format!( - "center() argument 2 must be a byte string of length 1, not {}", - &v + "{}() argument 2 must be a byte string of length 1, not {}", + &fn_name, &v ))); } } None => { return Err(vm.new_type_error(format!( - "center() argument 2 must be a byte string of length 1, not {}", - &v + "{}() argument 2 must be a byte string of length 1, not {}", + &fn_name, &v ))); } } } else { - 32 // default is space + b' ' // default is space }; - let width_b = match_class!(width_a, + let width_b = match_class!(width_a, i @PyInt => i, obj => {return Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)));} ); - // <0 = no change let width = if let Some(x) = width_b.as_bigint().to_usize() { - x - } else {return Ok(self.elements.clone());}; - + if x <= self.len() { + 0 + } else { + x + } + } else { + 0 + }; + + let diff: usize = if width != 0 { width - self.len() } else { 0 }; + + Ok((fillbyte, diff)) + } + + pub fn center( + &self, + width_a: PyObjectRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + let fn_name = "center".to_string(); + let (fillbyte, diff) = self.get_center_args(width_a, fillbyte, fn_name, vm)?; - // adjust right et left side - if width <= self.len() { - return Ok(self.elements.clone()); - } - let diff: usize = width - self.len(); let mut ln: usize = diff / 2; let mut rn: usize = ln; @@ -434,8 +453,7 @@ impl PyByteInner { let sub = match try_as_bytes_like(&sub) { Some(value) => value, None => match_class!(sub, - i @ PyInt => - vec![i.as_bigint().byte_or(vm)?], + i @ PyInt => vec![i.as_bigint().byte_or(vm)?], obj => {return Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)));}), }; let start = if let OptionalArg::Present(st) = start { @@ -563,8 +581,7 @@ impl PyByteInner { let sub = match try_as_bytes_like(&sub) { Some(value) => value, None => match_class!(sub, - i @ PyInt => - vec![i.as_bigint().byte_or(vm)?], + i @ PyInt => vec![i.as_bigint().byte_or(vm)?], obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}), }; @@ -582,63 +599,78 @@ impl PyByteInner { let slice = &self.elements[range]; for (n, _) in slice.iter().enumerate() { - if n + sub.len() <= slice.len() - && &slice[n..n + sub.len()] == sub.as_slice() - { - return Ok((start+n) as isize); + if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { + return Ok((start + n) as isize); } } Ok(-1isize) } - pub fn maketrans(from: PyObjectRef, to: PyObjectRef, vm :&VirtualMachine) -> PyResult{ - let mut res = vec![]; + pub fn maketrans(from: PyObjectRef, to: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut res = vec![]; let from = match try_as_bytes_like(&from) { Some(value) => value, - None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", from)));}, + None => { + return Err( + vm.new_type_error(format!("a bytes-like object is required, not {}", from)) + ); + } }; let to = match try_as_bytes_like(&to) { Some(value) => value, - None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", to)));}, + None => { + return Err( + vm.new_type_error(format!("a bytes-like object is required, not {}", to)) + ); + } }; for i in 0..=255 { - res.push(if let Some(position) = from.iter().position(|&x| x ==i) { + res.push(if let Some(position) = from.iter().position(|&x| x == i) { to[position] - } else { + } else { i }); } Ok(vm.ctx.new_bytes(res)) } - pub fn translate(&self, table: PyObjectRef, delete: OptionalArg, vm :&VirtualMachine) -> PyResult{ + pub fn translate( + &self, + table: PyObjectRef, + delete: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let table = match try_as_bytes_like(&table) { Some(value) => value, - None => { - match_class!(table, + None => match_class!(table, - _n @ PyNone => (0..=255).collect::>(), - obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, - ) - } + _n @ PyNone => (0..=255).collect::>(), + obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, + ), }; if table.len() != 256 { - return Err(vm.new_value_error("translation table must be 256 characters long".to_string())); + return Err( + vm.new_value_error("translation table must be 256 characters long".to_string()) + ); } - - let delete = if let OptionalArg::Present(value) = delete { - match try_as_bytes_like(&value) { + match try_as_bytes_like(&value) { Some(value) => value, - None => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", value)));} + None => { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + value + ))); + } } - } else {vec![]}; - + } else { + vec![] + }; let mut res = vec![]; From 84d61a9212be2f23c74c7f5138f23c66d7ec3474 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 14:56:47 +0200 Subject: [PATCH 303/658] bytes.new with kwargs --- tests/snippets/bytes.py | 23 +++++++++++++++-------- vm/src/obj/objbyteinner.rs | 24 +++++++++++++++--------- vm/src/obj/objbytes.rs | 7 +++---- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a4a32273a..a36a9b2fd 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -6,12 +6,18 @@ assert bytes((1, 2, 3)) assert bytes(range(4)) assert bytes(3) assert b"bla" -assert bytes("bla", "utf8") +assert bytes("bla", "utf8") == bytes("bla", encoding="utf-8") == b"bla" with assertRaises(TypeError): bytes("bla") -assert b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" == bytes(range(0,256)) -assert b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff' == bytes(range(0,256)) +assert ( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + == bytes(range(0, 256)) +) +assert ( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + == bytes(range(0, 256)) +) assert b"omkmok\Xaa" == bytes([111, 109, 107, 109, 111, 107, 92, 88, 97, 97]) @@ -243,8 +249,9 @@ assert ( # fmt: on # translate -assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"h") == b'btuybyubuyb' -assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab"), b"a") == b'abatuybyubuyb' -assert b'hjhtuyjyujuyj'.translate(bytes.maketrans(b"hj", b"ab")) == b'abatuybyubuyb' -assert b'hjhtuyfjtyhuhjuyj'.translate(None, b"ht") == b'juyfjyujuyj' - +assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab"), b"h") == b"btuybyubuyb" +assert ( + b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab"), b"a") == b"abatuybyubuyb" +) +assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab")) == b"abatuybyubuyb" +assert b"hjhtuyfjtyhuhjuyj".translate(None, b"ht") == b"juyfjyujuyj" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index f666affc5..e65fbb982 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -29,15 +29,19 @@ pub struct PyByteInner { pub elements: Vec, } -impl PyByteInner { - pub fn new( - val_option: OptionalArg, - enc_option: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { +#[derive(FromArgs)] +pub struct BytesNewOptions { + #[pyarg(positional_only, optional = true)] + val_option: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + encoding: OptionalArg, +} + +impl BytesNewOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) - if let OptionalArg::Present(enc) = enc_option { - if let OptionalArg::Present(eval) = val_option { + if let OptionalArg::Present(enc) = self.encoding { + if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { if let Ok(encoding) = enc.clone().downcast::() { if &encoding.value.to_lowercase() == "utf8" @@ -66,7 +70,7 @@ impl PyByteInner { } // Only one argument } else { - let value = if let OptionalArg::Present(ival) = val_option { + let value = if let OptionalArg::Present(ival) = self.val_option { match_class!(ival.clone(), i @ PyInt => { let size = objint::get_value(&i.into_object()).to_usize().unwrap(); @@ -98,7 +102,9 @@ impl PyByteInner { } } } +} +impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); for i in self.elements.iter() { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 90f7adc07..85881a887 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::PyByteInner; +use super::objbyteinner::{BytesNewOptions, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -77,12 +77,11 @@ impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( cls: PyClassRef, - val_option: OptionalArg, - enc_option: OptionalArg, + options: BytesNewOptions, vm: &VirtualMachine, ) -> PyResult { PyBytes { - inner: PyByteInner::new(val_option, enc_option, vm)?, + inner: options.get_value(vm)?, } .into_ref_with_type(vm, cls) } From 473ae241996a6abbc63036d666481bdfab1c7c37 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 17:31:58 +0200 Subject: [PATCH 304/658] use PyStringRef in new --- tests/snippets/bytes.py | 2 ++ vm/src/obj/objbyteinner.rs | 35 ++++++++++++++--------------------- vm/src/obj/objbytes.rs | 4 ++-- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a36a9b2fd..2e6c658be 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -9,6 +9,8 @@ assert b"bla" assert bytes("bla", "utf8") == bytes("bla", encoding="utf-8") == b"bla" with assertRaises(TypeError): bytes("bla") +with assertRaises(TypeError): + bytes("bla", encoding = b"jilj") assert ( b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index e65fbb982..ffd26d483 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -7,7 +7,7 @@ use crate::vm::VirtualMachine; use crate::pyobject::{PyResult, TypeProtocol}; -use crate::obj::objstr::PyString; +use crate::obj::objstr::{PyString, PyStringRef}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -30,37 +30,30 @@ pub struct PyByteInner { } #[derive(FromArgs)] -pub struct BytesNewOptions { +pub struct ByteInnerNewOptions { #[pyarg(positional_only, optional = true)] val_option: OptionalArg, #[pyarg(positional_or_keyword, optional = true)] - encoding: OptionalArg, + encoding: OptionalArg, } -impl BytesNewOptions { +impl ByteInnerNewOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = self.encoding { if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { - if let Ok(encoding) = enc.clone().downcast::() { - if &encoding.value.to_lowercase() == "utf8" - || &encoding.value.to_lowercase() == "utf-8" - // TODO: different encoding - { - return Ok(PyByteInner { - elements: input.value.as_bytes().to_vec(), - }); - } else { - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error - ); - } + let encoding = enc.as_str(); + if encoding.to_lowercase() == "utf8" || encoding.to_lowercase() == "utf-8" + // TODO: different encoding + { + return Ok(PyByteInner { + elements: input.value.as_bytes().to_vec(), + }); } else { - return Err(vm.new_type_error(format!( - "bytes() argument 2 must be str, not {}", - enc.class().name - ))); + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding)), //should be lookup error + ); } } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 85881a887..060732750 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{BytesNewOptions, PyByteInner}; +use super::objbyteinner::{ByteInnerNewOptions, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -77,7 +77,7 @@ impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( cls: PyClassRef, - options: BytesNewOptions, + options: ByteInnerNewOptions, vm: &VirtualMachine, ) -> PyResult { PyBytes { From 44d58f63abc3c744aa23a7a988e0916a4e540f37 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 19:03:08 +0200 Subject: [PATCH 305/658] refactor contains --- vm/src/obj/objbyteinner.rs | 27 ++++++++++++++++----------- vm/src/obj/objbytes.rs | 5 +---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index ffd26d483..8ca040776 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -177,11 +177,20 @@ impl PyByteInner { elements } - pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + pub fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match try_as_bytes_like(&needle) { + Some(value) => self.contains_bytes(&value, vm), + None => match_class!(needle, + i @ PyInt => self.contains_int(&i, vm), + obj => {Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))}), + } + } + + fn contains_bytes(&self, other: &[u8], vm: &VirtualMachine) -> PyResult { for (n, i) in self.elements.iter().enumerate() { if n + other.len() <= self.len() - && *i == other.elements[0] - && &self.elements[n..n + other.len()] == other.elements.as_slice() + && *i == other[0] + && &self.elements[n..n + other.len()] == other { return Ok(vm.new_bool(true)); } @@ -189,15 +198,11 @@ impl PyByteInner { Ok(vm.new_bool(false)) } - pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { - if let Some(int) = int.as_bigint().to_u8() { - if self.elements.contains(&int) { - Ok(vm.new_bool(true)) - } else { - Ok(vm.new_bool(false)) - } + fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if self.elements.contains(&int.as_bigint().byte_or(vm)?) { + Ok(vm.new_bool(true)) } else { - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + Ok(vm.new_bool(false)) } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 060732750..00a398705 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -149,10 +149,7 @@ impl PyBytesRef { #[pymethod(name = "__contains__")] fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(needle, - bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), - int @ PyInt => self.inner.contains_int(&int, vm), - obj => Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))) + self.inner.contains(needle, vm) } #[pymethod(name = "__getitem__")] From 4a4d163626e774cf590806c70853a7e5d8b1319b Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 19:10:37 +0200 Subject: [PATCH 306/658] refactor getitem --- vm/src/obj/objbyteinner.rs | 18 ++++++++++++------ vm/src/obj/objbytes.rs | 8 ++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8ca040776..7edfa7dde 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,3 +1,4 @@ +use crate::obj::objslice::PySlice; use crate::pyobject::{PyIterable, PyObjectRef}; use num_bigint::BigInt; @@ -206,18 +207,23 @@ impl PyByteInner { } } - pub fn getitem_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { - if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + pub fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(needle, + int @ PyInt => //self.inner.getitem_int(&int, vm), + { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { Ok(vm.new_int(self.elements[idx])) } else { Err(vm.new_index_error("index out of range".to_string())) } - } - - pub fn getitem_slice(&self, slice: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + }, + slice @ PySlice => //self.inner.getitem_slice(slice.as_object(), vm), + { Ok(vm .ctx - .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) + }, + obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) } pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 00a398705..46bd34624 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,3 @@ -use crate::obj::objint::PyInt; use crate::obj::objstr::PyString; use crate::vm::VirtualMachine; use core::cell::Cell; @@ -9,7 +8,7 @@ use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, Py use super::objbyteinner::{ByteInnerNewOptions, PyByteInner}; use super::objiter; -use super::objslice::PySlice; + use super::objtype::PyClassRef; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ @@ -154,10 +153,7 @@ impl PyBytesRef { #[pymethod(name = "__getitem__")] fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(needle, - int @ PyInt => self.inner.getitem_int(&int, vm), - slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), - obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) + self.inner.getitem(needle, vm) } #[pymethod(name = "isalnum")] From ba67f3c264ded2514eed37373ca118b1f773ed7d Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 19:24:10 +0200 Subject: [PATCH 307/658] refactore fromhex --- tests/snippets/bytes.py | 3 ++- vm/src/obj/objbyteinner.rs | 2 +- vm/src/obj/objbytes.rs | 13 +++---------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2e6c658be..e3ddcd51d 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -142,7 +142,8 @@ try: bytes.fromhex("6Z2") except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" - +with assertRaises(TypeError): + bytes.fromhex(b'hhjjk') # center assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ b"koki", diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 7edfa7dde..b3e51084a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -342,7 +342,7 @@ impl PyByteInner { Ok(vm.ctx.new_str(bla)) } - pub fn fromhex(string: String, vm: &VirtualMachine) -> Result, PyObjectRef> { + pub fn fromhex(string: &str, vm: &VirtualMachine) -> PyResult> { // first check for invalid character for (i, c) in string.char_indices() { if !c.is_digit(16) && !c.is_whitespace() { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 46bd34624..773675c58 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,4 @@ -use crate::obj::objstr::PyString; +use crate::obj::objstr::PyStringRef; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -216,15 +216,8 @@ impl PyBytesRef { self.inner.hex(vm) } - // #[pymethod(name = "fromhex")] - fn fromhex(string: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(string, - s @ PyString => { - match PyByteInner::fromhex(s.to_string(), vm) { - Ok(x) => Ok(vm.ctx.new_bytes(x)), - Err(y) => Err(y)}}, - obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) - ) + fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(PyByteInner::fromhex(string.as_str(), vm)?)) } #[pymethod(name = "center")] From 8bcdbbf21b0c782274fe80c4e48f4c25aab726f3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 19:52:17 +0200 Subject: [PATCH 308/658] use PyInt Ref, add ljust, rjust --- tests/snippets/bytes.py | 67 ++++++++++++++++++++++++++++++++++++-- vm/src/obj/objbyteinner.rs | 47 +++++++++++++++++++++----- vm/src/obj/objbytes.rs | 23 ++++++++++++- 3 files changed, 124 insertions(+), 13 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index e3ddcd51d..ecb29612a 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -10,7 +10,7 @@ assert bytes("bla", "utf8") == bytes("bla", encoding="utf-8") == b"bla" with assertRaises(TypeError): bytes("bla") with assertRaises(TypeError): - bytes("bla", encoding = b"jilj") + bytes("bla", encoding=b"jilj") assert ( b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" @@ -143,7 +143,7 @@ try: except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" with assertRaises(TypeError): - bytes.fromhex(b'hhjjk') + bytes.fromhex(b"hhjjk") # center assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ b"koki", @@ -170,9 +170,70 @@ with assertRaises(TypeError): b"b".center(2, "a") with assertRaises(TypeError): b"b".center(2, b"ba") -b"kok".center(5, bytearray(b"x")) +assert b"kok".center(5, bytearray(b"x")) == b"xkokx" b"kok".center(-5) + +# ljust +assert [b"koki".ljust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"koki|", + b"koki||", + b"koki|||", + b"koki||||", + b"koki|||||", +] +assert [b"kok".ljust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"kok||", + b"kok|||", + b"kok||||", + b"kok|||||", + b"kok||||||", +] + +b"kok".ljust(4) == b"kok " # " test no arg" +with assertRaises(TypeError): + b"b".ljust(2, "a") +with assertRaises(TypeError): + b"b".ljust(2, b"ba") +assert b"kok".ljust(5, bytearray(b"x")) == b"kokxx" +assert b"kok".ljust(-5) == b"kok" + +# rjust +assert [b"koki".rjust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"||koki", + b"|||koki", + b"||||koki", + b"|||||koki", +] +assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"|kok", + b"||kok", + b"|||kok", + b"||||kok", + b"|||||kok", + b"||||||kok", +] + + +b"kok".rjust(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".rjust(2, "a") +with assertRaises(TypeError): + b"b".rjust(2, b"ba") +assert b"kok".rjust(5, bytearray(b"x")) == b"xxkok" +assert b"kok".rjust(-5) == b"kok" + + # count assert b"azeazerazeazopia".count(b"aze") == 3 assert b"azeazerazeazopia".count(b"az") == 4 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b3e51084a..114742e8c 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,3 +1,4 @@ +use crate::obj::objint::PyIntRef; use crate::obj::objslice::PySlice; use crate::pyobject::{PyIterable, PyObjectRef}; use num_bigint::BigInt; @@ -376,7 +377,7 @@ impl PyByteInner { fn get_center_args( &self, - width_a: PyObjectRef, + width: PyIntRef, fillbyte: OptionalArg, fn_name: String, vm: &VirtualMachine, @@ -404,13 +405,8 @@ impl PyByteInner { b' ' // default is space }; - let width_b = match_class!(width_a, - i @PyInt => i, - obj => {return Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)));} - ); - // <0 = no change - let width = if let Some(x) = width_b.as_bigint().to_usize() { + let width = if let Some(x) = width.as_bigint().to_usize() { if x <= self.len() { 0 } else { @@ -427,12 +423,12 @@ impl PyByteInner { pub fn center( &self, - width_a: PyObjectRef, + width: PyIntRef, fillbyte: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { let fn_name = "center".to_string(); - let (fillbyte, diff) = self.get_center_args(width_a, fillbyte, fn_name, vm)?; + let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; let mut ln: usize = diff / 2; let mut rn: usize = ln; @@ -453,6 +449,39 @@ impl PyByteInner { Ok(res) } + pub fn ljust( + &self, + width: PyIntRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + let fn_name = "ljust".to_string(); + let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; + + // merge all + let mut res = vec![]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; diff][..]); + + Ok(res) + } + + pub fn rjust( + &self, + width: PyIntRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + let fn_name = "ljust".to_string(); + let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; + + // merge all + let mut res = vec![fillbyte; diff]; + res.extend_from_slice(&self.elements[..]); + + Ok(res) + } + pub fn count( &self, sub: PyObjectRef, diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 773675c58..f9d17c4e0 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,3 +1,4 @@ +use crate::obj::objint::PyIntRef; use crate::obj::objstr::PyStringRef; use crate::vm::VirtualMachine; use core::cell::Cell; @@ -223,13 +224,33 @@ impl PyBytesRef { #[pymethod(name = "center")] fn center( self, - width: PyObjectRef, + width: PyIntRef, fillbyte: OptionalArg, vm: &VirtualMachine, ) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.center(width, fillbyte, vm)?)) } + #[pymethod(name = "ljust")] + fn ljust( + self, + width: PyIntRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.ljust(width, fillbyte, vm)?)) + } + + #[pymethod(name = "rjust")] + fn rjust( + self, + width: PyIntRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.rjust(width, fillbyte, vm)?)) + } + #[pymethod(name = "count")] fn count( self, From 1669d5280f20bfbf0043595edf1643812935105c Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 20:41:12 +0200 Subject: [PATCH 309/658] refactor count --- vm/src/obj/objbyteinner.rs | 83 +++++++++++++++++++++++--------------- vm/src/obj/objbytes.rs | 12 ++---- 2 files changed, 53 insertions(+), 42 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 114742e8c..32c2f4240 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,6 +1,7 @@ use crate::obj::objint::PyIntRef; use crate::obj::objslice::PySlice; use crate::pyobject::{PyIterable, PyObjectRef}; +use core::ops::Range; use num_bigint::BigInt; use crate::function::OptionalArg; @@ -99,6 +100,53 @@ impl ByteInnerNewOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerFindOptions { + #[pyarg(positional_only, optional = false)] + sub: PyObjectRef, + #[pyarg(positional_only, optional = true)] + start: OptionalArg, + #[pyarg(positional_only, optional = true)] + end: OptionalArg, +} + +impl ByteInnerFindOptions { + pub fn get_value( + self, + elements: &[u8], + vm: &VirtualMachine, + ) -> PyResult<(Vec, Range)> { + let sub = match try_as_bytes_like(&self.sub.clone()) { + Some(value) => value, + None => match_class!(self.sub, + i @ PyInt => vec![i.as_bigint().byte_or(vm)?], + obj => {return Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)));}), + }; + let start = if let OptionalArg::Present(st) = self.start { + match_class!(st, + i @ PyInt => {Some(i.as_bigint().clone())}, + _obj @ PyNone => None, + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + ) + } else { + None + }; + let end = if let OptionalArg::Present(e) = self.end { + match_class!(e, + i @ PyInt => {Some(i.as_bigint().clone())}, + _obj @ PyNone => None, + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + ) + } else { + None + }; + + let range = elements.to_vec().get_slice_range(&start, &end); + + Ok((sub, range)) + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -482,39 +530,8 @@ impl PyByteInner { Ok(res) } - pub fn count( - &self, - sub: PyObjectRef, - start: OptionalArg, - end: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let sub = match try_as_bytes_like(&sub) { - Some(value) => value, - None => match_class!(sub, - i @ PyInt => vec![i.as_bigint().byte_or(vm)?], - obj => {return Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)));}), - }; - let start = if let OptionalArg::Present(st) = start { - match_class!(st, - i @ PyInt => {Some(i.as_bigint().clone())}, - _obj @ PyNone => None, - _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} - ) - } else { - None - }; - let end = if let OptionalArg::Present(e) = end { - match_class!(e, - i @ PyInt => {Some(i.as_bigint().clone())}, - _obj @ PyNone => None, - _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} - ) - } else { - None - }; - - let range = self.elements.get_slice_range(&start, &end); + pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let (sub, range) = options.get_value(&self.elements, vm)?; if sub.is_empty() { return Ok(self.len() + 1); diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f9d17c4e0..8e8393d12 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{ByteInnerNewOptions, PyByteInner}; +use super::objbyteinner::{ByteInnerFindOptions, ByteInnerNewOptions, PyByteInner}; use super::objiter; use super::objtype::PyClassRef; @@ -252,14 +252,8 @@ impl PyBytesRef { } #[pymethod(name = "count")] - fn count( - self, - sub: PyObjectRef, - start: OptionalArg, - end: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - self.inner.count(sub, start, end, vm) + fn count(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.count(options, vm) } #[pymethod(name = "join")] From 56c5790745b892d731c2ebac75d4864e89f2aad3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 21:00:33 +0200 Subject: [PATCH 310/658] reformat center, ljsut, rjust args --- tests/snippets/bytes.py | 25 ++++---- vm/src/obj/objbyteinner.rs | 119 +++++++++++++++++++------------------ vm/src/obj/objbytes.rs | 32 +++------- 3 files changed, 84 insertions(+), 92 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index ecb29612a..0a3fb11ab 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -170,6 +170,8 @@ with assertRaises(TypeError): b"b".center(2, "a") with assertRaises(TypeError): b"b".center(2, b"ba") +with assertRaises(TypeError): + b"b".center(b"ba") assert b"kok".center(5, bytearray(b"x")) == b"xkokx" b"kok".center(-5) @@ -200,6 +202,8 @@ with assertRaises(TypeError): b"b".ljust(2, "a") with assertRaises(TypeError): b"b".ljust(2, b"ba") +with assertRaises(TypeError): + b"b".ljust(b"ba") assert b"kok".ljust(5, bytearray(b"x")) == b"kokxx" assert b"kok".ljust(-5) == b"kok" @@ -213,16 +217,15 @@ assert [b"koki".rjust(i, b"|") for i in range(3, 10)] == [ b"||||koki", b"|||||koki", ] -assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [ - b"kok", - b"kok", - b"|kok", - b"||kok", - b"|||kok", - b"||||kok", - b"|||||kok", - b"||||||kok", -] +assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [b'kok', + b'kok', + b'|kok', + b'||kok', + b'|||kok', + b'||||kok', + b'|||||kok', + b'||||||kok'] + b"kok".rjust(4) == b" kok" # " test no arg" @@ -230,6 +233,8 @@ with assertRaises(TypeError): b"b".rjust(2, "a") with assertRaises(TypeError): b"b".rjust(2, b"ba") +with assertRaises(TypeError): + b"b".rjust(b"ba") assert b"kok".rjust(5, bytearray(b"x")) == b"xxkok" assert b"kok".rjust(-5) == b"kok" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 32c2f4240..b6e9ae49e 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -116,7 +116,7 @@ impl ByteInnerFindOptions { elements: &[u8], vm: &VirtualMachine, ) -> PyResult<(Vec, Range)> { - let sub = match try_as_bytes_like(&self.sub.clone()) { + let sub = match try_as_bytes_like(&self.sub) { Some(value) => value, None => match_class!(self.sub, i @ PyInt => vec![i.as_bigint().byte_or(vm)?], @@ -147,6 +147,56 @@ impl ByteInnerFindOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerPaddingOptions { + #[pyarg(positional_only, optional = false)] + width: PyIntRef, + #[pyarg(positional_only, optional = true)] + fillbyte: OptionalArg, +} + +impl ByteInnerPaddingOptions { + fn get_value(&self, fn_name: &str, len: usize, vm: &VirtualMachine) -> PyResult<(u8, usize)> { + let fillbyte = if let OptionalArg::Present(v) = &self.fillbyte { + match try_as_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, &v + ))); + } + } + } else { + b' ' // default is space + }; + + // <0 = no change + let width = if let Some(x) = self.width.as_bigint().to_usize() { + if x <= len { + 0 + } else { + x + } + } else { + 0 + }; + + let diff: usize = if width != 0 { width - len } else { 0 }; + + Ok((fillbyte, diff)) + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -423,60 +473,13 @@ impl PyByteInner { .collect::>()) } - fn get_center_args( - &self, - width: PyIntRef, - fillbyte: OptionalArg, - fn_name: String, - vm: &VirtualMachine, - ) -> PyResult<(u8, usize)> { - let fillbyte = if let OptionalArg::Present(v) = fillbyte { - match try_as_byte(&v) { - Some(x) => { - if x.len() == 1 { - x[0] - } else { - return Err(vm.new_type_error(format!( - "{}() argument 2 must be a byte string of length 1, not {}", - &fn_name, &v - ))); - } - } - None => { - return Err(vm.new_type_error(format!( - "{}() argument 2 must be a byte string of length 1, not {}", - &fn_name, &v - ))); - } - } - } else { - b' ' // default is space - }; - - // <0 = no change - let width = if let Some(x) = width.as_bigint().to_usize() { - if x <= self.len() { - 0 - } else { - x - } - } else { - 0 - }; - - let diff: usize = if width != 0 { width - self.len() } else { 0 }; - - Ok((fillbyte, diff)) - } - pub fn center( &self, - width: PyIntRef, - fillbyte: OptionalArg, + options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - let fn_name = "center".to_string(); - let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; + // let fn_name = "center".to_string(); + let (fillbyte, diff) = options.get_value("center", self.len(), vm)?; let mut ln: usize = diff / 2; let mut rn: usize = ln; @@ -499,12 +502,11 @@ impl PyByteInner { pub fn ljust( &self, - width: PyIntRef, - fillbyte: OptionalArg, + options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - let fn_name = "ljust".to_string(); - let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; + // let fn_name = "ljust".to_string(); + let (fillbyte, diff) = options.get_value("ljust", self.len(), vm)?; // merge all let mut res = vec![]; @@ -516,12 +518,11 @@ impl PyByteInner { pub fn rjust( &self, - width: PyIntRef, - fillbyte: OptionalArg, + options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - let fn_name = "ljust".to_string(); - let (fillbyte, diff) = self.get_center_args(width, fillbyte, fn_name, vm)?; + // let fn_name = "rjust".to_string(); + let (fillbyte, diff) = options.get_value("rjust", self.len(), vm)?; // merge all let mut res = vec![fillbyte; diff]; diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8e8393d12..3c1b323c8 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,3 @@ -use crate::obj::objint::PyIntRef; use crate::obj::objstr::PyStringRef; use crate::vm::VirtualMachine; use core::cell::Cell; @@ -7,7 +6,9 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::{ByteInnerFindOptions, ByteInnerNewOptions, PyByteInner}; +use super::objbyteinner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, PyByteInner, +}; use super::objiter; use super::objtype::PyClassRef; @@ -222,33 +223,18 @@ impl PyBytesRef { } #[pymethod(name = "center")] - fn center( - self, - width: PyIntRef, - fillbyte: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - Ok(vm.ctx.new_bytes(self.inner.center(width, fillbyte, vm)?)) + fn center(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.center(options, vm)?)) } #[pymethod(name = "ljust")] - fn ljust( - self, - width: PyIntRef, - fillbyte: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - Ok(vm.ctx.new_bytes(self.inner.ljust(width, fillbyte, vm)?)) + fn ljust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.ljust(options, vm)?)) } #[pymethod(name = "rjust")] - fn rjust( - self, - width: PyIntRef, - fillbyte: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - Ok(vm.ctx.new_bytes(self.inner.rjust(width, fillbyte, vm)?)) + fn rjust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.rjust(options, vm)?)) } #[pymethod(name = "count")] From 03997273ed630b97806ce804b3363d80cdcdb6d7 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 21:12:45 +0200 Subject: [PATCH 311/658] refactor index/find with ByteInnerOptions --- vm/src/obj/objbyteinner.rs | 21 ++------------------- vm/src/obj/objbytes.rs | 20 ++++---------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b6e9ae49e..6d5d49148 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -628,25 +628,8 @@ impl PyByteInner { Ok(vm.new_bool(suff.as_slice() == &self.elements.do_slice(range)[offset])) } - pub fn find( - &self, - sub: PyObjectRef, - start: OptionalArg, - end: OptionalArg, - vm: &VirtualMachine, - ) -> Result { - let sub = match try_as_bytes_like(&sub) { - Some(value) => value, - None => match_class!(sub, - i @ PyInt => vec![i.as_bigint().byte_or(vm)?], - obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}), - }; - - let range = self.elements.get_slice_range( - &is_valid_slice_arg(start, vm)?, - &is_valid_slice_arg(end, vm)?, - ); - + pub fn find(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let (sub, range) = options.get_value(&self.elements, vm)?; // not allowed for this method if range.end < range.start { return Ok(-1isize); diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 3c1b323c8..8ecc9dac1 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -270,25 +270,13 @@ impl PyBytesRef { } #[pymethod(name = "find")] - fn find( - self, - sub: PyObjectRef, - start: OptionalArg, - end: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - Ok(vm.new_int(self.inner.find(sub, start, end, vm)?)) + fn find(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.find(options, vm) } #[pymethod(name = "index")] - fn index( - self, - sub: PyObjectRef, - start: OptionalArg, - end: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let res = self.inner.find(sub, start, end, vm)?; + fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.find(options, vm)?; if res == -1 { return Err(vm.new_value_error("substring not found".to_string())); } From a42bfae84d07f5743480af26ab8814d486513b5d Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 21:41:46 +0200 Subject: [PATCH 312/658] kwargs for translate --- tests/snippets/bytes.py | 1 + vm/src/obj/objbyteinner.rs | 81 ++++++++++++++++++++++---------------- vm/src/obj/objbytes.rs | 12 ++---- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 0a3fb11ab..99709e768 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -324,3 +324,4 @@ assert ( ) assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab")) == b"abatuybyubuyb" assert b"hjhtuyfjtyhuhjuyj".translate(None, b"ht") == b"juyfjyujuyj" +assert b"hjhtuyfjtyhuhjuyj".translate(None, delete = b"ht") == b"juyfjyujuyj" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 6d5d49148..19a40f0a1 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -154,9 +154,8 @@ pub struct ByteInnerPaddingOptions { #[pyarg(positional_only, optional = true)] fillbyte: OptionalArg, } - impl ByteInnerPaddingOptions { - fn get_value(&self, fn_name: &str, len: usize, vm: &VirtualMachine) -> PyResult<(u8, usize)> { + fn get_value(self, fn_name: &str, len: usize, vm: &VirtualMachine) -> PyResult<(u8, usize)> { let fillbyte = if let OptionalArg::Present(v) = &self.fillbyte { match try_as_byte(&v) { Some(x) => { @@ -197,6 +196,49 @@ impl ByteInnerPaddingOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerTranslateOptions { + #[pyarg(positional_only, optional = false)] + table: PyObjectRef, + #[pyarg(positional_or_keyword, optional = true)] + delete: OptionalArg, +} + +impl ByteInnerTranslateOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec, Vec)> { + let table = match try_as_bytes_like(&self.table) { + Some(value) => value, + None => match_class!(self.table, + + _n @ PyNone => (0..=255).collect::>(), + obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, + ), + }; + + if table.len() != 256 { + return Err( + vm.new_value_error("translation table must be 256 characters long".to_string()) + ); + } + + let delete = if let OptionalArg::Present(value) = &self.delete { + match try_as_bytes_like(&value) { + Some(value) => value, + None => { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + value + ))); + } + } + } else { + vec![] + }; + + Ok((table, delete)) + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -677,40 +719,9 @@ impl PyByteInner { Ok(vm.ctx.new_bytes(res)) } - pub fn translate( - &self, - table: PyObjectRef, - delete: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let table = match try_as_bytes_like(&table) { - Some(value) => value, - None => match_class!(table, - _n @ PyNone => (0..=255).collect::>(), - obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, - ), - }; - - if table.len() != 256 { - return Err( - vm.new_value_error("translation table must be 256 characters long".to_string()) - ); - } - - let delete = if let OptionalArg::Present(value) = delete { - match try_as_bytes_like(&value) { - Some(value) => value, - None => { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - value - ))); - } - } - } else { - vec![] - }; + pub fn translate(&self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + let (table, delete) = options.get_value(vm)?; let mut res = vec![]; diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8ecc9dac1..c3627c1d7 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,8 @@ use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, PyByteInner, + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerTranslateOptions, + PyByteInner, }; use super::objiter; @@ -284,13 +285,8 @@ impl PyBytesRef { } #[pymethod(name = "translate")] - fn translate( - self, - table: PyObjectRef, - delete: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - self.inner.translate(table, delete, vm) + fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + self.inner.translate(options, vm) } } From 8a2ce9e260a3949584c7d329f44f6ada322b4982 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 21:58:15 +0200 Subject: [PATCH 313/658] add swapcase --- tests/snippets/bytes.py | 4 +++- vm/src/obj/objbyteinner.rs | 12 ++++++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 99709e768..3b3dcb457 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -121,7 +121,7 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower, capitalize +# upper lower, capitalize, swapcase l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() @@ -129,6 +129,8 @@ assert b.upper().isupper() assert l.capitalize() == b"Lower" assert b.capitalize() == b"Upper" assert bytes().capitalize() == bytes() +assert b"AaBbCc123'@/".swapcase().swapcase() == b"AaBbCc123'@/" +assert b"AaBbCc123'@/".swapcase() == b"aAbBcC123'@/" # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 19a40f0a1..9b4ec8bea 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -474,6 +474,18 @@ impl PyByteInner { new } + pub fn swapcase(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::new(); + for w in &self.elements { + match w { + 65..=90 => new.push(w.to_ascii_lowercase()), + 97..=122 => new.push(w.to_ascii_uppercase()), + x => new.push(*x), + } + } + new + } + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { let bla = self .elements diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index c3627c1d7..964675f57 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -214,6 +214,11 @@ impl PyBytesRef { Ok(vm.ctx.new_bytes(self.inner.capitalize(vm))) } + #[pymethod(name = "swapcase")] + fn swapcase(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.swapcase(vm))) + } + #[pymethod(name = "hex")] fn hex(self, vm: &VirtualMachine) -> PyResult { self.inner.hex(vm) From 9ea823abf148fd8c8ccf6d77fb9236c536bde83d Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 18 Apr 2019 23:06:11 +0200 Subject: [PATCH 314/658] add rfind, rindex --- tests/snippets/bytes.py | 8 ++++++++ vm/src/obj/objbyteinner.rs | 27 +++++++++++++++++++++------ vm/src/obj/objbytes.rs | 18 ++++++++++++++++-- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 3b3dcb457..380a19386 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -310,6 +310,14 @@ assert b"abcd".find(b"b", 3, 4) == -1 assert b"abcd".find(1) == -1 assert b"abcd".find(99) == 2 +assert b"abcdabcda".find(b"a") == 0 +assert b"abcdabcda".rfind(b"a") == 8 +assert b"abcdabcda".rfind(b"a", 2, 6) == 4 +assert b"abcdabcda".rfind(b"a", None, 6) == 4 +assert b"abcdabcda".rfind(b"a", 2, None) == 8 +assert b"abcdabcda".index(b"a") == 0 +assert b"abcdabcda".rindex(b"a") == 8 + # make trans # fmt: off diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 9b4ec8bea..050553ea8 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -682,7 +682,12 @@ impl PyByteInner { Ok(vm.new_bool(suff.as_slice() == &self.elements.do_slice(range)[offset])) } - pub fn find(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + pub fn find( + &self, + options: ByteInnerFindOptions, + reverse: bool, + vm: &VirtualMachine, + ) -> PyResult { let (sub, range) = options.get_value(&self.elements, vm)?; // not allowed for this method if range.end < range.start { @@ -690,13 +695,23 @@ impl PyByteInner { } let start = range.start; + let end = range.end; - let slice = &self.elements[range]; - for (n, _) in slice.iter().enumerate() { - if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { - return Ok((start + n) as isize); + if reverse { + let slice = self.elements.do_slice_reverse(range); + for (n, _) in slice.iter().enumerate() { + if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { + return Ok((end - n - 1) as isize); + } } - } + } else { + let slice = self.elements.do_slice(range); + for (n, _) in slice.iter().enumerate() { + if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { + return Ok((start + n) as isize); + } + } + }; Ok(-1isize) } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 964675f57..ba23751f7 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -277,12 +277,26 @@ impl PyBytesRef { #[pymethod(name = "find")] fn find(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - self.inner.find(options, vm) + self.inner.find(options, false, vm) } #[pymethod(name = "index")] fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - let res = self.inner.find(options, vm)?; + let res = self.inner.find(options, false, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(vm.new_int(res)) + } + + #[pymethod(name = "rfind")] + fn rfind(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.find(options, true, vm) + } + + #[pymethod(name = "rindex")] + fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.find(options, true, vm)?; if res == -1 { return Err(vm.new_value_error("substring not found".to_string())); } From 61ee15b97b6999484f77da677f517f5114870eb9 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Fri, 19 Apr 2019 00:04:39 +0200 Subject: [PATCH 315/658] add strip lstrip rstrip --- tests/snippets/bytes.py | 30 +++++++++++++++-------- vm/src/obj/objbyteinner.rs | 49 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 25 +++++++++++++++++-- 3 files changed, 92 insertions(+), 12 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 380a19386..2733533d8 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -219,15 +219,16 @@ assert [b"koki".rjust(i, b"|") for i in range(3, 10)] == [ b"||||koki", b"|||||koki", ] -assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [b'kok', - b'kok', - b'|kok', - b'||kok', - b'|||kok', - b'||||kok', - b'|||||kok', - b'||||||kok'] - +assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"|kok", + b"||kok", + b"|||kok", + b"||||kok", + b"|||||kok", + b"||||||kok", +] b"kok".rjust(4) == b" kok" # " test no arg" @@ -334,4 +335,13 @@ assert ( ) assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab")) == b"abatuybyubuyb" assert b"hjhtuyfjtyhuhjuyj".translate(None, b"ht") == b"juyfjyujuyj" -assert b"hjhtuyfjtyhuhjuyj".translate(None, delete = b"ht") == b"juyfjyujuyj" +assert b"hjhtuyfjtyhuhjuyj".translate(None, delete=b"ht") == b"juyfjyujuyj" + + +# strip lstrip rstrip +assert b" spacious ".strip() == b"spacious" +assert b"www.example.com".strip(b"cmowz.") == b"example" +assert b" spacious ".lstrip() == b"spacious " +assert b"www.example.com".lstrip(b"cmowz.") == b"example.com" +assert b" spacious ".rstrip() == b" spacious" +assert b"mississippi".rstrip(b"ipz") == b"mississ" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 050553ea8..46f55ec8d 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -760,6 +760,49 @@ impl PyByteInner { Ok(vm.ctx.new_bytes(res)) } + + pub fn strip( + &self, + chars: OptionalArg, + position: ByteInnerPosition, + vm: &VirtualMachine, + ) -> PyResult> { + let chars = if let OptionalArg::Present(content) = chars { + match try_as_bytes_like(&content) { + Some(value) => value, + None => { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + content + ))); + } + } + } else { + vec![b' '] + }; + + let mut start = 0; + let mut end = self.len(); + + if let ByteInnerPosition::Left | ByteInnerPosition::All = position { + for (n, i) in self.elements.iter().enumerate() { + if !chars.contains(i) { + start = n; + break; + } + } + } + + if let ByteInnerPosition::Right | ByteInnerPosition::All = position { + for (n, i) in self.elements.iter().rev().enumerate() { + if !chars.contains(i) { + end = self.len() - n; + break; + } + } + } + Ok(self.elements[start..end].to_vec()) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { @@ -789,3 +832,9 @@ pub trait ByteOr: ToPrimitive { } impl ByteOr for BigInt {} + +pub enum ByteInnerPosition { + Left, + Right, + All, +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index ba23751f7..bd483be18 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,8 +7,8 @@ use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerTranslateOptions, - PyByteInner, + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, + ByteInnerTranslateOptions, PyByteInner, }; use super::objiter; @@ -307,6 +307,27 @@ impl PyBytesRef { fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { self.inner.translate(options, vm) } + + #[pymethod(name = "strip")] + fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::All, vm)?)) + } + + #[pymethod(name = "lstrip")] + fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::Left, vm)?)) + } + + #[pymethod(name = "rstrip")] + fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::Right, vm)?)) + } } #[derive(Debug)] From a75e7635d5b39e17d8d3158a04bbcc9807a0eded Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 12:09:11 +0300 Subject: [PATCH 316/658] Add os.sep --- Lib/os.py | 5 +++++ tests/snippets/stdlib_os.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Lib/os.py b/Lib/os.py index 8c19a89c6..3cadb5f61 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,5 +1,10 @@ from _os import * +if name == 'nt': + sep = '\\' +else: + sep = '/' + # Change environ to automatically call putenv(), unsetenv if they exist. from _collections_abc import MutableMapping diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 7b9981bdf..6552596d7 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -58,3 +58,8 @@ if os.name == "posix": os.putenv(ENV_KEY, ENV_VALUE) os.unsetenv(ENV_KEY) assert os.getenv(ENV_KEY) == None + +if os.name == "nt": + assert os.sep == "\\" +else: + assert os.sep == "/" From abb9cde62c5e416954368ae1844158faf0c6d393 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 12:23:45 +0300 Subject: [PATCH 317/658] Use temp dir for os test --- tests/snippets/stdlib_os.py | 64 +++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 6552596d7..476627b38 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -7,33 +7,6 @@ assert fd > 0 os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) - -FNAME = "test_file_that_no_one_will_have_on_disk" -CONTENT = b"testing" -CONTENT2 = b"rustpython" -CONTENT3 = b"BOYA" - -class TestWithFile(): - def __enter__(self): - open(FNAME, "wb") - return FNAME - - def __exit__(self, exc_type, exc_val, exc_tb): - os.remove(FNAME) - - -with TestWithFile() as fname: - fd = os.open(fname, 1) - assert os.write(fd, CONTENT2) == len(CONTENT2) - assert os.write(fd, CONTENT3) == len(CONTENT3) - os.close(fd) - - fd = os.open(fname, 0) - assert os.read(fd, len(CONTENT2)) == CONTENT2 - assert os.read(fd, len(CONTENT3)) == CONTENT3 - os.close(fd) - - assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) @@ -59,7 +32,44 @@ if os.name == "posix": os.unsetenv(ENV_KEY) assert os.getenv(ENV_KEY) == None + if os.name == "nt": assert os.sep == "\\" else: assert os.sep == "/" + +class TestWithTempDir(): + def __enter__(self): + if os.name == "nt": + base_folder = os.environ["%TEMP%"] + else: + base_folder = "/tmp" + name = base_folder + os.sep + "test_os" + os.mkdir(name) + self.name = name + return name + + def __exit__(self, exc_type, exc_val, exc_tb): + for f in os.listdir(self.name): + # Currently don't support dir delete. + os.remove(self.name + os.sep + f) + os.rmdir(self.name) + + +FILE_NAME = "test1" +CONTENT = b"testing" +CONTENT2 = b"rustpython" +CONTENT3 = b"BOYA" + +with TestWithTempDir() as tmpdir: + fname = tmpdir + os.sep + FILE_NAME + open(fname, "wb") + fd = os.open(fname, 1) + assert os.write(fd, CONTENT2) == len(CONTENT2) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) + + fd = os.open(fname, 0) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) From 00e46b7d723e7988fbdd20eeecdb98762f26b098 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 13:50:34 +0300 Subject: [PATCH 318/658] Try to fix windows test --- tests/snippets/stdlib_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 476627b38..e812353a5 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -41,7 +41,7 @@ else: class TestWithTempDir(): def __enter__(self): if os.name == "nt": - base_folder = os.environ["%TEMP%"] + base_folder = os.environ["TEMP"] else: base_folder = "/tmp" name = base_folder + os.sep + "test_os" From 08f2a5705bf59d413647bfbd1b50fbbf5f03dc9b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 14:40:01 +0300 Subject: [PATCH 319/658] Add prints for windows debug --- tests/snippets/stdlib_os.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index e812353a5..f7c47a95c 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -51,8 +51,10 @@ class TestWithTempDir(): def __exit__(self, exc_type, exc_val, exc_tb): for f in os.listdir(self.name): + print("Remove file", self.name + os.sep + f) # Currently don't support dir delete. os.remove(self.name + os.sep + f) + print("Remove dir", self.name) os.rmdir(self.name) From cf0a9207438c54948988830ef4a96a6ab05bfff3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 15:50:06 +0300 Subject: [PATCH 320/658] Add socket.fileno for unix --- tests/snippets/stdlib_socket.py | 13 +++++++++++++ vm/src/stdlib/socket.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py index 79efaa363..d7afe1bd3 100644 --- a/tests/snippets/stdlib_socket.py +++ b/tests/snippets/stdlib_socket.py @@ -1,4 +1,5 @@ import socket +import os from testutils import assertRaises MESSAGE_A = b'aaaa' @@ -21,6 +22,18 @@ recv_a = connection.recv(len(MESSAGE_A)) recv_b = connector.recv(len(MESSAGE_B)) assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B + +# fileno +if os.name == "posix": + connector_fd = connector.fileno() + connection_fd = connection.fileno() + os.write(connector_fd, MESSAGE_A) + connection.send(MESSAGE_B) + recv_a = connection.recv(len(MESSAGE_A)) + recv_b = os.read(connector_fd, (len(MESSAGE_B))) + assert recv_a == MESSAGE_A + assert recv_b == MESSAGE_B + connection.close() connector.close() listener.close() diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index e893e0dab..a7a87d554 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -111,6 +111,19 @@ impl Write for Connection { } } +#[cfg(unix)] +use std::os::unix::io::{AsRawFd, RawFd}; +#[cfg(unix)] +impl AsRawFd for Connection { + fn as_raw_fd(&self) -> RawFd { + match self { + Connection::TcpListener(con) => con.as_raw_fd(), + Connection::UdpSocket(con) => con.as_raw_fd(), + Connection::TcpStream(con) => con.as_raw_fd(), + } + } +} + #[derive(Debug)] pub struct Socket { address_family: AddressFamily, @@ -387,6 +400,25 @@ fn socket_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[cfg(unix)] +fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + use std::os::unix::io::AsRawFd; + arg_check!(vm, args, required = [(zelf, None)]); + + let socket = get_socket(zelf); + + let fileno = match socket.con.borrow_mut().as_mut() { + Some(v) => v.as_raw_fd(), + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_int(i64::from(fileno))) +} + +#[cfg(all(not(unix)))] +fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + unimplemented!(); +} + fn socket_getsockname(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(zelf, None)]); let socket = get_socket(zelf); @@ -424,6 +456,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "getsockname" => ctx.new_rustfunc(socket_getsockname), "sendto" => ctx.new_rustfunc(socket_sendto), "recvfrom" => ctx.new_rustfunc(socket_recvfrom), + "fileno" => ctx.new_rustfunc(socket_fileno), }); py_module!(vm, "socket", { From e7b059ed9c7c4b961dfce3614a0a135f72305f5a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 16:00:00 +0300 Subject: [PATCH 321/658] Add socket.fileno for windows --- tests/snippets/stdlib_socket.py | 17 ++++++++--------- vm/src/stdlib/socket.rs | 29 ++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py index d7afe1bd3..16c2f96ce 100644 --- a/tests/snippets/stdlib_socket.py +++ b/tests/snippets/stdlib_socket.py @@ -24,15 +24,14 @@ assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B # fileno -if os.name == "posix": - connector_fd = connector.fileno() - connection_fd = connection.fileno() - os.write(connector_fd, MESSAGE_A) - connection.send(MESSAGE_B) - recv_a = connection.recv(len(MESSAGE_A)) - recv_b = os.read(connector_fd, (len(MESSAGE_B))) - assert recv_a == MESSAGE_A - assert recv_b == MESSAGE_B +connector_fd = connector.fileno() +connection_fd = connection.fileno() +os.write(connector_fd, MESSAGE_A) +connection.send(MESSAGE_B) +recv_a = connection.recv(len(MESSAGE_A)) +recv_b = os.read(connector_fd, (len(MESSAGE_B))) +assert recv_a == MESSAGE_A +assert recv_b == MESSAGE_B connection.close() connector.close() diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index a7a87d554..6e65ddfd9 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -124,6 +124,19 @@ impl AsRawFd for Connection { } } +#[cfg(windows)] +use std::os::windows::io::{AsRawSocket, RawSocket}; +#[cfg(windows)] +impl AsRawSocket for Connection { + fn as_raw_socket(&self) -> RawSocket { + match self { + Connection::TcpListener(con) => con.as_raw_socket(), + Connection::UdpSocket(con) => con.as_raw_socket(), + Connection::TcpStream(con) => con.as_raw_socket(), + } + } +} + #[derive(Debug)] pub struct Socket { address_family: AddressFamily, @@ -414,7 +427,21 @@ fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(i64::from(fileno))) } -#[cfg(all(not(unix)))] +#[cfg(windows)] +fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + use std::os::windows::io::AsRawSocket; + arg_check!(vm, args, required = [(zelf, None)]); + + let socket = get_socket(zelf); + + let fileno = match socket.con.borrow_mut().as_mut() { + Some(v) => v.as_raw_socket(), + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_int(i64::from(fileno))) +} + +#[cfg(all(not(unix), not(windows)))] fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { unimplemented!(); } From 2bc72dae9ff03f4afdf357d31e933d04f2db5ae3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 16:10:55 +0300 Subject: [PATCH 322/658] Move fileno logic to Connection impl --- vm/src/stdlib/socket.rs | 78 +++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 6e65ddfd9..9d7a703e1 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -86,6 +86,33 @@ impl Connection { _ => Err(io::Error::new(io::ErrorKind::Other, "oh no!")), } } + + #[cfg(unix)] + fn fileno(&self) -> i64 { + use std::os::unix::io::AsRawFd; + let raw_fd = match self { + Connection::TcpListener(con) => con.as_raw_fd(), + Connection::UdpSocket(con) => con.as_raw_fd(), + Connection::TcpStream(con) => con.as_raw_fd(), + }; + raw_fd as i64 + } + + #[cfg(windows)] + fn fileno(&self) -> i64 { + use std::os::windows::io::AsRawSocket; + let raw_fd = match self { + Connection::TcpListener(con) => con.as_raw_socket(), + Connection::UdpSocket(con) => con.as_raw_socket(), + Connection::TcpStream(con) => con.as_raw_socket(), + }; + raw_fd as i64 + } + + #[cfg(all(not(unix), not(windows)))] + fn fileno(&self) -> i64 { + unimplemented!(); + } } impl Read for Connection { @@ -111,32 +138,6 @@ impl Write for Connection { } } -#[cfg(unix)] -use std::os::unix::io::{AsRawFd, RawFd}; -#[cfg(unix)] -impl AsRawFd for Connection { - fn as_raw_fd(&self) -> RawFd { - match self { - Connection::TcpListener(con) => con.as_raw_fd(), - Connection::UdpSocket(con) => con.as_raw_fd(), - Connection::TcpStream(con) => con.as_raw_fd(), - } - } -} - -#[cfg(windows)] -use std::os::windows::io::{AsRawSocket, RawSocket}; -#[cfg(windows)] -impl AsRawSocket for Connection { - fn as_raw_socket(&self) -> RawSocket { - match self { - Connection::TcpListener(con) => con.as_raw_socket(), - Connection::UdpSocket(con) => con.as_raw_socket(), - Connection::TcpStream(con) => con.as_raw_socket(), - } - } -} - #[derive(Debug)] pub struct Socket { address_family: AddressFamily, @@ -413,37 +414,16 @@ fn socket_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -#[cfg(unix)] fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - use std::os::unix::io::AsRawFd; arg_check!(vm, args, required = [(zelf, None)]); let socket = get_socket(zelf); let fileno = match socket.con.borrow_mut().as_mut() { - Some(v) => v.as_raw_fd(), + Some(v) => v.fileno(), None => return Err(vm.new_type_error("".to_string())), }; - Ok(vm.ctx.new_int(i64::from(fileno))) -} - -#[cfg(windows)] -fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - use std::os::windows::io::AsRawSocket; - arg_check!(vm, args, required = [(zelf, None)]); - - let socket = get_socket(zelf); - - let fileno = match socket.con.borrow_mut().as_mut() { - Some(v) => v.as_raw_socket(), - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.ctx.new_int(i64::from(fileno))) -} - -#[cfg(all(not(unix), not(windows)))] -fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - unimplemented!(); + Ok(vm.ctx.new_int(fileno)) } fn socket_getsockname(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { From aef973802c0cab214a179c2d8166707aaed869c9 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 16:20:44 +0300 Subject: [PATCH 323/658] Close open reference to test file --- tests/snippets/stdlib_os.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index f7c47a95c..900b2fa5b 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -51,10 +51,8 @@ class TestWithTempDir(): def __exit__(self, exc_type, exc_val, exc_tb): for f in os.listdir(self.name): - print("Remove file", self.name + os.sep + f) # Currently don't support dir delete. os.remove(self.name + os.sep + f) - print("Remove dir", self.name) os.rmdir(self.name) @@ -65,7 +63,8 @@ CONTENT3 = b"BOYA" with TestWithTempDir() as tmpdir: fname = tmpdir + os.sep + FILE_NAME - open(fname, "wb") + with open(fname, "wb"): + pass fd = os.open(fname, 1) assert os.write(fd, CONTENT2) == len(CONTENT2) assert os.write(fd, CONTENT3) == len(CONTENT3) From f42c2a6ba527901eadc1429a5a3a7008d34395e9 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 16:58:43 +0300 Subject: [PATCH 324/658] Test fileno only on posix --- tests/snippets/stdlib_socket.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py index 16c2f96ce..d7afe1bd3 100644 --- a/tests/snippets/stdlib_socket.py +++ b/tests/snippets/stdlib_socket.py @@ -24,14 +24,15 @@ assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B # fileno -connector_fd = connector.fileno() -connection_fd = connection.fileno() -os.write(connector_fd, MESSAGE_A) -connection.send(MESSAGE_B) -recv_a = connection.recv(len(MESSAGE_A)) -recv_b = os.read(connector_fd, (len(MESSAGE_B))) -assert recv_a == MESSAGE_A -assert recv_b == MESSAGE_B +if os.name == "posix": + connector_fd = connector.fileno() + connection_fd = connection.fileno() + os.write(connector_fd, MESSAGE_A) + connection.send(MESSAGE_B) + recv_a = connection.recv(len(MESSAGE_A)) + recv_b = os.read(connector_fd, (len(MESSAGE_B))) + assert recv_a == MESSAGE_A + assert recv_b == MESSAGE_B connection.close() connector.close() From 8215ec0a7c01896ecfa3bc7ab990aec22072b8b0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 17:04:04 +0300 Subject: [PATCH 325/658] Don't delete test dir --- tests/snippets/stdlib_os.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 900b2fa5b..f7d0a9a1f 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -1,4 +1,5 @@ -import os +import os +import time from testutils import assert_raises @@ -44,16 +45,14 @@ class TestWithTempDir(): base_folder = os.environ["TEMP"] else: base_folder = "/tmp" - name = base_folder + os.sep + "test_os" + name = base_folder + os.sep + "test_os_" + str(int(time.time())) os.mkdir(name) self.name = name return name def __exit__(self, exc_type, exc_val, exc_tb): - for f in os.listdir(self.name): - # Currently don't support dir delete. - os.remove(self.name + os.sep + f) - os.rmdir(self.name) + # TODO: Delete temp dir + pass FILE_NAME = "test1" From d704a0fcdab4d06c4b11da64b815918a56976417 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 19 Apr 2019 17:14:14 +0300 Subject: [PATCH 326/658] Rename os test folder --- tests/snippets/stdlib_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index f7d0a9a1f..4d64fc793 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -45,7 +45,7 @@ class TestWithTempDir(): base_folder = os.environ["TEMP"] else: base_folder = "/tmp" - name = base_folder + os.sep + "test_os_" + str(int(time.time())) + name = base_folder + os.sep + "rustpython_test_os_" + str(int(time.time())) os.mkdir(name) self.name = name return name From 2a74e48d98a5fef053cf82cd267eb4832bd67090 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 10:10:56 -0500 Subject: [PATCH 327/658] Add vm.is_callable --- vm/src/builtins.rs | 4 ++-- vm/src/vm.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index d054c34fc..9f393d333 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -67,8 +67,8 @@ fn builtin_bin(x: PyIntRef, _vm: &VirtualMachine) -> String { // builtin_breakpoint -fn builtin_callable(obj: PyObjectRef, _vm: &VirtualMachine) -> bool { - objtype::class_has_attr(&obj.class(), "__call__") +fn builtin_callable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { + vm.is_callable(&obj) } fn builtin_chr(i: u32, _vm: &VirtualMachine) -> String { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 6e7710516..81aa5367b 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -666,6 +666,21 @@ impl VirtualMachine { crate::stdlib::json::de_pyobject(self, s) } + pub fn is_callable(&self, obj: &PyObjectRef) -> bool { + match_class!(obj, + PyFunction => true, + PyMethod => true, + PyBuiltinFunction => true, + obj => { + if let Some(dict) = &obj.dict { + dict.contains_key("__call__", self) + } else { + false + } + }, + ) + } + pub fn _sub(&self, a: PyObjectRef, b: PyObjectRef) -> PyResult { self.call_or_reflection(a, b, "__sub__", "__rsub__", |vm, a, b| { Err(vm.new_unsupported_operand_error(a, b, "-")) From a6f3b9a5ff2ae0e56e7bd9db5a07a2a3b6178d29 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 10:14:57 -0500 Subject: [PATCH 328/658] Add PyCallable --- vm/src/pyobject.rs | 33 +++++++++++++++++++++++++++++++++ vm/src/vm.rs | 33 ++++++++++++++++++--------------- wasm/lib/src/browser_module.rs | 15 ++++++++------- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index fdac31b83..d172961a3 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -878,6 +878,39 @@ where } } +#[derive(Clone)] +pub struct PyCallable { + obj: PyObjectRef, +} + +impl PyCallable { + #[inline] + pub fn invoke(&self, args: impl Into, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.obj.clone(), args) + } + + #[inline] + pub fn into_object(self) -> PyObjectRef { + self.obj + } +} + +impl TryFromObject for PyCallable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if vm.is_callable(&obj) { + Ok(PyCallable { obj }) + } else { + Err(vm.new_type_error(format!("'{}' object is not callable", obj.class().name))) + } + } +} + +impl IntoPyObject for PyCallable { + fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { + Ok(self.into_object()) + } +} + pub trait IdProtocol { fn get_id(&self) -> usize; fn is(&self, other: &T) -> bool diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 81aa5367b..3a1aec6fc 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -341,11 +341,7 @@ impl VirtualMachine { } } - pub fn invoke(&self, func_ref: PyObjectRef, args: T) -> PyResult - where - T: Into, - { - let args = args.into(); + fn _invoke(&self, func_ref: PyObjectRef, args: PyFuncArgs) -> PyResult { trace!("Invoke: {:?} {:?}", func_ref, args); if let Some(PyFunction { ref code, @@ -354,22 +350,29 @@ impl VirtualMachine { ref kw_only_defaults, }) = func_ref.payload() { - return self.invoke_python_function(code, scope, defaults, kw_only_defaults, args); - } - if let Some(PyMethod { + self.invoke_python_function(code, scope, defaults, kw_only_defaults, args) + } else if let Some(PyMethod { ref function, ref object, }) = func_ref.payload() { - return self.invoke(function.clone(), args.insert(object.clone())); - } - if let Some(PyBuiltinFunction { ref value }) = func_ref.payload() { - return value(self, args); + self.invoke(function.clone(), args.insert(object.clone())) + } else if let Some(PyBuiltinFunction { ref value }) = func_ref.payload() { + value(self, args) + } else { + // TODO: is it safe to just invoke __call__ otherwise? + trace!("invoke __call__ for: {:?}", &func_ref.payload); + self.call_method(&func_ref, "__call__", args) } + } - // TODO: is it safe to just invoke __call__ otherwise? - trace!("invoke __call__ for: {:?}", &func_ref.payload); - self.call_method(&func_ref, "__call__", args) + // TODO: make func_ref an &PyObjectRef + #[inline] + pub fn invoke(&self, func_ref: PyObjectRef, args: T) -> PyResult + where + T: Into, + { + self._invoke(func_ref, args.into()) } fn invoke_python_function( diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 478ebb9c5..6fb7c6baa 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,10 +7,11 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ - objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, - objtype::PyClassRef, + objdict::PyDictRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, +}; +use rustpython_vm::pyobject::{ + PyCallable, PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue, }; -use rustpython_vm::pyobject::{PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; @@ -113,7 +114,7 @@ fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyRe Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_request_animation_frame(func: PyFunctionRef, vm: &VirtualMachine) -> PyResult { +fn browser_request_animation_frame(func: PyCallable, vm: &VirtualMachine) -> PyResult { use std::{cell::RefCell, rc::Rc}; // this basic setup for request_animation_frame taken from: @@ -192,8 +193,8 @@ impl PyPromise { #[pymethod] fn then( &self, - on_fulfill: PyFunctionRef, - on_reject: OptionalArg, + on_fulfill: PyCallable, + on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { let weak_vm = weak_vm(vm); @@ -224,7 +225,7 @@ impl PyPromise { } #[pymethod] - fn catch(&self, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + fn catch(&self, on_reject: PyCallable, vm: &VirtualMachine) -> PyPromiseRef { let weak_vm = weak_vm(vm); let ret_future = JsFuture::from(self.value.clone()).then(move |res| { From 29ea307f4c633b61f727689c535fdf33ddc044be Mon Sep 17 00:00:00 2001 From: Joey Date: Fri, 19 Apr 2019 09:26:24 -0700 Subject: [PATCH 329/658] unicode-casing: specify rev explicitly, add todo comment --- Cargo.lock | 6 +++--- vm/Cargo.toml | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 100f6a359..274daba69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -827,7 +827,7 @@ dependencies = [ "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing)", + "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1106,7 +1106,7 @@ dependencies = [ [[package]] name = "unicode-casing" version = "0.1.0" -source = "git+https://github.com/OddCoincidence/unicode-casing#90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" +source = "git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb#90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" [[package]] name = "unicode-normalization" @@ -1437,7 +1437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" "checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" "checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" -"checksum unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing)" = "" +"checksum unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)" = "" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1ba1986b7..5af648084 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -23,8 +23,12 @@ regex = "1" rustc_version_runtime = "0.1.*" statrs = "0.10.0" caseless = "0.2.1" -unicode-casing = { git = "https://github.com/OddCoincidence/unicode-casing" } unicode-segmentation = "1.2.1" lazy_static = "^1.0.1" lexical = "2.0.0" itertools = "^0.8.0" + +# TODO: release and publish to crates.io +[dependencies.unicode-casing] +git = "https://github.com/OddCoincidence/unicode-casing" +rev = "90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" From cf5e106748b6eb11ff54e567f0bfa6dd1590a140 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 11:43:24 -0500 Subject: [PATCH 330/658] Fix is_callable --- vm/src/vm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 3a1aec6fc..3538cde30 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -678,7 +678,7 @@ impl VirtualMachine { if let Some(dict) = &obj.dict { dict.contains_key("__call__", self) } else { - false + objtype::class_has_attr(&obj.class(), "__call__") } }, ) From ac3cfca0fa1a4349257eac547318607da9021af3 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 12:04:22 -0500 Subject: [PATCH 331/658] Fix callable (again) --- vm/src/vm.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 3538cde30..ab15e14e9 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -674,13 +674,7 @@ impl VirtualMachine { PyFunction => true, PyMethod => true, PyBuiltinFunction => true, - obj => { - if let Some(dict) = &obj.dict { - dict.contains_key("__call__", self) - } else { - objtype::class_has_attr(&obj.class(), "__call__") - } - }, + obj => objtype::class_has_attr(&obj.class(), "__call__"), ) } From a984fd34d9cae5ba06f71b36184e318283371661 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 12:11:18 -0500 Subject: [PATCH 332/658] Update callable test --- tests/snippets/builtin_callable.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/snippets/builtin_callable.py b/tests/snippets/builtin_callable.py index c22db0779..db554df24 100644 --- a/tests/snippets/builtin_callable.py +++ b/tests/snippets/builtin_callable.py @@ -1,9 +1,8 @@ assert not callable(1) def f(): pass -# TODO uncomment when callable types get unified __call__ (or equivalent) -#assert callable(f) -#assert callable(len) -#assert callable(lambda: 1) +assert callable(f) +assert callable(len) +assert callable(lambda: 1) assert callable(int) class C: @@ -13,7 +12,7 @@ class C: def f(self): pass assert callable(C) assert not callable(C()) -#assert callable(C().f) +assert callable(C().f) class C: def __call__(self): pass From 7822f6d8d8ff12cedcaa3936645ddf20e501d728 Mon Sep 17 00:00:00 2001 From: luozijun Date: Thu, 18 Apr 2019 15:27:51 +0800 Subject: [PATCH 333/658] Add benches --- Cargo.lock | 22 + Cargo.toml | 8 + benchmarks/bench.rs | 125 ++ benchmarks/benchmarks/mandelbrot.py | 8 +- benchmarks/benchmarks/minidom.py | 1981 +++++++++++++++++++++++++++ benchmarks/benchmarks/nbody.py | 2 +- 6 files changed, 2142 insertions(+), 4 deletions(-) create mode 100644 benchmarks/bench.rs create mode 100644 benchmarks/benchmarks/minidom.py diff --git a/Cargo.lock b/Cargo.lock index 864e90ce5..2a329d025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,16 @@ name = "constant_time_eq" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cpython" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "diff" version = "0.1.11" @@ -551,6 +561,15 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "python3-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quick-error" version = "1.2.2" @@ -771,6 +790,7 @@ name = "rustpython" version = "0.0.1" dependencies = [ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_parser 0.0.1", @@ -1328,6 +1348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" +"checksum cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b489034e723e7f5109fecd19b719e664f89ef925be785885252469e9822fa940" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" "checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" @@ -1374,6 +1395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" +"checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" "checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" diff --git a/Cargo.toml b/Cargo.toml index c109e3fea..ef2838a1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,11 @@ edition = "2018" [workspace] members = [".", "derive", "vm", "wasm/lib", "parser"] +[[bench]] +name = "bench" +path = "./benchmarks/bench.rs" + + [dependencies] log="0.4.1" env_logger="0.5.10" @@ -15,3 +20,6 @@ rustpython_parser = {path = "parser"} rustpython_vm = {path = "vm"} rustyline = "2.1.0" xdg = "2.2.0" + +[dev-dependencies.cpython] +version = "0.2" diff --git a/benchmarks/bench.rs b/benchmarks/bench.rs new file mode 100644 index 000000000..b475da7cb --- /dev/null +++ b/benchmarks/bench.rs @@ -0,0 +1,125 @@ +#![feature(test)] + +extern crate cpython; +extern crate rustpython_parser; +extern crate rustpython_vm; +extern crate test; + +use rustpython_vm::pyobject::PyResult; +use rustpython_vm::{compile, VirtualMachine}; + +#[bench] +fn bench_tokenization(b: &mut test::Bencher) { + use rustpython_parser::lexer::{make_tokenizer, Tok}; + + let source = include_str!("./benchmarks/minidom.py"); + + b.bytes = source.len() as _; + b.iter(|| { + let lexer = make_tokenizer(source); + for res in lexer { + let _token: Tok = res.unwrap().1; + } + }) +} + +#[bench] +fn bench_rustpy_parse_to_ast(b: &mut test::Bencher) { + use rustpython_parser::parser::parse_program; + + let source = include_str!("./benchmarks/minidom.py"); + + b.bytes = source.len() as _; + b.iter(|| parse_program(source).unwrap()) +} + +#[bench] +fn bench_cpython_parse_to_ast(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/minidom.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = cpython::PyDict::new(python); + + locals.set_item(python, "SOURCE_CODE", source).unwrap(); + + let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")"; + + b.bytes = source.len() as _; + b.iter(|| { + let res: cpython::PyResult = python.eval(code, globals, Some(&locals)); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_cpython_nbody(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/nbody.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = None; + + b.iter(|| { + let res: cpython::PyResult<()> = python.run(source, globals, locals); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_cpython_mandelbrot(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/mandelbrot.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = None; + + b.iter(|| { + let res: cpython::PyResult<()> = python.run(source, globals, locals); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_rustpy_nbody(b: &mut test::Bencher) { + // NOTE: Take long time. + let source = include_str!("./benchmarks/nbody.py"); + + let vm = VirtualMachine::new(); + + let code = match compile::compile(&vm, source, &compile::Mode::Single, "".to_string()) { + Ok(code) => code, + Err(e) => panic!("{:?}", e), + }; + + b.iter(|| { + let scope = vm.ctx.new_scope(); + let res: PyResult = vm.run_code_obj(code.clone(), scope); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_rustpy_mandelbrot(b: &mut test::Bencher) { + // NOTE: Take long time. + let source = include_str!("./benchmarks/mandelbrot.py"); + + let vm = VirtualMachine::new(); + + let code = match compile::compile(&vm, source, &compile::Mode::Single, "".to_string()) { + Ok(code) => code, + Err(e) => panic!("{:?}", e), + }; + + b.iter(|| { + let scope = vm.ctx.new_scope(); + let res: PyResult = vm.run_code_obj(code.clone(), scope); + assert!(res.is_ok()); + }) +} diff --git a/benchmarks/benchmarks/mandelbrot.py b/benchmarks/benchmarks/mandelbrot.py index f0106cd74..88ae04101 100644 --- a/benchmarks/benchmarks/mandelbrot.py +++ b/benchmarks/benchmarks/mandelbrot.py @@ -21,11 +21,13 @@ while y < h: i = i+1 if Tr+Ti <= 4: - print('*', end='') + # print('*', end='') + pass else: - print('·', end='') + # print('·', end='') + pass x = x+1 - print() + # print() y = y+1 diff --git a/benchmarks/benchmarks/minidom.py b/benchmarks/benchmarks/minidom.py new file mode 100644 index 000000000..24957ea14 --- /dev/null +++ b/benchmarks/benchmarks/minidom.py @@ -0,0 +1,1981 @@ +"""Simple implementation of the Level 1 DOM. + +Namespaces and other minor Level 2 features are also supported. + +parse("foo.xml") + +parseString("") + +Todo: +===== + * convenience methods for getting elements and text. + * more testing + * bring some of the writer and linearizer code into conformance with this + interface + * SAX 2 namespaces +""" + +import io +import xml.dom + +from xml.dom import EMPTY_NAMESPACE, EMPTY_PREFIX, XMLNS_NAMESPACE, domreg +from xml.dom.minicompat import * +from xml.dom.xmlbuilder import DOMImplementationLS, DocumentLS + +# This is used by the ID-cache invalidation checks; the list isn't +# actually complete, since the nodes being checked will never be the +# DOCUMENT_NODE or DOCUMENT_FRAGMENT_NODE. (The node being checked is +# the node being added or removed, not the node being modified.) +# +_nodeTypes_with_children = (xml.dom.Node.ELEMENT_NODE, + xml.dom.Node.ENTITY_REFERENCE_NODE) + + +class Node(xml.dom.Node): + namespaceURI = None # this is non-null only for elements and attributes + parentNode = None + ownerDocument = None + nextSibling = None + previousSibling = None + + prefix = EMPTY_PREFIX # non-null only for NS elements and attributes + + def __bool__(self): + return True + + def toxml(self, encoding=None): + return self.toprettyxml("", "", encoding) + + def toprettyxml(self, indent="\t", newl="\n", encoding=None): + if encoding is None: + writer = io.StringIO() + else: + writer = io.TextIOWrapper(io.BytesIO(), + encoding=encoding, + errors="xmlcharrefreplace", + newline='\n') + if self.nodeType == Node.DOCUMENT_NODE: + # Can pass encoding only to document, to put it into XML header + self.writexml(writer, "", indent, newl, encoding) + else: + self.writexml(writer, "", indent, newl) + if encoding is None: + return writer.getvalue() + else: + return writer.detach().getvalue() + + def hasChildNodes(self): + return bool(self.childNodes) + + def _get_childNodes(self): + return self.childNodes + + def _get_firstChild(self): + if self.childNodes: + return self.childNodes[0] + + def _get_lastChild(self): + if self.childNodes: + return self.childNodes[-1] + + def insertBefore(self, newChild, refChild): + if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: + for c in tuple(newChild.childNodes): + self.insertBefore(c, refChild) + ### The DOM does not clearly specify what to return in this case + return newChild + if newChild.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(newChild), repr(self))) + if newChild.parentNode is not None: + newChild.parentNode.removeChild(newChild) + if refChild is None: + self.appendChild(newChild) + else: + try: + index = self.childNodes.index(refChild) + except ValueError: + raise xml.dom.NotFoundErr() + if newChild.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + self.childNodes.insert(index, newChild) + newChild.nextSibling = refChild + refChild.previousSibling = newChild + if index: + node = self.childNodes[index-1] + node.nextSibling = newChild + newChild.previousSibling = node + else: + newChild.previousSibling = None + newChild.parentNode = self + return newChild + + def appendChild(self, node): + if node.nodeType == self.DOCUMENT_FRAGMENT_NODE: + for c in tuple(node.childNodes): + self.appendChild(c) + ### The DOM does not clearly specify what to return in this case + return node + if node.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + elif node.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + if node.parentNode is not None: + node.parentNode.removeChild(node) + _append_child(self, node) + node.nextSibling = None + return node + + def replaceChild(self, newChild, oldChild): + if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: + refChild = oldChild.nextSibling + self.removeChild(oldChild) + return self.insertBefore(newChild, refChild) + if newChild.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(newChild), repr(self))) + if newChild is oldChild: + return + if newChild.parentNode is not None: + newChild.parentNode.removeChild(newChild) + try: + index = self.childNodes.index(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + self.childNodes[index] = newChild + newChild.parentNode = self + oldChild.parentNode = None + if (newChild.nodeType in _nodeTypes_with_children + or oldChild.nodeType in _nodeTypes_with_children): + _clear_id_cache(self) + newChild.nextSibling = oldChild.nextSibling + newChild.previousSibling = oldChild.previousSibling + oldChild.nextSibling = None + oldChild.previousSibling = None + if newChild.previousSibling: + newChild.previousSibling.nextSibling = newChild + if newChild.nextSibling: + newChild.nextSibling.previousSibling = newChild + return oldChild + + def removeChild(self, oldChild): + try: + self.childNodes.remove(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + if oldChild.nextSibling is not None: + oldChild.nextSibling.previousSibling = oldChild.previousSibling + if oldChild.previousSibling is not None: + oldChild.previousSibling.nextSibling = oldChild.nextSibling + oldChild.nextSibling = oldChild.previousSibling = None + if oldChild.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + + oldChild.parentNode = None + return oldChild + + def normalize(self): + L = [] + for child in self.childNodes: + if child.nodeType == Node.TEXT_NODE: + if not child.data: + # empty text node; discard + if L: + L[-1].nextSibling = child.nextSibling + if child.nextSibling: + child.nextSibling.previousSibling = child.previousSibling + child.unlink() + elif L and L[-1].nodeType == child.nodeType: + # collapse text node + node = L[-1] + node.data = node.data + child.data + node.nextSibling = child.nextSibling + if child.nextSibling: + child.nextSibling.previousSibling = node + child.unlink() + else: + L.append(child) + else: + L.append(child) + if child.nodeType == Node.ELEMENT_NODE: + child.normalize() + self.childNodes[:] = L + + def cloneNode(self, deep): + return _clone_node(self, deep, self.ownerDocument or self) + + def isSupported(self, feature, version): + return self.ownerDocument.implementation.hasFeature(feature, version) + + def _get_localName(self): + # Overridden in Element and Attr where localName can be Non-Null + return None + + # Node interfaces from Level 3 (WD 9 April 2002) + + def isSameNode(self, other): + return self is other + + def getInterface(self, feature): + if self.isSupported(feature, None): + return self + else: + return None + + # The "user data" functions use a dictionary that is only present + # if some user data has been set, so be careful not to assume it + # exists. + + def getUserData(self, key): + try: + return self._user_data[key][0] + except (AttributeError, KeyError): + return None + + def setUserData(self, key, data, handler): + old = None + try: + d = self._user_data + except AttributeError: + d = {} + self._user_data = d + if key in d: + old = d[key][0] + if data is None: + # ignore handlers passed for None + handler = None + if old is not None: + del d[key] + else: + d[key] = (data, handler) + return old + + def _call_user_data_handler(self, operation, src, dst): + if hasattr(self, "_user_data"): + for key, (data, handler) in list(self._user_data.items()): + if handler is not None: + handler.handle(operation, key, data, src, dst) + + # minidom-specific API: + + def unlink(self): + self.parentNode = self.ownerDocument = None + if self.childNodes: + for child in self.childNodes: + child.unlink() + self.childNodes = NodeList() + self.previousSibling = None + self.nextSibling = None + + # A Node is its own context manager, to ensure that an unlink() call occurs. + # This is similar to how a file object works. + def __enter__(self): + return self + + def __exit__(self, et, ev, tb): + self.unlink() + +defproperty(Node, "firstChild", doc="First child node, or None.") +defproperty(Node, "lastChild", doc="Last child node, or None.") +defproperty(Node, "localName", doc="Namespace-local name of this node.") + + +def _append_child(self, node): + # fast path with less checks; usable by DOM builders if careful + childNodes = self.childNodes + if childNodes: + last = childNodes[-1] + node.previousSibling = last + last.nextSibling = node + childNodes.append(node) + node.parentNode = self + +def _in_document(node): + # return True iff node is part of a document tree + while node is not None: + if node.nodeType == Node.DOCUMENT_NODE: + return True + node = node.parentNode + return False + +def _write_data(writer, data): + "Writes datachars to writer." + if data: + data = data.replace("&", "&").replace("<", "<"). \ + replace("\"", """).replace(">", ">") + writer.write(data) + +def _get_elements_by_tagName_helper(parent, name, rc): + for node in parent.childNodes: + if node.nodeType == Node.ELEMENT_NODE and \ + (name == "*" or node.tagName == name): + rc.append(node) + _get_elements_by_tagName_helper(node, name, rc) + return rc + +def _get_elements_by_tagName_ns_helper(parent, nsURI, localName, rc): + for node in parent.childNodes: + if node.nodeType == Node.ELEMENT_NODE: + if ((localName == "*" or node.localName == localName) and + (nsURI == "*" or node.namespaceURI == nsURI)): + rc.append(node) + _get_elements_by_tagName_ns_helper(node, nsURI, localName, rc) + return rc + +class DocumentFragment(Node): + nodeType = Node.DOCUMENT_FRAGMENT_NODE + nodeName = "#document-fragment" + nodeValue = None + attributes = None + parentNode = None + _child_node_types = (Node.ELEMENT_NODE, + Node.TEXT_NODE, + Node.CDATA_SECTION_NODE, + Node.ENTITY_REFERENCE_NODE, + Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, + Node.NOTATION_NODE) + + def __init__(self): + self.childNodes = NodeList() + + +class Attr(Node): + __slots__=('_name', '_value', 'namespaceURI', + '_prefix', 'childNodes', '_localName', 'ownerDocument', 'ownerElement') + nodeType = Node.ATTRIBUTE_NODE + attributes = None + specified = False + _is_id = False + + _child_node_types = (Node.TEXT_NODE, Node.ENTITY_REFERENCE_NODE) + + def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, + prefix=None): + self.ownerElement = None + self._name = qName + self.namespaceURI = namespaceURI + self._prefix = prefix + self.childNodes = NodeList() + + # Add the single child node that represents the value of the attr + self.childNodes.append(Text()) + + # nodeValue and value are set elsewhere + + def _get_localName(self): + try: + return self._localName + except AttributeError: + return self.nodeName.split(":", 1)[-1] + + def _get_specified(self): + return self.specified + + def _get_name(self): + return self._name + + def _set_name(self, value): + self._name = value + if self.ownerElement is not None: + _clear_id_cache(self.ownerElement) + + nodeName = name = property(_get_name, _set_name) + + def _get_value(self): + return self._value + + def _set_value(self, value): + self._value = value + self.childNodes[0].data = value + if self.ownerElement is not None: + _clear_id_cache(self.ownerElement) + self.childNodes[0].data = value + + nodeValue = value = property(_get_value, _set_value) + + def _get_prefix(self): + return self._prefix + + def _set_prefix(self, prefix): + nsuri = self.namespaceURI + if prefix == "xmlns": + if nsuri and nsuri != XMLNS_NAMESPACE: + raise xml.dom.NamespaceErr( + "illegal use of 'xmlns' prefix for the wrong namespace") + self._prefix = prefix + if prefix is None: + newName = self.localName + else: + newName = "%s:%s" % (prefix, self.localName) + if self.ownerElement: + _clear_id_cache(self.ownerElement) + self.name = newName + + prefix = property(_get_prefix, _set_prefix) + + def unlink(self): + # This implementation does not call the base implementation + # since most of that is not needed, and the expense of the + # method call is not warranted. We duplicate the removal of + # children, but that's all we needed from the base class. + elem = self.ownerElement + if elem is not None: + del elem._attrs[self.nodeName] + del elem._attrsNS[(self.namespaceURI, self.localName)] + if self._is_id: + self._is_id = False + elem._magic_id_nodes -= 1 + self.ownerDocument._magic_id_count -= 1 + for child in self.childNodes: + child.unlink() + del self.childNodes[:] + + def _get_isId(self): + if self._is_id: + return True + doc = self.ownerDocument + elem = self.ownerElement + if doc is None or elem is None: + return False + + info = doc._get_elem_info(elem) + if info is None: + return False + if self.namespaceURI: + return info.isIdNS(self.namespaceURI, self.localName) + else: + return info.isId(self.nodeName) + + def _get_schemaType(self): + doc = self.ownerDocument + elem = self.ownerElement + if doc is None or elem is None: + return _no_type + + info = doc._get_elem_info(elem) + if info is None: + return _no_type + if self.namespaceURI: + return info.getAttributeTypeNS(self.namespaceURI, self.localName) + else: + return info.getAttributeType(self.nodeName) + +defproperty(Attr, "isId", doc="True if this attribute is an ID.") +defproperty(Attr, "localName", doc="Namespace-local name of this attribute.") +defproperty(Attr, "schemaType", doc="Schema type for this attribute.") + + +class NamedNodeMap(object): + """The attribute list is a transient interface to the underlying + dictionaries. Mutations here will change the underlying element's + dictionary. + + Ordering is imposed artificially and does not reflect the order of + attributes as found in an input document. + """ + + __slots__ = ('_attrs', '_attrsNS', '_ownerElement') + + def __init__(self, attrs, attrsNS, ownerElement): + self._attrs = attrs + self._attrsNS = attrsNS + self._ownerElement = ownerElement + + def _get_length(self): + return len(self._attrs) + + def item(self, index): + try: + return self[list(self._attrs.keys())[index]] + except IndexError: + return None + + def items(self): + L = [] + for node in self._attrs.values(): + L.append((node.nodeName, node.value)) + return L + + def itemsNS(self): + L = [] + for node in self._attrs.values(): + L.append(((node.namespaceURI, node.localName), node.value)) + return L + + def __contains__(self, key): + if isinstance(key, str): + return key in self._attrs + else: + return key in self._attrsNS + + def keys(self): + return self._attrs.keys() + + def keysNS(self): + return self._attrsNS.keys() + + def values(self): + return self._attrs.values() + + def get(self, name, value=None): + return self._attrs.get(name, value) + + __len__ = _get_length + + def _cmp(self, other): + if self._attrs is getattr(other, "_attrs", None): + return 0 + else: + return (id(self) > id(other)) - (id(self) < id(other)) + + def __eq__(self, other): + return self._cmp(other) == 0 + + def __ge__(self, other): + return self._cmp(other) >= 0 + + def __gt__(self, other): + return self._cmp(other) > 0 + + def __le__(self, other): + return self._cmp(other) <= 0 + + def __lt__(self, other): + return self._cmp(other) < 0 + + def __getitem__(self, attname_or_tuple): + if isinstance(attname_or_tuple, tuple): + return self._attrsNS[attname_or_tuple] + else: + return self._attrs[attname_or_tuple] + + # same as set + def __setitem__(self, attname, value): + if isinstance(value, str): + try: + node = self._attrs[attname] + except KeyError: + node = Attr(attname) + node.ownerDocument = self._ownerElement.ownerDocument + self.setNamedItem(node) + node.value = value + else: + if not isinstance(value, Attr): + raise TypeError("value must be a string or Attr object") + node = value + self.setNamedItem(node) + + def getNamedItem(self, name): + try: + return self._attrs[name] + except KeyError: + return None + + def getNamedItemNS(self, namespaceURI, localName): + try: + return self._attrsNS[(namespaceURI, localName)] + except KeyError: + return None + + def removeNamedItem(self, name): + n = self.getNamedItem(name) + if n is not None: + _clear_id_cache(self._ownerElement) + del self._attrs[n.nodeName] + del self._attrsNS[(n.namespaceURI, n.localName)] + if hasattr(n, 'ownerElement'): + n.ownerElement = None + return n + else: + raise xml.dom.NotFoundErr() + + def removeNamedItemNS(self, namespaceURI, localName): + n = self.getNamedItemNS(namespaceURI, localName) + if n is not None: + _clear_id_cache(self._ownerElement) + del self._attrsNS[(n.namespaceURI, n.localName)] + del self._attrs[n.nodeName] + if hasattr(n, 'ownerElement'): + n.ownerElement = None + return n + else: + raise xml.dom.NotFoundErr() + + def setNamedItem(self, node): + if not isinstance(node, Attr): + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + old = self._attrs.get(node.name) + if old: + old.unlink() + self._attrs[node.name] = node + self._attrsNS[(node.namespaceURI, node.localName)] = node + node.ownerElement = self._ownerElement + _clear_id_cache(node.ownerElement) + return old + + def setNamedItemNS(self, node): + return self.setNamedItem(node) + + def __delitem__(self, attname_or_tuple): + node = self[attname_or_tuple] + _clear_id_cache(node.ownerElement) + node.unlink() + + def __getstate__(self): + return self._attrs, self._attrsNS, self._ownerElement + + def __setstate__(self, state): + self._attrs, self._attrsNS, self._ownerElement = state + +defproperty(NamedNodeMap, "length", + doc="Number of nodes in the NamedNodeMap.") + +AttributeList = NamedNodeMap + + +class TypeInfo(object): + __slots__ = 'namespace', 'name' + + def __init__(self, namespace, name): + self.namespace = namespace + self.name = name + + def __repr__(self): + if self.namespace: + return "<%s %r (from %r)>" % (self.__class__.__name__, self.name, + self.namespace) + else: + return "<%s %r>" % (self.__class__.__name__, self.name) + + def _get_name(self): + return self.name + + def _get_namespace(self): + return self.namespace + +_no_type = TypeInfo(None, None) + +class Element(Node): + __slots__=('ownerDocument', 'parentNode', 'tagName', 'nodeName', 'prefix', + 'namespaceURI', '_localName', 'childNodes', '_attrs', '_attrsNS', + 'nextSibling', 'previousSibling') + nodeType = Node.ELEMENT_NODE + nodeValue = None + schemaType = _no_type + + _magic_id_nodes = 0 + + _child_node_types = (Node.ELEMENT_NODE, + Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, + Node.TEXT_NODE, + Node.CDATA_SECTION_NODE, + Node.ENTITY_REFERENCE_NODE) + + def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, + localName=None): + self.parentNode = None + self.tagName = self.nodeName = tagName + self.prefix = prefix + self.namespaceURI = namespaceURI + self.childNodes = NodeList() + self.nextSibling = self.previousSibling = None + + # Attribute dictionaries are lazily created + # attributes are double-indexed: + # tagName -> Attribute + # URI,localName -> Attribute + # in the future: consider lazy generation + # of attribute objects this is too tricky + # for now because of headaches with + # namespaces. + self._attrs = None + self._attrsNS = None + + def _ensure_attributes(self): + if self._attrs is None: + self._attrs = {} + self._attrsNS = {} + + def _get_localName(self): + try: + return self._localName + except AttributeError: + return self.tagName.split(":", 1)[-1] + + def _get_tagName(self): + return self.tagName + + def unlink(self): + if self._attrs is not None: + for attr in list(self._attrs.values()): + attr.unlink() + self._attrs = None + self._attrsNS = None + Node.unlink(self) + + def getAttribute(self, attname): + if self._attrs is None: + return "" + try: + return self._attrs[attname].value + except KeyError: + return "" + + def getAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return "" + try: + return self._attrsNS[(namespaceURI, localName)].value + except KeyError: + return "" + + def setAttribute(self, attname, value): + attr = self.getAttributeNode(attname) + if attr is None: + attr = Attr(attname) + attr.value = value # also sets nodeValue + attr.ownerDocument = self.ownerDocument + self.setAttributeNode(attr) + elif value != attr.value: + attr.value = value + if attr.isId: + _clear_id_cache(self) + + def setAttributeNS(self, namespaceURI, qualifiedName, value): + prefix, localname = _nssplit(qualifiedName) + attr = self.getAttributeNodeNS(namespaceURI, localname) + if attr is None: + attr = Attr(qualifiedName, namespaceURI, localname, prefix) + attr.value = value + attr.ownerDocument = self.ownerDocument + self.setAttributeNode(attr) + else: + if value != attr.value: + attr.value = value + if attr.isId: + _clear_id_cache(self) + if attr.prefix != prefix: + attr.prefix = prefix + attr.nodeName = qualifiedName + + def getAttributeNode(self, attrname): + if self._attrs is None: + return None + return self._attrs.get(attrname) + + def getAttributeNodeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return None + return self._attrsNS.get((namespaceURI, localName)) + + def setAttributeNode(self, attr): + if attr.ownerElement not in (None, self): + raise xml.dom.InuseAttributeErr("attribute node already owned") + self._ensure_attributes() + old1 = self._attrs.get(attr.name, None) + if old1 is not None: + self.removeAttributeNode(old1) + old2 = self._attrsNS.get((attr.namespaceURI, attr.localName), None) + if old2 is not None and old2 is not old1: + self.removeAttributeNode(old2) + _set_attribute_node(self, attr) + + if old1 is not attr: + # It might have already been part of this node, in which case + # it doesn't represent a change, and should not be returned. + return old1 + if old2 is not attr: + return old2 + + setAttributeNodeNS = setAttributeNode + + def removeAttribute(self, name): + if self._attrsNS is None: + raise xml.dom.NotFoundErr() + try: + attr = self._attrs[name] + except KeyError: + raise xml.dom.NotFoundErr() + self.removeAttributeNode(attr) + + def removeAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + raise xml.dom.NotFoundErr() + try: + attr = self._attrsNS[(namespaceURI, localName)] + except KeyError: + raise xml.dom.NotFoundErr() + self.removeAttributeNode(attr) + + def removeAttributeNode(self, node): + if node is None: + raise xml.dom.NotFoundErr() + try: + self._attrs[node.name] + except KeyError: + raise xml.dom.NotFoundErr() + _clear_id_cache(self) + node.unlink() + # Restore this since the node is still useful and otherwise + # unlinked + node.ownerDocument = self.ownerDocument + + removeAttributeNodeNS = removeAttributeNode + + def hasAttribute(self, name): + if self._attrs is None: + return False + return name in self._attrs + + def hasAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return False + return (namespaceURI, localName) in self._attrsNS + + def getElementsByTagName(self, name): + return _get_elements_by_tagName_helper(self, name, NodeList()) + + def getElementsByTagNameNS(self, namespaceURI, localName): + return _get_elements_by_tagName_ns_helper( + self, namespaceURI, localName, NodeList()) + + def __repr__(self): + return "" % (self.tagName, id(self)) + + def writexml(self, writer, indent="", addindent="", newl=""): + # indent = current indentation + # addindent = indentation to add to higher levels + # newl = newline string + writer.write(indent+"<" + self.tagName) + + attrs = self._get_attributes() + a_names = sorted(attrs.keys()) + + for a_name in a_names: + writer.write(" %s=\"" % a_name) + _write_data(writer, attrs[a_name].value) + writer.write("\"") + if self.childNodes: + writer.write(">") + if (len(self.childNodes) == 1 and + self.childNodes[0].nodeType == Node.TEXT_NODE): + self.childNodes[0].writexml(writer, '', '', '') + else: + writer.write(newl) + for node in self.childNodes: + node.writexml(writer, indent+addindent, addindent, newl) + writer.write(indent) + writer.write("%s" % (self.tagName, newl)) + else: + writer.write("/>%s"%(newl)) + + def _get_attributes(self): + self._ensure_attributes() + return NamedNodeMap(self._attrs, self._attrsNS, self) + + def hasAttributes(self): + if self._attrs: + return True + else: + return False + + # DOM Level 3 attributes, based on the 22 Oct 2002 draft + + def setIdAttribute(self, name): + idAttr = self.getAttributeNode(name) + self.setIdAttributeNode(idAttr) + + def setIdAttributeNS(self, namespaceURI, localName): + idAttr = self.getAttributeNodeNS(namespaceURI, localName) + self.setIdAttributeNode(idAttr) + + def setIdAttributeNode(self, idAttr): + if idAttr is None or not self.isSameNode(idAttr.ownerElement): + raise xml.dom.NotFoundErr() + if _get_containing_entref(self) is not None: + raise xml.dom.NoModificationAllowedErr() + if not idAttr._is_id: + idAttr._is_id = True + self._magic_id_nodes += 1 + self.ownerDocument._magic_id_count += 1 + _clear_id_cache(self) + +defproperty(Element, "attributes", + doc="NamedNodeMap of attributes on the element.") +defproperty(Element, "localName", + doc="Namespace-local name of this element.") + + +def _set_attribute_node(element, attr): + _clear_id_cache(element) + element._ensure_attributes() + element._attrs[attr.name] = attr + element._attrsNS[(attr.namespaceURI, attr.localName)] = attr + + # This creates a circular reference, but Element.unlink() + # breaks the cycle since the references to the attribute + # dictionaries are tossed. + attr.ownerElement = element + +class Childless: + """Mixin that makes childless-ness easy to implement and avoids + the complexity of the Node methods that deal with children. + """ + __slots__ = () + + attributes = None + childNodes = EmptyNodeList() + firstChild = None + lastChild = None + + def _get_firstChild(self): + return None + + def _get_lastChild(self): + return None + + def appendChild(self, node): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes cannot have children") + + def hasChildNodes(self): + return False + + def insertBefore(self, newChild, refChild): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes do not have children") + + def removeChild(self, oldChild): + raise xml.dom.NotFoundErr( + self.nodeName + " nodes do not have children") + + def normalize(self): + # For childless nodes, normalize() has nothing to do. + pass + + def replaceChild(self, newChild, oldChild): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes do not have children") + + +class ProcessingInstruction(Childless, Node): + nodeType = Node.PROCESSING_INSTRUCTION_NODE + __slots__ = ('target', 'data') + + def __init__(self, target, data): + self.target = target + self.data = data + + # nodeValue is an alias for data + def _get_nodeValue(self): + return self.data + def _set_nodeValue(self, value): + self.data = value + nodeValue = property(_get_nodeValue, _set_nodeValue) + + # nodeName is an alias for target + def _get_nodeName(self): + return self.target + def _set_nodeName(self, value): + self.target = value + nodeName = property(_get_nodeName, _set_nodeName) + + def writexml(self, writer, indent="", addindent="", newl=""): + writer.write("%s%s" % (indent,self.target, self.data, newl)) + + +class CharacterData(Childless, Node): + __slots__=('_data', 'ownerDocument','parentNode', 'previousSibling', 'nextSibling') + + def __init__(self): + self.ownerDocument = self.parentNode = None + self.previousSibling = self.nextSibling = None + self._data = '' + Node.__init__(self) + + def _get_length(self): + return len(self.data) + __len__ = _get_length + + def _get_data(self): + return self._data + def _set_data(self, data): + self._data = data + + data = nodeValue = property(_get_data, _set_data) + + def __repr__(self): + data = self.data + if len(data) > 10: + dotdotdot = "..." + else: + dotdotdot = "" + return '' % ( + self.__class__.__name__, data[0:10], dotdotdot) + + def substringData(self, offset, count): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + return self.data[offset:offset+count] + + def appendData(self, arg): + self.data = self.data + arg + + def insertData(self, offset, arg): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if arg: + self.data = "%s%s%s" % ( + self.data[:offset], arg, self.data[offset:]) + + def deleteData(self, offset, count): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + if count: + self.data = self.data[:offset] + self.data[offset+count:] + + def replaceData(self, offset, count, arg): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + if count: + self.data = "%s%s%s" % ( + self.data[:offset], arg, self.data[offset+count:]) + +defproperty(CharacterData, "length", doc="Length of the string data.") + + +class Text(CharacterData): + __slots__ = () + + nodeType = Node.TEXT_NODE + nodeName = "#text" + attributes = None + + def splitText(self, offset): + if offset < 0 or offset > len(self.data): + raise xml.dom.IndexSizeErr("illegal offset value") + newText = self.__class__() + newText.data = self.data[offset:] + newText.ownerDocument = self.ownerDocument + next = self.nextSibling + if self.parentNode and self in self.parentNode.childNodes: + if next is None: + self.parentNode.appendChild(newText) + else: + self.parentNode.insertBefore(newText, next) + self.data = self.data[:offset] + return newText + + def writexml(self, writer, indent="", addindent="", newl=""): + _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + + # DOM Level 3 (WD 9 April 2002) + + def _get_wholeText(self): + L = [self.data] + n = self.previousSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + L.insert(0, n.data) + n = n.previousSibling + else: + break + n = self.nextSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + L.append(n.data) + n = n.nextSibling + else: + break + return ''.join(L) + + def replaceWholeText(self, content): + # XXX This needs to be seriously changed if minidom ever + # supports EntityReference nodes. + parent = self.parentNode + n = self.previousSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + next = n.previousSibling + parent.removeChild(n) + n = next + else: + break + n = self.nextSibling + if not content: + parent.removeChild(self) + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + next = n.nextSibling + parent.removeChild(n) + n = next + else: + break + if content: + self.data = content + return self + else: + return None + + def _get_isWhitespaceInElementContent(self): + if self.data.strip(): + return False + elem = _get_containing_element(self) + if elem is None: + return False + info = self.ownerDocument._get_elem_info(elem) + if info is None: + return False + else: + return info.isElementContent() + +defproperty(Text, "isWhitespaceInElementContent", + doc="True iff this text node contains only whitespace" + " and is in element content.") +defproperty(Text, "wholeText", + doc="The text of all logically-adjacent text nodes.") + + +def _get_containing_element(node): + c = node.parentNode + while c is not None: + if c.nodeType == Node.ELEMENT_NODE: + return c + c = c.parentNode + return None + +def _get_containing_entref(node): + c = node.parentNode + while c is not None: + if c.nodeType == Node.ENTITY_REFERENCE_NODE: + return c + c = c.parentNode + return None + + +class Comment(CharacterData): + nodeType = Node.COMMENT_NODE + nodeName = "#comment" + + def __init__(self, data): + CharacterData.__init__(self) + self._data = data + + def writexml(self, writer, indent="", addindent="", newl=""): + if "--" in self.data: + raise ValueError("'--' is not allowed in a comment node") + writer.write("%s%s" % (indent, self.data, newl)) + + +class CDATASection(Text): + __slots__ = () + + nodeType = Node.CDATA_SECTION_NODE + nodeName = "#cdata-section" + + def writexml(self, writer, indent="", addindent="", newl=""): + if self.data.find("]]>") >= 0: + raise ValueError("']]>' not allowed in a CDATA section") + writer.write("" % self.data) + + +class ReadOnlySequentialNamedNodeMap(object): + __slots__ = '_seq', + + def __init__(self, seq=()): + # seq should be a list or tuple + self._seq = seq + + def __len__(self): + return len(self._seq) + + def _get_length(self): + return len(self._seq) + + def getNamedItem(self, name): + for n in self._seq: + if n.nodeName == name: + return n + + def getNamedItemNS(self, namespaceURI, localName): + for n in self._seq: + if n.namespaceURI == namespaceURI and n.localName == localName: + return n + + def __getitem__(self, name_or_tuple): + if isinstance(name_or_tuple, tuple): + node = self.getNamedItemNS(*name_or_tuple) + else: + node = self.getNamedItem(name_or_tuple) + if node is None: + raise KeyError(name_or_tuple) + return node + + def item(self, index): + if index < 0: + return None + try: + return self._seq[index] + except IndexError: + return None + + def removeNamedItem(self, name): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def removeNamedItemNS(self, namespaceURI, localName): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def setNamedItem(self, node): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def setNamedItemNS(self, node): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def __getstate__(self): + return [self._seq] + + def __setstate__(self, state): + self._seq = state[0] + +defproperty(ReadOnlySequentialNamedNodeMap, "length", + doc="Number of entries in the NamedNodeMap.") + + +class Identified: + """Mix-in class that supports the publicId and systemId attributes.""" + + __slots__ = 'publicId', 'systemId' + + def _identified_mixin_init(self, publicId, systemId): + self.publicId = publicId + self.systemId = systemId + + def _get_publicId(self): + return self.publicId + + def _get_systemId(self): + return self.systemId + +class DocumentType(Identified, Childless, Node): + nodeType = Node.DOCUMENT_TYPE_NODE + nodeValue = None + name = None + publicId = None + systemId = None + internalSubset = None + + def __init__(self, qualifiedName): + self.entities = ReadOnlySequentialNamedNodeMap() + self.notations = ReadOnlySequentialNamedNodeMap() + if qualifiedName: + prefix, localname = _nssplit(qualifiedName) + self.name = localname + self.nodeName = self.name + + def _get_internalSubset(self): + return self.internalSubset + + def cloneNode(self, deep): + if self.ownerDocument is None: + # it's ok + clone = DocumentType(None) + clone.name = self.name + clone.nodeName = self.name + operation = xml.dom.UserDataHandler.NODE_CLONED + if deep: + clone.entities._seq = [] + clone.notations._seq = [] + for n in self.notations._seq: + notation = Notation(n.nodeName, n.publicId, n.systemId) + clone.notations._seq.append(notation) + n._call_user_data_handler(operation, n, notation) + for e in self.entities._seq: + entity = Entity(e.nodeName, e.publicId, e.systemId, + e.notationName) + entity.actualEncoding = e.actualEncoding + entity.encoding = e.encoding + entity.version = e.version + clone.entities._seq.append(entity) + e._call_user_data_handler(operation, e, entity) + self._call_user_data_handler(operation, self, clone) + return clone + else: + return None + + def writexml(self, writer, indent="", addindent="", newl=""): + writer.write(""+newl) + +class Entity(Identified, Node): + attributes = None + nodeType = Node.ENTITY_NODE + nodeValue = None + + actualEncoding = None + encoding = None + version = None + + def __init__(self, name, publicId, systemId, notation): + self.nodeName = name + self.notationName = notation + self.childNodes = NodeList() + self._identified_mixin_init(publicId, systemId) + + def _get_actualEncoding(self): + return self.actualEncoding + + def _get_encoding(self): + return self.encoding + + def _get_version(self): + return self.version + + def appendChild(self, newChild): + raise xml.dom.HierarchyRequestErr( + "cannot append children to an entity node") + + def insertBefore(self, newChild, refChild): + raise xml.dom.HierarchyRequestErr( + "cannot insert children below an entity node") + + def removeChild(self, oldChild): + raise xml.dom.HierarchyRequestErr( + "cannot remove children from an entity node") + + def replaceChild(self, newChild, oldChild): + raise xml.dom.HierarchyRequestErr( + "cannot replace children of an entity node") + +class Notation(Identified, Childless, Node): + nodeType = Node.NOTATION_NODE + nodeValue = None + + def __init__(self, name, publicId, systemId): + self.nodeName = name + self._identified_mixin_init(publicId, systemId) + + +class DOMImplementation(DOMImplementationLS): + _features = [("core", "1.0"), + ("core", "2.0"), + ("core", None), + ("xml", "1.0"), + ("xml", "2.0"), + ("xml", None), + ("ls-load", "3.0"), + ("ls-load", None), + ] + + def hasFeature(self, feature, version): + if version == "": + version = None + return (feature.lower(), version) in self._features + + def createDocument(self, namespaceURI, qualifiedName, doctype): + if doctype and doctype.parentNode is not None: + raise xml.dom.WrongDocumentErr( + "doctype object owned by another DOM tree") + doc = self._create_document() + + add_root_element = not (namespaceURI is None + and qualifiedName is None + and doctype is None) + + if not qualifiedName and add_root_element: + # The spec is unclear what to raise here; SyntaxErr + # would be the other obvious candidate. Since Xerces raises + # InvalidCharacterErr, and since SyntaxErr is not listed + # for createDocument, that seems to be the better choice. + # XXX: need to check for illegal characters here and in + # createElement. + + # DOM Level III clears this up when talking about the return value + # of this function. If namespaceURI, qName and DocType are + # Null the document is returned without a document element + # Otherwise if doctype or namespaceURI are not None + # Then we go back to the above problem + raise xml.dom.InvalidCharacterErr("Element with no name") + + if add_root_element: + prefix, localname = _nssplit(qualifiedName) + if prefix == "xml" \ + and namespaceURI != "http://www.w3.org/XML/1998/namespace": + raise xml.dom.NamespaceErr("illegal use of 'xml' prefix") + if prefix and not namespaceURI: + raise xml.dom.NamespaceErr( + "illegal use of prefix without namespaces") + element = doc.createElementNS(namespaceURI, qualifiedName) + if doctype: + doc.appendChild(doctype) + doc.appendChild(element) + + if doctype: + doctype.parentNode = doctype.ownerDocument = doc + + doc.doctype = doctype + doc.implementation = self + return doc + + def createDocumentType(self, qualifiedName, publicId, systemId): + doctype = DocumentType(qualifiedName) + doctype.publicId = publicId + doctype.systemId = systemId + return doctype + + # DOM Level 3 (WD 9 April 2002) + + def getInterface(self, feature): + if self.hasFeature(feature, None): + return self + else: + return None + + # internal + def _create_document(self): + return Document() + +class ElementInfo(object): + """Object that represents content-model information for an element. + + This implementation is not expected to be used in practice; DOM + builders should provide implementations which do the right thing + using information available to it. + + """ + + __slots__ = 'tagName', + + def __init__(self, name): + self.tagName = name + + def getAttributeType(self, aname): + return _no_type + + def getAttributeTypeNS(self, namespaceURI, localName): + return _no_type + + def isElementContent(self): + return False + + def isEmpty(self): + """Returns true iff this element is declared to have an EMPTY + content model.""" + return False + + def isId(self, aname): + """Returns true iff the named attribute is a DTD-style ID.""" + return False + + def isIdNS(self, namespaceURI, localName): + """Returns true iff the identified attribute is a DTD-style ID.""" + return False + + def __getstate__(self): + return self.tagName + + def __setstate__(self, state): + self.tagName = state + +def _clear_id_cache(node): + if node.nodeType == Node.DOCUMENT_NODE: + node._id_cache.clear() + node._id_search_stack = None + elif _in_document(node): + node.ownerDocument._id_cache.clear() + node.ownerDocument._id_search_stack= None + +class Document(Node, DocumentLS): + __slots__ = ('_elem_info', 'doctype', + '_id_search_stack', 'childNodes', '_id_cache') + _child_node_types = (Node.ELEMENT_NODE, Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, Node.DOCUMENT_TYPE_NODE) + + implementation = DOMImplementation() + nodeType = Node.DOCUMENT_NODE + nodeName = "#document" + nodeValue = None + attributes = None + parentNode = None + previousSibling = nextSibling = None + + + # Document attributes from Level 3 (WD 9 April 2002) + + actualEncoding = None + encoding = None + standalone = None + version = None + strictErrorChecking = False + errorHandler = None + documentURI = None + + _magic_id_count = 0 + + def __init__(self): + self.doctype = None + self.childNodes = NodeList() + # mapping of (namespaceURI, localName) -> ElementInfo + # and tagName -> ElementInfo + self._elem_info = {} + self._id_cache = {} + self._id_search_stack = None + + def _get_elem_info(self, element): + if element.namespaceURI: + key = element.namespaceURI, element.localName + else: + key = element.tagName + return self._elem_info.get(key) + + def _get_actualEncoding(self): + return self.actualEncoding + + def _get_doctype(self): + return self.doctype + + def _get_documentURI(self): + return self.documentURI + + def _get_encoding(self): + return self.encoding + + def _get_errorHandler(self): + return self.errorHandler + + def _get_standalone(self): + return self.standalone + + def _get_strictErrorChecking(self): + return self.strictErrorChecking + + def _get_version(self): + return self.version + + def appendChild(self, node): + if node.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + if node.parentNode is not None: + # This needs to be done before the next test since this + # may *be* the document element, in which case it should + # end up re-ordered to the end. + node.parentNode.removeChild(node) + + if node.nodeType == Node.ELEMENT_NODE \ + and self._get_documentElement(): + raise xml.dom.HierarchyRequestErr( + "two document elements disallowed") + return Node.appendChild(self, node) + + def removeChild(self, oldChild): + try: + self.childNodes.remove(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + oldChild.nextSibling = oldChild.previousSibling = None + oldChild.parentNode = None + if self.documentElement is oldChild: + self.documentElement = None + + return oldChild + + def _get_documentElement(self): + for node in self.childNodes: + if node.nodeType == Node.ELEMENT_NODE: + return node + + def unlink(self): + if self.doctype is not None: + self.doctype.unlink() + self.doctype = None + Node.unlink(self) + + def cloneNode(self, deep): + if not deep: + return None + clone = self.implementation.createDocument(None, None, None) + clone.encoding = self.encoding + clone.standalone = self.standalone + clone.version = self.version + for n in self.childNodes: + childclone = _clone_node(n, deep, clone) + assert childclone.ownerDocument.isSameNode(clone) + clone.childNodes.append(childclone) + if childclone.nodeType == Node.DOCUMENT_NODE: + assert clone.documentElement is None + elif childclone.nodeType == Node.DOCUMENT_TYPE_NODE: + assert clone.doctype is None + clone.doctype = childclone + childclone.parentNode = clone + self._call_user_data_handler(xml.dom.UserDataHandler.NODE_CLONED, + self, clone) + return clone + + def createDocumentFragment(self): + d = DocumentFragment() + d.ownerDocument = self + return d + + def createElement(self, tagName): + e = Element(tagName) + e.ownerDocument = self + return e + + def createTextNode(self, data): + if not isinstance(data, str): + raise TypeError("node contents must be a string") + t = Text() + t.data = data + t.ownerDocument = self + return t + + def createCDATASection(self, data): + if not isinstance(data, str): + raise TypeError("node contents must be a string") + c = CDATASection() + c.data = data + c.ownerDocument = self + return c + + def createComment(self, data): + c = Comment(data) + c.ownerDocument = self + return c + + def createProcessingInstruction(self, target, data): + p = ProcessingInstruction(target, data) + p.ownerDocument = self + return p + + def createAttribute(self, qName): + a = Attr(qName) + a.ownerDocument = self + a.value = "" + return a + + def createElementNS(self, namespaceURI, qualifiedName): + prefix, localName = _nssplit(qualifiedName) + e = Element(qualifiedName, namespaceURI, prefix) + e.ownerDocument = self + return e + + def createAttributeNS(self, namespaceURI, qualifiedName): + prefix, localName = _nssplit(qualifiedName) + a = Attr(qualifiedName, namespaceURI, localName, prefix) + a.ownerDocument = self + a.value = "" + return a + + # A couple of implementation-specific helpers to create node types + # not supported by the W3C DOM specs: + + def _create_entity(self, name, publicId, systemId, notationName): + e = Entity(name, publicId, systemId, notationName) + e.ownerDocument = self + return e + + def _create_notation(self, name, publicId, systemId): + n = Notation(name, publicId, systemId) + n.ownerDocument = self + return n + + def getElementById(self, id): + if id in self._id_cache: + return self._id_cache[id] + if not (self._elem_info or self._magic_id_count): + return None + + stack = self._id_search_stack + if stack is None: + # we never searched before, or the cache has been cleared + stack = [self.documentElement] + self._id_search_stack = stack + elif not stack: + # Previous search was completed and cache is still valid; + # no matching node. + return None + + result = None + while stack: + node = stack.pop() + # add child elements to stack for continued searching + stack.extend([child for child in node.childNodes + if child.nodeType in _nodeTypes_with_children]) + # check this node + info = self._get_elem_info(node) + if info: + # We have to process all ID attributes before + # returning in order to get all the attributes set to + # be IDs using Element.setIdAttribute*(). + for attr in node.attributes.values(): + if attr.namespaceURI: + if info.isIdNS(attr.namespaceURI, attr.localName): + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif not node._magic_id_nodes: + break + elif info.isId(attr.name): + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif not node._magic_id_nodes: + break + elif attr._is_id: + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif node._magic_id_nodes == 1: + break + elif node._magic_id_nodes: + for attr in node.attributes.values(): + if attr._is_id: + self._id_cache[attr.value] = node + if attr.value == id: + result = node + if result is not None: + break + return result + + def getElementsByTagName(self, name): + return _get_elements_by_tagName_helper(self, name, NodeList()) + + def getElementsByTagNameNS(self, namespaceURI, localName): + return _get_elements_by_tagName_ns_helper( + self, namespaceURI, localName, NodeList()) + + def isSupported(self, feature, version): + return self.implementation.hasFeature(feature, version) + + def importNode(self, node, deep): + if node.nodeType == Node.DOCUMENT_NODE: + raise xml.dom.NotSupportedErr("cannot import document nodes") + elif node.nodeType == Node.DOCUMENT_TYPE_NODE: + raise xml.dom.NotSupportedErr("cannot import document type nodes") + return _clone_node(node, deep, self) + + def writexml(self, writer, indent="", addindent="", newl="", encoding=None): + if encoding is None: + writer.write(''+newl) + else: + writer.write('%s' % ( + encoding, newl)) + for node in self.childNodes: + node.writexml(writer, indent, addindent, newl) + + # DOM Level 3 (WD 9 April 2002) + + def renameNode(self, n, namespaceURI, name): + if n.ownerDocument is not self: + raise xml.dom.WrongDocumentErr( + "cannot rename nodes from other documents;\n" + "expected %s,\nfound %s" % (self, n.ownerDocument)) + if n.nodeType not in (Node.ELEMENT_NODE, Node.ATTRIBUTE_NODE): + raise xml.dom.NotSupportedErr( + "renameNode() only applies to element and attribute nodes") + if namespaceURI != EMPTY_NAMESPACE: + if ':' in name: + prefix, localName = name.split(':', 1) + if ( prefix == "xmlns" + and namespaceURI != xml.dom.XMLNS_NAMESPACE): + raise xml.dom.NamespaceErr( + "illegal use of 'xmlns' prefix") + else: + if ( name == "xmlns" + and namespaceURI != xml.dom.XMLNS_NAMESPACE + and n.nodeType == Node.ATTRIBUTE_NODE): + raise xml.dom.NamespaceErr( + "illegal use of the 'xmlns' attribute") + prefix = None + localName = name + else: + prefix = None + localName = None + if n.nodeType == Node.ATTRIBUTE_NODE: + element = n.ownerElement + if element is not None: + is_id = n._is_id + element.removeAttributeNode(n) + else: + element = None + n.prefix = prefix + n._localName = localName + n.namespaceURI = namespaceURI + n.nodeName = name + if n.nodeType == Node.ELEMENT_NODE: + n.tagName = name + else: + # attribute node + n.name = name + if element is not None: + element.setAttributeNode(n) + if is_id: + element.setIdAttributeNode(n) + # It's not clear from a semantic perspective whether we should + # call the user data handlers for the NODE_RENAMED event since + # we're re-using the existing node. The draft spec has been + # interpreted as meaning "no, don't call the handler unless a + # new node is created." + return n + +defproperty(Document, "documentElement", + doc="Top-level element of this document.") + + +def _clone_node(node, deep, newOwnerDocument): + """ + Clone a node and give it the new owner document. + Called by Node.cloneNode and Document.importNode + """ + if node.ownerDocument.isSameNode(newOwnerDocument): + operation = xml.dom.UserDataHandler.NODE_CLONED + else: + operation = xml.dom.UserDataHandler.NODE_IMPORTED + if node.nodeType == Node.ELEMENT_NODE: + clone = newOwnerDocument.createElementNS(node.namespaceURI, + node.nodeName) + for attr in node.attributes.values(): + clone.setAttributeNS(attr.namespaceURI, attr.nodeName, attr.value) + a = clone.getAttributeNodeNS(attr.namespaceURI, attr.localName) + a.specified = attr.specified + + if deep: + for child in node.childNodes: + c = _clone_node(child, deep, newOwnerDocument) + clone.appendChild(c) + + elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: + clone = newOwnerDocument.createDocumentFragment() + if deep: + for child in node.childNodes: + c = _clone_node(child, deep, newOwnerDocument) + clone.appendChild(c) + + elif node.nodeType == Node.TEXT_NODE: + clone = newOwnerDocument.createTextNode(node.data) + elif node.nodeType == Node.CDATA_SECTION_NODE: + clone = newOwnerDocument.createCDATASection(node.data) + elif node.nodeType == Node.PROCESSING_INSTRUCTION_NODE: + clone = newOwnerDocument.createProcessingInstruction(node.target, + node.data) + elif node.nodeType == Node.COMMENT_NODE: + clone = newOwnerDocument.createComment(node.data) + elif node.nodeType == Node.ATTRIBUTE_NODE: + clone = newOwnerDocument.createAttributeNS(node.namespaceURI, + node.nodeName) + clone.specified = True + clone.value = node.value + elif node.nodeType == Node.DOCUMENT_TYPE_NODE: + assert node.ownerDocument is not newOwnerDocument + operation = xml.dom.UserDataHandler.NODE_IMPORTED + clone = newOwnerDocument.implementation.createDocumentType( + node.name, node.publicId, node.systemId) + clone.ownerDocument = newOwnerDocument + if deep: + clone.entities._seq = [] + clone.notations._seq = [] + for n in node.notations._seq: + notation = Notation(n.nodeName, n.publicId, n.systemId) + notation.ownerDocument = newOwnerDocument + clone.notations._seq.append(notation) + if hasattr(n, '_call_user_data_handler'): + n._call_user_data_handler(operation, n, notation) + for e in node.entities._seq: + entity = Entity(e.nodeName, e.publicId, e.systemId, + e.notationName) + entity.actualEncoding = e.actualEncoding + entity.encoding = e.encoding + entity.version = e.version + entity.ownerDocument = newOwnerDocument + clone.entities._seq.append(entity) + if hasattr(e, '_call_user_data_handler'): + e._call_user_data_handler(operation, e, entity) + else: + # Note the cloning of Document and DocumentType nodes is + # implementation specific. minidom handles those cases + # directly in the cloneNode() methods. + raise xml.dom.NotSupportedErr("Cannot clone node %s" % repr(node)) + + # Check for _call_user_data_handler() since this could conceivably + # used with other DOM implementations (one of the FourThought + # DOMs, perhaps?). + if hasattr(node, '_call_user_data_handler'): + node._call_user_data_handler(operation, node, clone) + return clone + + +def _nssplit(qualifiedName): + fields = qualifiedName.split(':', 1) + if len(fields) == 2: + return fields + else: + return (None, fields[0]) + + +def _do_pulldom_parse(func, args, kwargs): + events = func(*args, **kwargs) + toktype, rootNode = events.getEvent() + events.expandNode(rootNode) + events.clear() + return rootNode + +def parse(file, parser=None, bufsize=None): + """Parse a file into a DOM by filename or file object.""" + if parser is None and not bufsize: + from xml.dom import expatbuilder + return expatbuilder.parse(file) + else: + from xml.dom import pulldom + return _do_pulldom_parse(pulldom.parse, (file,), + {'parser': parser, 'bufsize': bufsize}) + +def parseString(string, parser=None): + """Parse a file into a DOM from a string.""" + if parser is None: + from xml.dom import expatbuilder + return expatbuilder.parseString(string) + else: + from xml.dom import pulldom + return _do_pulldom_parse(pulldom.parseString, (string,), + {'parser': parser}) + +def getDOMImplementation(features=None): + if features: + if isinstance(features, str): + features = domreg._parse_feature_string(features) + for f, v in features: + if not Document.implementation.hasFeature(f, v): + return None + return Document.implementation diff --git a/benchmarks/benchmarks/nbody.py b/benchmarks/benchmarks/nbody.py index 2f36bdf0e..b9ab142df 100644 --- a/benchmarks/benchmarks/nbody.py +++ b/benchmarks/benchmarks/nbody.py @@ -89,7 +89,7 @@ def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0): e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5) for (r, [vx, vy, vz], m) in bodies: e += m * (vx * vx + vy * vy + vz * vz) / 2. - print(f"{e}") + # print(f"{e}") def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0): for (r, [vx, vy, vz], m) in bodies: From de3900bcad3253ad52e68033d0caf75d8d59d7a4 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 19 Apr 2019 15:17:03 -0500 Subject: [PATCH 334/658] Upgrade deps --- wasm/tests/Pipfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/wasm/tests/Pipfile.lock b/wasm/tests/Pipfile.lock index 7f8b97f87..32e0040b0 100644 --- a/wasm/tests/Pipfile.lock +++ b/wasm/tests/Pipfile.lock @@ -32,11 +32,11 @@ }, "more-itertools": { "hashes": [ - "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", - "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" ], "markers": "python_version > '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "pluggy": { "hashes": [ @@ -54,11 +54,11 @@ }, "pytest": { "hashes": [ - "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", - "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.3.1" + "version": "==4.4.1" }, "selenium": { "hashes": [ @@ -77,10 +77,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" } }, "develop": {} From 3828652da61db2775e18c099d8b01fe958be6e08 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 11:17:36 +0300 Subject: [PATCH 335/658] Add os.scandir --- vm/src/stdlib/os.rs | 76 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index fae9d6adb..c103b3d46 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::fs::File; use std::fs::OpenOptions; use std::io::{ErrorKind, Read, Write}; @@ -10,9 +11,11 @@ use crate::obj::objbytes::PyBytesRef; use crate::obj::objdict::PyDictRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; +use crate::obj::objiter; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -190,6 +193,65 @@ fn _os_environ(vm: &VirtualMachine) -> PyDictRef { environ } +#[derive(Debug)] +struct DirEntry { + entry: fs::DirEntry, +} + +type DirEntryRef = PyRef; + +impl PyValue for DirEntry { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("os", "DirEntry") + } +} + +impl DirEntryRef { + fn name(self, _vm: &VirtualMachine) -> String { + self.entry.file_name().into_string().unwrap() + } +} + +#[derive(Debug)] +pub struct ScandirIterator { + entries: RefCell, +} + +impl PyValue for ScandirIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("os", "ScandirIter") + } +} + +type ScandirIteratorRef = PyRef; + +impl ScandirIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + match self.entries.borrow_mut().next() { + Some(entry) => match entry { + Ok(entry) => Ok(DirEntry { entry }.into_ref(vm).into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +fn os_scandir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => Ok(ScandirIterator { + entries: RefCell::new(iter), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -201,6 +263,15 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let environ = _os_environ(vm); + let scandir_iter = py_class!(ctx, "ScandirIter", ctx.object(), { + "__iter__" => ctx.new_rustfunc(ScandirIteratorRef::iter), + "__next__" => ctx.new_rustfunc(ScandirIteratorRef::next), + }); + + let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { + "name" => ctx.new_rustfunc(DirEntryRef::name), + }); + py_module!(vm, "_os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), @@ -217,6 +288,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "unsetenv" => ctx.new_rustfunc(os_unsetenv), "environ" => environ, "name" => ctx.new_str(os_name), + "scandir" => ctx.new_rustfunc(os_scandir), + "ScandirIter" => scandir_iter, + "DirEntry" => dir_entry, "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From 858a6d03b097863384d9b9c8f2e4f1e7e739b310 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 11:42:57 +0300 Subject: [PATCH 336/658] Add DirEntry.{name, path} --- tests/snippets/stdlib_os.py | 14 ++++++++++++++ vm/src/stdlib/os.rs | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 4d64fc793..237acf94f 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -56,6 +56,7 @@ class TestWithTempDir(): FILE_NAME = "test1" +FILE_NAME2 = "test2" CONTENT = b"testing" CONTENT2 = b"rustpython" CONTENT3 = b"BOYA" @@ -73,3 +74,16 @@ with TestWithTempDir() as tmpdir: assert os.read(fd, len(CONTENT2)) == CONTENT2 assert os.read(fd, len(CONTENT3)) == CONTENT3 os.close(fd) + + + fname2 = tmpdir + os.sep + FILE_NAME2 + with open(fname2, "wb"): + pass + files = set() + paths = set() + for dir_entry in os.scandir(tmpdir): + files.add(dir_entry.name) + paths.add(dir_entry.path) + + assert files == set([FILE_NAME, FILE_NAME2]) + assert paths == set([fname, fname2]) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c103b3d46..395e269d5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -210,6 +210,10 @@ impl DirEntryRef { fn name(self, _vm: &VirtualMachine) -> String { self.entry.file_name().into_string().unwrap() } + + fn path(self, _vm: &VirtualMachine) -> String { + self.entry.path().to_str().unwrap().to_string() + } } #[derive(Debug)] @@ -269,7 +273,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { - "name" => ctx.new_rustfunc(DirEntryRef::name), + "name" => ctx.new_property(DirEntryRef::name), + "path" => ctx.new_property(DirEntryRef::path), }); py_module!(vm, "_os", { From 548f3f34c48ff55e63223fdb1a99f44fbae627fb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 12:04:12 +0300 Subject: [PATCH 337/658] Add DirEntry.{is_dir} --- tests/snippets/stdlib_os.py | 23 +++++++++++++++-------- vm/src/stdlib/os.rs | 9 +++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 237acf94f..4bc50ee41 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -57,6 +57,7 @@ class TestWithTempDir(): FILE_NAME = "test1" FILE_NAME2 = "test2" +FOLDER = "dir1" CONTENT = b"testing" CONTENT2 = b"rustpython" CONTENT3 = b"BOYA" @@ -75,15 +76,21 @@ with TestWithTempDir() as tmpdir: assert os.read(fd, len(CONTENT3)) == CONTENT3 os.close(fd) - fname2 = tmpdir + os.sep + FILE_NAME2 with open(fname2, "wb"): pass - files = set() - paths = set() - for dir_entry in os.scandir(tmpdir): - files.add(dir_entry.name) - paths.add(dir_entry.path) + folder = tmpdir + os.sep + FOLDER + os.mkdir(folder) - assert files == set([FILE_NAME, FILE_NAME2]) - assert paths == set([fname, fname2]) + names = set() + paths = set() + dirs = set() + for dir_entry in os.scandir(tmpdir): + names.add(dir_entry.name) + paths.add(dir_entry.path) + if dir_entry.is_dir(): + dirs.add(dir_entry.name) + + assert names == set([FILE_NAME, FILE_NAME2, FOLDER]) + assert paths == set([fname, fname2, folder]) + assert dirs == set([FOLDER]) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 395e269d5..43c65129a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -214,6 +214,14 @@ impl DirEntryRef { fn path(self, _vm: &VirtualMachine) -> String { self.entry.path().to_str().unwrap().to_string() } + + fn is_dir(self, vm: &VirtualMachine) -> PyResult { + Ok(self + .entry + .file_type() + .map_err(|s| vm.new_os_error(s.to_string()))? + .is_dir()) + } } #[derive(Debug)] @@ -275,6 +283,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { "name" => ctx.new_property(DirEntryRef::name), "path" => ctx.new_property(DirEntryRef::path), + "is_dir" => ctx.new_rustfunc(DirEntryRef::is_dir), }); py_module!(vm, "_os", { From 64e6ea001662ad93b6bc4757c8331e05e2aed4ed Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 12:06:49 +0300 Subject: [PATCH 338/658] Add DirEntry.{is_file} --- tests/snippets/stdlib_os.py | 4 ++++ vm/src/stdlib/os.rs | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 4bc50ee41..0de5b52b6 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -85,12 +85,16 @@ with TestWithTempDir() as tmpdir: names = set() paths = set() dirs = set() + files = set() for dir_entry in os.scandir(tmpdir): names.add(dir_entry.name) paths.add(dir_entry.path) if dir_entry.is_dir(): dirs.add(dir_entry.name) + if dir_entry.is_file(): + files.add(dir_entry.name) assert names == set([FILE_NAME, FILE_NAME2, FOLDER]) assert paths == set([fname, fname2, folder]) assert dirs == set([FOLDER]) + assert files == set([FILE_NAME, FILE_NAME2]) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 43c65129a..9717c8f22 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -222,6 +222,14 @@ impl DirEntryRef { .map_err(|s| vm.new_os_error(s.to_string()))? .is_dir()) } + + fn is_file(self, vm: &VirtualMachine) -> PyResult { + Ok(self + .entry + .file_type() + .map_err(|s| vm.new_os_error(s.to_string()))? + .is_file()) + } } #[derive(Debug)] @@ -284,6 +292,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "name" => ctx.new_property(DirEntryRef::name), "path" => ctx.new_property(DirEntryRef::path), "is_dir" => ctx.new_rustfunc(DirEntryRef::is_dir), + "is_file" => ctx.new_rustfunc(DirEntryRef::is_file), }); py_module!(vm, "_os", { From 3986e2f87d5c6396f8d7acc55c3334fe9eb5b8c0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 16:41:36 +0300 Subject: [PATCH 339/658] Add set.isdisjoint --- tests/snippets/set.py | 8 ++++++++ vm/src/obj/objset.rs | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 393db0754..ba20fbae6 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -79,6 +79,10 @@ assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) +assert set([1,2,3]).isdisjoint(set([5,6])) == True +assert set([1,2,3]).isdisjoint(set([2,5,6])) == False +assert set([1,2,3]).isdisjoint([5,6]) == True + assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -227,6 +231,10 @@ assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) +assert frozenset([1,2,3]).isdisjoint(frozenset([5,6])) == True +assert frozenset([1,2,3]).isdisjoint(frozenset([2,5,6])) == False +assert frozenset([1,2,3]).isdisjoint([5,6]) == True + assert_raises(TypeError, lambda: frozenset([[]])) a = frozenset([1,2,3]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b8f9f6d3b..b4f3bfabc 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -222,6 +222,16 @@ impl PySetInner { Ok(new_inner) } + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in other.iter(vm)? { + let obj = item?; + if self.contains(obj.clone(), vm)? { + return Ok(false); + } + } + Ok(true) + } + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { let items = self.elements.values().cloned().collect(); let set_list = vm.ctx.new_list(items); @@ -426,6 +436,10 @@ impl PySetRef { )) } + fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isdisjoint(other, vm) + } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } @@ -633,6 +647,10 @@ impl PyFrozenSetRef { )) } + fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.isdisjoint(other, vm) + } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } @@ -788,7 +806,8 @@ pub fn init(context: &PyContext) { "__isub__" => context.new_rustfunc(PySetRef::isub), "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), "__ixor__" => context.new_rustfunc(PySetRef::ixor), - "__iter__" => context.new_rustfunc(PySetRef::iter) + "__iter__" => context.new_rustfunc(PySetRef::iter), + "isdisjoint" => context.new_rustfunc(PySetRef::isdisjoint), }); let frozenset_type = &context.frozenset_type; @@ -819,6 +838,7 @@ pub fn init(context: &PyContext) { "__doc__" => context.new_str(frozenset_doc.to_string()), "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), "copy" => context.new_rustfunc(PyFrozenSetRef::copy), - "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter) + "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter), + "isdisjoint" => context.new_rustfunc(PyFrozenSetRef::isdisjoint), }); } From 206ba4dbfb3afa30cf9835d30edf625c5a89907d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 17:20:33 +0300 Subject: [PATCH 340/658] Change socket to new args style --- vm/src/stdlib/socket.rs | 200 +++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 117 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 9d7a703e1..96d382970 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -7,9 +7,10 @@ use std::ops::Deref; use crate::function::PyFuncArgs; use crate::obj::objbytes; +use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; -use crate::obj::objsequence::get_elements; use crate::obj::objstr; +use crate::obj::objtuple::PyTupleRef; use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; @@ -167,88 +168,94 @@ fn get_socket<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { type SocketRef = PyRef; -fn socket_new( - cls: PyClassRef, - family: AddressFamily, - kind: SocketKind, - vm: &VirtualMachine, -) -> PyResult { - Socket::new(family, kind).into_ref_with_type(vm, cls) -} +impl SocketRef { + fn new( + cls: PyClassRef, + family: AddressFamily, + kind: SocketKind, + vm: &VirtualMachine, + ) -> PyResult { + Socket::new(family, kind).into_ref_with_type(vm, cls) + } -fn socket_connect(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] - ); + fn connect(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + let address_string = get_address_string(vm, address)?; - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Stream => match TcpStream::connect(address_string) { - Ok(stream) => { - socket - .con - .borrow_mut() - .replace(Connection::TcpStream(stream)); - Ok(vm.get_none()) + match self.socket_kind { + SocketKind::Stream => match TcpStream::connect(address_string) { + Ok(stream) => { + self.con.borrow_mut().replace(Connection::TcpStream(stream)); + Ok(vm.get_none()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + SocketKind::Dgram => { + if let Some(Connection::UdpSocket(con)) = self.con.borrow().as_ref() { + match con.connect(address_string) { + Ok(_) => Ok(vm.get_none()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } + } else { + Err(vm.new_type_error("".to_string())) + } } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - SocketKind::Dgram => { - if let Some(Connection::UdpSocket(con)) = socket.con.borrow().as_ref() { - match con.connect(address_string) { - Ok(_) => Ok(vm.get_none()), + } + } + + fn bind(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + let address_string = get_address_string(vm, address)?; + + match self.socket_kind { + SocketKind::Stream => match TcpListener::bind(address_string) { + Ok(stream) => { + self.con + .borrow_mut() + .replace(Connection::TcpListener(stream)); + Ok(vm.get_none()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + SocketKind::Dgram => match UdpSocket::bind(address_string) { + Ok(dgram) => { + self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); + Ok(vm.get_none()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + } + } + + fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + let address_string = get_address_string(vm, address)?; + + match self.socket_kind { + SocketKind::Dgram => { + if let Some(v) = self.con.borrow().as_ref() { + return match v.send_to(&bytes, address_string) { + Ok(_) => Ok(vm.get_none()), + Err(s) => Err(vm.new_os_error(s.to_string())), + }; + } + // Doing implicit bind + match UdpSocket::bind("0.0.0.0:0") { + Ok(dgram) => match dgram.send_to(&bytes, address_string) { + Ok(_) => { + self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); + Ok(vm.get_none()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, Err(s) => Err(vm.new_os_error(s.to_string())), } - } else { - Err(vm.new_type_error("".to_string())) } + _ => Err(vm.new_not_implemented_error("".to_string())), } } } -fn socket_bind(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] - ); - - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Stream => match TcpListener::bind(address_string) { - Ok(stream) => { - socket - .con - .borrow_mut() - .replace(Connection::TcpListener(stream)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - SocketKind::Dgram => match UdpSocket::bind(address_string) { - Ok(dgram) => { - socket - .con - .borrow_mut() - .replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - } -} - -fn get_address_string(vm: &VirtualMachine, address: &PyObjectRef) -> Result { +fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result { let args = PyFuncArgs { - args: get_elements(address).to_vec(), + args: address.elements.borrow().to_vec(), kwargs: vec![], }; arg_check!( @@ -365,47 +372,6 @@ fn socket_send(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -fn socket_sendto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (zelf, None), - (bytes, Some(vm.ctx.bytes_type())), - (address, Some(vm.ctx.tuple_type())) - ] - ); - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Dgram => { - if let Some(v) = socket.con.borrow().as_ref() { - return match v.send_to(&objbytes::get_value(&bytes), address_string) { - Ok(_) => Ok(vm.get_none()), - Err(s) => Err(vm.new_os_error(s.to_string())), - }; - } - // Doing implicit bind - match UdpSocket::bind("0.0.0.0:0") { - Ok(dgram) => match dgram.send_to(&objbytes::get_value(&bytes), address_string) { - Ok(_) => { - socket - .con - .borrow_mut() - .replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - Err(s) => Err(vm.new_os_error(s.to_string())), - } - } - _ => Err(vm.new_not_implemented_error("".to_string())), - } -} - fn socket_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(zelf, None)]); @@ -452,16 +418,16 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let socket = py_class!(ctx, "socket", ctx.object(), { - "__new__" => ctx.new_rustfunc(socket_new), - "connect" => ctx.new_rustfunc(socket_connect), + "__new__" => ctx.new_rustfunc(SocketRef::new), + "connect" => ctx.new_rustfunc(SocketRef::connect), "recv" => ctx.new_rustfunc(socket_recv), "send" => ctx.new_rustfunc(socket_send), - "bind" => ctx.new_rustfunc(socket_bind), + "bind" => ctx.new_rustfunc(SocketRef::bind), "accept" => ctx.new_rustfunc(socket_accept), "listen" => ctx.new_rustfunc(socket_listen), "close" => ctx.new_rustfunc(socket_close), "getsockname" => ctx.new_rustfunc(socket_getsockname), - "sendto" => ctx.new_rustfunc(socket_sendto), + "sendto" => ctx.new_rustfunc(SocketRef::sendto), "recvfrom" => ctx.new_rustfunc(socket_recvfrom), "fileno" => ctx.new_rustfunc(socket_fileno), }); From 9954361df9fc7565efeaddf1d57ceeaca7567b29 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 17:28:41 +0300 Subject: [PATCH 341/658] Move more socket methods to new arg style --- vm/src/stdlib/socket.rs | 99 +++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 58 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 96d382970..dc165d316 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -9,6 +9,7 @@ use crate::function::PyFuncArgs; use crate::obj::objbytes; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; +use crate::obj::objint::PyIntRef; use crate::obj::objstr; use crate::obj::objtuple::PyTupleRef; use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; @@ -251,6 +252,43 @@ impl SocketRef { _ => Err(vm.new_not_implemented_error("".to_string())), } } + + fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) -> () {} + + fn accept(self, vm: &VirtualMachine) -> PyResult { + let ret = match self.con.borrow_mut().as_mut() { + Some(v) => v.accept(), + None => return Err(vm.new_type_error("".to_string())), + }; + + let (tcp_stream, addr) = match ret { + Ok((socket, addr)) => (socket, addr), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + let socket = Socket { + address_family: self.address_family, + socket_kind: self.socket_kind, + con: RefCell::new(Some(Connection::TcpStream(tcp_stream))), + } + .into_ref(vm); + + let addr_tuple = get_addr_tuple(vm, addr)?; + + Ok(vm.ctx.new_tuple(vec![socket.into_object(), addr_tuple])) + } + + fn recv(self, bufsize: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; bufsize.as_bigint().to_usize().unwrap()]; + match self.con.borrow_mut().as_mut() { + Some(v) => match v.read_exact(&mut buffer) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }, + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_bytes(buffer)) + } } fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result { @@ -274,61 +312,6 @@ fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result PyResult { - arg_check!( - vm, - args, - required = [(_zelf, None), (_num, Some(vm.ctx.int_type()))] - ); - Ok(vm.get_none()) -} - -fn socket_accept(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - - let socket = get_socket(zelf); - - let ret = match socket.con.borrow_mut().as_mut() { - Some(v) => v.accept(), - None => return Err(vm.new_type_error("".to_string())), - }; - - let (tcp_stream, addr) = match ret { - Ok((socket, addr)) => (socket, addr), - Err(s) => return Err(vm.new_os_error(s.to_string())), - }; - - let socket = Socket { - address_family: socket.address_family, - socket_kind: socket.socket_kind, - con: RefCell::new(Some(Connection::TcpStream(tcp_stream))), - } - .into_ref(vm); - - let addr_tuple = get_addr_tuple(vm, addr)?; - - Ok(vm.ctx.new_tuple(vec![socket.into_object(), addr_tuple])) -} - -fn socket_recv(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bufsize, Some(vm.ctx.int_type()))] - ); - let socket = get_socket(zelf); - - let mut buffer = vec![0u8; objint::get_value(bufsize).to_usize().unwrap()]; - match socket.con.borrow_mut().as_mut() { - Some(v) => match v.read_exact(&mut buffer) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - }, - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.ctx.new_bytes(buffer)) -} - fn socket_recvfrom(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -420,11 +403,11 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let socket = py_class!(ctx, "socket", ctx.object(), { "__new__" => ctx.new_rustfunc(SocketRef::new), "connect" => ctx.new_rustfunc(SocketRef::connect), - "recv" => ctx.new_rustfunc(socket_recv), + "recv" => ctx.new_rustfunc(SocketRef::recv), "send" => ctx.new_rustfunc(socket_send), "bind" => ctx.new_rustfunc(SocketRef::bind), - "accept" => ctx.new_rustfunc(socket_accept), - "listen" => ctx.new_rustfunc(socket_listen), + "accept" => ctx.new_rustfunc(SocketRef::accept), + "listen" => ctx.new_rustfunc(SocketRef::listen), "close" => ctx.new_rustfunc(socket_close), "getsockname" => ctx.new_rustfunc(socket_getsockname), "sendto" => ctx.new_rustfunc(SocketRef::sendto), From 1f733932558611219a09f843469cc4fe2b37b34b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 17:35:20 +0300 Subject: [PATCH 342/658] Move more socket methods to new arg style --- vm/src/stdlib/socket.rs | 201 +++++++++++++++++----------------------- 1 file changed, 85 insertions(+), 116 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index dc165d316..4082fca4f 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -3,10 +3,8 @@ use std::io; use std::io::Read; use std::io::Write; use std::net::{SocketAddr, TcpListener, TcpStream, ToSocketAddrs, UdpSocket}; -use std::ops::Deref; use crate::function::PyFuncArgs; -use crate::obj::objbytes; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; @@ -163,10 +161,6 @@ impl Socket { } } -fn get_socket<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap() -} - type SocketRef = PyRef; impl SocketRef { @@ -226,33 +220,6 @@ impl SocketRef { } } - fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { - let address_string = get_address_string(vm, address)?; - - match self.socket_kind { - SocketKind::Dgram => { - if let Some(v) = self.con.borrow().as_ref() { - return match v.send_to(&bytes, address_string) { - Ok(_) => Ok(vm.get_none()), - Err(s) => Err(vm.new_os_error(s.to_string())), - }; - } - // Doing implicit bind - match UdpSocket::bind("0.0.0.0:0") { - Ok(dgram) => match dgram.send_to(&bytes, address_string) { - Ok(_) => { - self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - Err(s) => Err(vm.new_os_error(s.to_string())), - } - } - _ => Err(vm.new_not_implemented_error("".to_string())), - } - } - fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) -> () {} fn accept(self, vm: &VirtualMachine) -> PyResult { @@ -289,6 +256,86 @@ impl SocketRef { }; Ok(vm.ctx.new_bytes(buffer)) } + + fn recvfrom(self, bufsize: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; bufsize.as_bigint().to_usize().unwrap()]; + let ret = match self.con.borrow().as_ref() { + Some(v) => v.recv_from(&mut buffer), + None => return Err(vm.new_type_error("".to_string())), + }; + + let addr = match ret { + Ok((_size, addr)) => addr, + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + let addr_tuple = get_addr_tuple(vm, addr)?; + + Ok(vm.ctx.new_tuple(vec![vm.ctx.new_bytes(buffer), addr_tuple])) + } + + fn send(self, bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + match self.con.borrow_mut().as_mut() { + Some(v) => match v.write(&bytes) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }, + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.get_none()) + } + + fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + let address_string = get_address_string(vm, address)?; + + match self.socket_kind { + SocketKind::Dgram => { + if let Some(v) = self.con.borrow().as_ref() { + return match v.send_to(&bytes, address_string) { + Ok(_) => Ok(vm.get_none()), + Err(s) => Err(vm.new_os_error(s.to_string())), + }; + } + // Doing implicit bind + match UdpSocket::bind("0.0.0.0:0") { + Ok(dgram) => match dgram.send_to(&bytes, address_string) { + Ok(_) => { + self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); + Ok(vm.get_none()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + Err(s) => Err(vm.new_os_error(s.to_string())), + } + } + _ => Err(vm.new_not_implemented_error("".to_string())), + } + } + + fn close(self, vm: &VirtualMachine) -> PyResult { + self.con.borrow_mut().take(); + Ok(vm.get_none()) + } + + fn fileno(self, vm: &VirtualMachine) -> PyResult { + let fileno = match self.con.borrow_mut().as_mut() { + Some(v) => v.fileno(), + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_int(fileno)) + } + + fn getsockname(self, vm: &VirtualMachine) -> PyResult { + let addr = match self.con.borrow().as_ref() { + Some(v) => v.local_addr(), + None => return Err(vm.new_type_error("".to_string())), + }; + + match addr { + Ok(addr) => get_addr_tuple(vm, addr), + Err(s) => Err(vm.new_os_error(s.to_string())), + } + } } fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result { @@ -312,84 +359,6 @@ fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bufsize, Some(vm.ctx.int_type()))] - ); - - let socket = get_socket(zelf); - - let mut buffer = vec![0u8; objint::get_value(bufsize).to_usize().unwrap()]; - let ret = match socket.con.borrow().as_ref() { - Some(v) => v.recv_from(&mut buffer), - None => return Err(vm.new_type_error("".to_string())), - }; - - let addr = match ret { - Ok((_size, addr)) => addr, - Err(s) => return Err(vm.new_os_error(s.to_string())), - }; - - let addr_tuple = get_addr_tuple(vm, addr)?; - - Ok(vm.ctx.new_tuple(vec![vm.ctx.new_bytes(buffer), addr_tuple])) -} - -fn socket_send(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bytes, Some(vm.ctx.bytes_type()))] - ); - let socket = get_socket(zelf); - - match socket.con.borrow_mut().as_mut() { - Some(v) => match v.write(&objbytes::get_value(&bytes)) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - }, - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.get_none()) -} - -fn socket_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - - let socket = get_socket(zelf); - socket.con.borrow_mut().take(); - Ok(vm.get_none()) -} - -fn socket_fileno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - - let socket = get_socket(zelf); - - let fileno = match socket.con.borrow_mut().as_mut() { - Some(v) => v.fileno(), - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.ctx.new_int(fileno)) -} - -fn socket_getsockname(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - let socket = get_socket(zelf); - - let addr = match socket.con.borrow().as_ref() { - Some(v) => v.local_addr(), - None => return Err(vm.new_type_error("".to_string())), - }; - - match addr { - Ok(addr) => get_addr_tuple(vm, addr), - Err(s) => Err(vm.new_os_error(s.to_string())), - } -} - fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { let port = vm.ctx.new_int(addr.port()); let ip = vm.ctx.new_str(addr.ip().to_string()); @@ -404,15 +373,15 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "__new__" => ctx.new_rustfunc(SocketRef::new), "connect" => ctx.new_rustfunc(SocketRef::connect), "recv" => ctx.new_rustfunc(SocketRef::recv), - "send" => ctx.new_rustfunc(socket_send), + "send" => ctx.new_rustfunc(SocketRef::send), "bind" => ctx.new_rustfunc(SocketRef::bind), "accept" => ctx.new_rustfunc(SocketRef::accept), "listen" => ctx.new_rustfunc(SocketRef::listen), - "close" => ctx.new_rustfunc(socket_close), - "getsockname" => ctx.new_rustfunc(socket_getsockname), + "close" => ctx.new_rustfunc(SocketRef::close), + "getsockname" => ctx.new_rustfunc(SocketRef::getsockname), "sendto" => ctx.new_rustfunc(SocketRef::sendto), - "recvfrom" => ctx.new_rustfunc(socket_recvfrom), - "fileno" => ctx.new_rustfunc(socket_fileno), + "recvfrom" => ctx.new_rustfunc(SocketRef::recvfrom), + "fileno" => ctx.new_rustfunc(SocketRef::fileno), }); py_module!(vm, "socket", { From b61cd6f011d79ea2c2583ab39113309dbf070a78 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 21 Apr 2019 17:44:41 +0300 Subject: [PATCH 343/658] Simplify return values --- vm/src/stdlib/socket.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 4082fca4f..25edf7a65 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -173,21 +173,21 @@ impl SocketRef { Socket::new(family, kind).into_ref_with_type(vm, cls) } - fn connect(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + fn connect(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { let address_string = get_address_string(vm, address)?; match self.socket_kind { SocketKind::Stream => match TcpStream::connect(address_string) { Ok(stream) => { self.con.borrow_mut().replace(Connection::TcpStream(stream)); - Ok(vm.get_none()) + Ok(()) } Err(s) => Err(vm.new_os_error(s.to_string())), }, SocketKind::Dgram => { if let Some(Connection::UdpSocket(con)) = self.con.borrow().as_ref() { match con.connect(address_string) { - Ok(_) => Ok(vm.get_none()), + Ok(_) => Ok(()), Err(s) => Err(vm.new_os_error(s.to_string())), } } else { @@ -197,7 +197,7 @@ impl SocketRef { } } - fn bind(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + fn bind(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { let address_string = get_address_string(vm, address)?; match self.socket_kind { @@ -206,14 +206,14 @@ impl SocketRef { self.con .borrow_mut() .replace(Connection::TcpListener(stream)); - Ok(vm.get_none()) + Ok(()) } Err(s) => Err(vm.new_os_error(s.to_string())), }, SocketKind::Dgram => match UdpSocket::bind(address_string) { Ok(dgram) => { self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) + Ok(()) } Err(s) => Err(vm.new_os_error(s.to_string())), }, @@ -274,7 +274,7 @@ impl SocketRef { Ok(vm.ctx.new_tuple(vec![vm.ctx.new_bytes(buffer), addr_tuple])) } - fn send(self, bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + fn send(self, bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> { match self.con.borrow_mut().as_mut() { Some(v) => match v.write(&bytes) { Ok(_) => (), @@ -282,17 +282,17 @@ impl SocketRef { }, None => return Err(vm.new_type_error("".to_string())), }; - Ok(vm.get_none()) + Ok(()) } - fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult { + fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { let address_string = get_address_string(vm, address)?; match self.socket_kind { SocketKind::Dgram => { if let Some(v) = self.con.borrow().as_ref() { return match v.send_to(&bytes, address_string) { - Ok(_) => Ok(vm.get_none()), + Ok(_) => Ok(()), Err(s) => Err(vm.new_os_error(s.to_string())), }; } @@ -301,7 +301,7 @@ impl SocketRef { Ok(dgram) => match dgram.send_to(&bytes, address_string) { Ok(_) => { self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) + Ok(()) } Err(s) => Err(vm.new_os_error(s.to_string())), }, @@ -312,9 +312,8 @@ impl SocketRef { } } - fn close(self, vm: &VirtualMachine) -> PyResult { + fn close(self, _vm: &VirtualMachine) -> () { self.con.borrow_mut().take(); - Ok(vm.get_none()) } fn fileno(self, vm: &VirtualMachine) -> PyResult { From 87f9a8e634c79d99ffd06542a91bfe095e6ac846 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Sun, 21 Apr 2019 23:48:13 +0900 Subject: [PATCH 344/658] Implement deleting tuples --- tests/snippets/delete.py | 3 +++ vm/src/compile.rs | 58 +++++++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/snippets/delete.py b/tests/snippets/delete.py index 078b6dcd9..29f8719fd 100644 --- a/tests/snippets/delete.py +++ b/tests/snippets/delete.py @@ -10,3 +10,6 @@ del foo.bar assert not hasattr(foo, 'bar') +x = 1 +y = 2 +del (x, y) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index e7953ad67..b7df81313 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -456,30 +456,7 @@ impl Compiler { } ast::Statement::Delete { targets } => { for target in targets { - match target { - ast::Expression::Identifier { name } => { - self.emit(Instruction::DeleteName { - name: name.to_string(), - }); - } - ast::Expression::Attribute { value, name } => { - self.compile_expression(value)?; - self.emit(Instruction::DeleteAttr { - name: name.to_string(), - }); - } - ast::Expression::Subscript { a, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - self.emit(Instruction::DeleteSubscript); - } - _ => { - return Err(CompileError { - error: CompileErrorType::Delete(target.name()), - location: self.current_source_location.clone(), - }); - } - } + self.compile_delete(target)?; } } ast::Statement::Pass => { @@ -489,6 +466,39 @@ impl Compiler { Ok(()) } + fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> { + match expression { + ast::Expression::Identifier { name } => { + self.emit(Instruction::DeleteName { + name: name.to_string(), + }); + } + ast::Expression::Attribute { value, name } => { + self.compile_expression(value)?; + self.emit(Instruction::DeleteAttr { + name: name.to_string(), + }); + } + ast::Expression::Subscript { a, b } => { + self.compile_expression(a)?; + self.compile_expression(b)?; + self.emit(Instruction::DeleteSubscript); + } + ast::Expression::Tuple { elements } => { + for element in elements { + self.compile_delete(element)?; + } + } + _ => { + return Err(CompileError { + error: CompileErrorType::Delete(expression.name()), + location: self.current_source_location.clone(), + }); + } + } + Ok(()) + } + fn enter_function( &mut self, name: &str, From 2aeb0ad59b2f05dbcdd782631eb4ca9e0f0e143d Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Mon, 22 Apr 2019 00:26:30 +0900 Subject: [PATCH 345/658] Assert deleted variables are really deleted --- tests/snippets/delete.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/snippets/delete.py b/tests/snippets/delete.py index 29f8719fd..57d432ff6 100644 --- a/tests/snippets/delete.py +++ b/tests/snippets/delete.py @@ -1,3 +1,4 @@ +from testutils import assert_raises a = 1 del a @@ -13,3 +14,5 @@ assert not hasattr(foo, 'bar') x = 1 y = 2 del (x, y) +assert_raises(NameError, lambda: x) +assert_raises(NameError, lambda: y) From 801c8646451625c4792d32c87256d41b91c58460 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 21 Apr 2019 11:30:40 -0500 Subject: [PATCH 346/658] Bump WASM npm version --- Cargo.lock | 2 +- wasm/lib/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73fd4ba76..33e9ddcfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,7 +853,7 @@ dependencies = [ [[package]] name = "rustpython_wasm" -version = "0.1.0-pre-alpha.1" +version = "0.1.0-pre-alpha.2" dependencies = [ "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 769455aea..cebc753a8 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython_wasm" -version = "0.1.0-pre-alpha.1" +version = "0.1.0-pre-alpha.2" authors = ["Ryan Liddle "] license = "MIT" description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to WASM" From ce8ea805ae8abda95f99892c920f25caad76a826 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 21 Apr 2019 19:40:14 +0200 Subject: [PATCH 347/658] Add check for nonlocal on top level. --- tests/snippets/global_nonlocal.py | 35 +++++++++++++++++++ vm/src/compile.rs | 8 ++--- vm/src/error.rs | 2 ++ vm/src/symboltable.rs | 57 ++++++++++++++++++++++++++----- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py index fd5a61e1c..03e0d386e 100644 --- a/tests/snippets/global_nonlocal.py +++ b/tests/snippets/global_nonlocal.py @@ -11,6 +11,7 @@ assert a == 2 b() assert a == 4 + def x(): def y(): nonlocal b @@ -21,3 +22,37 @@ def x(): res = x() assert res == 3, str(res) + +# Invalid syntax: +src = """ +b = 2 +global b +""" + +try: + exec(src) +except Exception as ex: + # print(ex) + pass +else: + raise RuntimeError('Must raise') + +# Invalid syntax: + +src = """ +nonlocal c +""" + +try: + exec(src) +except Exception as ex: + # print(ex) + pass +else: + raise RuntimeError('Must raise') + + +# class X: +# nonlocal c +# c = 2 + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index b7df81313..44f064456 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -41,17 +41,17 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source)?; - let symbol_table = make_symbol_table(&ast); + let symbol_table = make_symbol_table(&ast)?; compiler.compile_program(&ast, symbol_table) } Mode::Eval => { let statement = parser::parse_statement(source)?; - let symbol_table = statements_to_symbol_table(&statement); + let symbol_table = statements_to_symbol_table(&statement)?; compiler.compile_statement_eval(&statement, symbol_table) } Mode::Single => { let ast = parser::parse_program(source)?; - let symbol_table = make_symbol_table(&ast); + let symbol_table = make_symbol_table(&ast)?; compiler.compile_program_single(&ast, symbol_table) } }?; @@ -1776,7 +1776,7 @@ mod tests { compiler.source_path = Some("source_path".to_string()); compiler.push_new_code_object("".to_string()); let ast = parser::parse_program(&source.to_string()).unwrap(); - let symbol_scope = make_symbol_table(&ast); + let symbol_scope = make_symbol_table(&ast).unwrap(); compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() } diff --git a/vm/src/error.rs b/vm/src/error.rs index 81836fa44..af763168d 100644 --- a/vm/src/error.rs +++ b/vm/src/error.rs @@ -29,6 +29,7 @@ pub enum CompileErrorType { ExpectExpr, /// Parser error Parse(ParseError), + SyntaxError(String), /// Multiple `*` detected StarArgs, /// Break statement outside of loop. @@ -46,6 +47,7 @@ impl fmt::Display for CompileError { CompileErrorType::Delete(target) => write!(f, "can't delete {}", target), CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"), CompileErrorType::Parse(err) => write!(f, "{}", err), + CompileErrorType::SyntaxError(err) => write!(f, "{}", err), CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"), CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"), CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"), diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index d7f30c870..ca10382e4 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -5,23 +5,27 @@ Then the compiler can use the symbol table to generate proper load and store instructions for names. */ +use crate::error::{CompileError, CompileErrorType}; use rustpython_parser::ast; +use rustpython_parser::lexer::Location; use std::collections::HashMap; -pub fn make_symbol_table(program: &ast::Program) -> SymbolScope { +pub fn make_symbol_table(program: &ast::Program) -> Result { let mut builder = SymbolTableBuilder::new(); builder.enter_scope(); - builder.scan_program(program).unwrap(); + builder.scan_program(program)?; assert!(builder.scopes.len() == 1); - builder.scopes.pop().unwrap() + Ok(builder.scopes.pop().unwrap()) } -pub fn statements_to_symbol_table(statements: &[ast::LocatedStatement]) -> SymbolScope { +pub fn statements_to_symbol_table( + statements: &[ast::LocatedStatement], +) -> Result { let mut builder = SymbolTableBuilder::new(); builder.enter_scope(); - builder.scan_statements(statements).unwrap(); + builder.scan_statements(statements)?; assert!(builder.scopes.len() == 1); - builder.scopes.pop().unwrap() + Ok(builder.scopes.pop().unwrap()) } #[derive(Debug)] @@ -43,7 +47,22 @@ pub struct SymbolScope { pub sub_scopes: Vec, } -type SymbolTableResult = Result<(), String>; +#[derive(Debug)] +pub struct SymbolTableError { + error: String, + location: Location, +} + +impl From for CompileError { + fn from(error: SymbolTableError) -> Self { + CompileError { + error: CompileErrorType::SyntaxError(error.error), + location: error.location, + } + } +} + +type SymbolTableResult = Result<(), SymbolTableError>; impl SymbolScope { pub fn new() -> Self { @@ -85,7 +104,7 @@ impl SymbolTableBuilder { self.scopes.push(scope); } - pub fn leave_scope(&mut self) { + fn leave_scope(&mut self) { // Pop scope and add to subscopes of parent scope. let scope = self.scopes.pop().unwrap(); self.scopes.last_mut().unwrap().sub_scopes.push(scope); @@ -445,17 +464,37 @@ impl SymbolTableBuilder { } fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { + let scope_depth = self.scopes.len(); let current_scope = self.scopes.last_mut().unwrap(); + let location = Default::default(); if let Some(_old_role) = current_scope.symbols.get(name) { // Role already set.. // debug!("TODO: {:?}", old_role); match role { - SymbolRole::Global => return Err("global must appear first".to_string()), + SymbolRole::Global => { + return Err(SymbolTableError { + error: format!("name '{}' is used prior to global declaration", name), + location, + }) + } _ => { // Ok? } } } else { + match role { + SymbolRole::Nonlocal => { + if scope_depth < 2 { + return Err(SymbolTableError { + error: format!("name '{}' is used prior to global declaration", name), + location, + }); + } + } + _ => { + // Ok! + } + } current_scope.symbols.insert(name.to_string(), role); } Ok(()) From cb322019047ede2014e848e1b7d06d213b1de8fb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Apr 2019 00:18:19 +0300 Subject: [PATCH 348/658] Add Address --- vm/src/stdlib/socket.rs | 61 +++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 25edf7a65..edab574b3 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -4,11 +4,9 @@ use std::io::Read; use std::io::Write; use std::net::{SocketAddr, TcpListener, TcpStream, ToSocketAddrs, UdpSocket}; -use crate::function::PyFuncArgs; use crate::obj::objbytes::PyBytesRef; -use crate::obj::objint; use crate::obj::objint::PyIntRef; -use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; use crate::obj::objtuple::PyTupleRef; use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; @@ -173,8 +171,8 @@ impl SocketRef { Socket::new(family, kind).into_ref_with_type(vm, cls) } - fn connect(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let address_string = get_address_string(vm, address)?; + fn connect(self, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); match self.socket_kind { SocketKind::Stream => match TcpStream::connect(address_string) { @@ -197,8 +195,8 @@ impl SocketRef { } } - fn bind(self, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let address_string = get_address_string(vm, address)?; + fn bind(self, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); match self.socket_kind { SocketKind::Stream => match TcpListener::bind(address_string) { @@ -285,8 +283,8 @@ impl SocketRef { Ok(()) } - fn sendto(self, bytes: PyBytesRef, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let address_string = get_address_string(vm, address)?; + fn sendto(self, bytes: PyBytesRef, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); match self.socket_kind { SocketKind::Dgram => { @@ -337,25 +335,34 @@ impl SocketRef { } } -fn get_address_string(vm: &VirtualMachine, address: PyTupleRef) -> Result { - let args = PyFuncArgs { - args: address.elements.borrow().to_vec(), - kwargs: vec![], - }; - arg_check!( - vm, - args, - required = [ - (host, Some(vm.ctx.str_type())), - (port, Some(vm.ctx.int_type())) - ] - ); +struct Address { + host: String, + port: usize, +} - Ok(format!( - "{}:{}", - objstr::get_value(host), - objint::get_value(port).to_string() - )) +impl Address { + fn get_address_string(self) -> String { + format!("{}:{}", self.host, self.port.to_string()) + } +} + +impl TryFromObject for Address { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let tuple = PyTupleRef::try_from_object(vm, obj)?; + if tuple.elements.borrow().len() != 2 { + Err(vm.new_type_error("Address tuple should have only 2 values".to_string())) + } else { + Ok(Address { + host: PyStringRef::try_from_object(vm, tuple.elements.borrow()[0].clone())? + .value + .to_string(), + port: PyIntRef::try_from_object(vm, tuple.elements.borrow()[1].clone())? + .as_bigint() + .to_usize() + .unwrap(), + }) + } + } } fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { From 9f7676589fb62ea8cdaf1e8ab0381bf0ec80f558 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Apr 2019 21:30:57 +0900 Subject: [PATCH 349/658] Fix README not to refer moved file `cargo run tests/snippets/whats_left_to_implement.py` -> `./whats_left.sh` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3febdc13c..3c64397b8 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Another approach is to checkout the source code: builtin functions and object me and easiest way to contribute. You can also simply run -`cargo run tests/snippets/whats_left_to_implement.py` to assist in finding any +`./whats_left.sh` to assist in finding any unimplemented method. # Testing From 384bf5b5451d78e5933b1eda2de7e6b943a56904 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Apr 2019 18:27:39 +0300 Subject: [PATCH 350/658] Add os.stat --- tests/snippets/stdlib_os.py | 3 ++ vm/src/stdlib/os.rs | 58 ++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 0de5b52b6..cae8d8c8e 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -98,3 +98,6 @@ with TestWithTempDir() as tmpdir: assert paths == set([fname, fname2, folder]) assert dirs == set([FOLDER]) assert files == set([FILE_NAME, FILE_NAME2]) + + # Stat + stat_res = os.stat(fname) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 9717c8f22..7506a9987 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -233,7 +233,7 @@ impl DirEntryRef { } #[derive(Debug)] -pub struct ScandirIterator { +struct ScandirIterator { entries: RefCell, } @@ -272,6 +272,56 @@ fn os_scandir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +#[derive(Debug)] +struct StatResult { + st_mode: u32, +} + +impl PyValue for StatResult { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("os", "stat_result") + } +} + +type StatResultRef = PyRef; + +impl StatResultRef { + fn st_mode(self, _vm: &VirtualMachine) -> u32 { + self.st_mode + } +} + +#[cfg(unix)] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + use std::os::linux::fs::MetadataExt; + match fs::metadata(&path.value) { + Ok(meta) => Ok(StatResult { + st_mode: meta.st_mode(), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + +#[cfg(windows)] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + use std::os::windows::fs::MetadataExt; + match fs::metadata(&path.value) { + Ok(meta) => Ok(StatResult { + st_mode: meta.file_attributes(), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + +#[cfg(all(not(unix), not(windows)))] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + unimplemented!(); +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -295,6 +345,10 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "is_file" => ctx.new_rustfunc(DirEntryRef::is_file), }); + let stat_result = py_class!(ctx, "stat_result", ctx.object(), { + "st_mode" => ctx.new_property(StatResultRef::st_mode) + }); + py_module!(vm, "_os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), @@ -314,6 +368,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "scandir" => ctx.new_rustfunc(os_scandir), "ScandirIter" => scandir_iter, "DirEntry" => dir_entry, + "stat_result" => stat_result, + "stat" => ctx.new_rustfunc(os_stat), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From 9a1caa08c373eac1b7e89ae5d1fb541475685d02 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Apr 2019 18:35:38 +0300 Subject: [PATCH 351/658] Add stat_result.{st_ino, st_dev} --- tests/snippets/stdlib_os.py | 3 +++ vm/src/stdlib/os.rs | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index cae8d8c8e..405f999a6 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -101,3 +101,6 @@ with TestWithTempDir() as tmpdir: # Stat stat_res = os.stat(fname) + print(stat_res.st_mode) + print(stat_res.st_ino) + print(stat_res.st_dev) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 7506a9987..a2c55bd8f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -275,6 +275,8 @@ fn os_scandir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { #[derive(Debug)] struct StatResult { st_mode: u32, + st_ino: u64, + st_dev: u64, } impl PyValue for StatResult { @@ -289,6 +291,14 @@ impl StatResultRef { fn st_mode(self, _vm: &VirtualMachine) -> u32 { self.st_mode } + + fn st_ino(self, _vm: &VirtualMachine) -> u64 { + self.st_ino + } + + fn st_dev(self, _vm: &VirtualMachine) -> u64 { + self.st_dev + } } #[cfg(unix)] @@ -297,6 +307,8 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { match fs::metadata(&path.value) { Ok(meta) => Ok(StatResult { st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), } .into_ref(vm) .into_object()), @@ -310,6 +322,8 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { match fs::metadata(&path.value) { Ok(meta) => Ok(StatResult { st_mode: meta.file_attributes(), + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. } .into_ref(vm) .into_object()), @@ -346,7 +360,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); let stat_result = py_class!(ctx, "stat_result", ctx.object(), { - "st_mode" => ctx.new_property(StatResultRef::st_mode) + "st_mode" => ctx.new_property(StatResultRef::st_mode), + "st_ino" => ctx.new_property(StatResultRef::st_ino), + "st_dev" => ctx.new_property(StatResultRef::st_dev), }); py_module!(vm, "_os", { From 56958552f48ea63dc7fff8b9436646bcb49c839f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Apr 2019 18:47:10 +0300 Subject: [PATCH 352/658] Add stat_result.{st_nlink, st_uid, st_gid} --- tests/snippets/stdlib_os.py | 3 +++ vm/src/stdlib/os.rs | 28 ++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 405f999a6..cca39e3bf 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -104,3 +104,6 @@ with TestWithTempDir() as tmpdir: print(stat_res.st_mode) print(stat_res.st_ino) print(stat_res.st_dev) + print(stat_res.st_nlink) + print(stat_res.st_uid) + print(stat_res.st_gid) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a2c55bd8f..49eff7dd2 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -277,6 +277,9 @@ struct StatResult { st_mode: u32, st_ino: u64, st_dev: u64, + st_nlink: u64, + st_uid: u32, + st_gid: u32, } impl PyValue for StatResult { @@ -299,6 +302,18 @@ impl StatResultRef { fn st_dev(self, _vm: &VirtualMachine) -> u64 { self.st_dev } + + fn st_nlink(self, _vm: &VirtualMachine) -> u64 { + self.st_nlink + } + + fn st_uid(self, _vm: &VirtualMachine) -> u32 { + self.st_uid + } + + fn st_gid(self, _vm: &VirtualMachine) -> u32 { + self.st_gid + } } #[cfg(unix)] @@ -309,6 +324,9 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { st_mode: meta.st_mode(), st_ino: meta.st_ino(), st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), } .into_ref(vm) .into_object()), @@ -322,8 +340,11 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { match fs::metadata(&path.value) { Ok(meta) => Ok(StatResult { st_mode: meta.file_attributes(), - st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_uid: 0, // 0 on windows + st_gid: 0, // 0 on windows } .into_ref(vm) .into_object()), @@ -363,6 +384,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "st_mode" => ctx.new_property(StatResultRef::st_mode), "st_ino" => ctx.new_property(StatResultRef::st_ino), "st_dev" => ctx.new_property(StatResultRef::st_dev), + "st_nlink" => ctx.new_property(StatResultRef::st_nlink), + "st_uid" => ctx.new_property(StatResultRef::st_uid), + "st_gid" => ctx.new_property(StatResultRef::st_gid), }); py_module!(vm, "_os", { From 55bd00b6009012e695d6525cce9f0c0d1daea72b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Apr 2019 18:52:35 +0300 Subject: [PATCH 353/658] Add stat_result.{st_size} --- tests/snippets/stdlib_os.py | 2 ++ vm/src/stdlib/os.rs | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index cca39e3bf..0fb44cbb5 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -107,3 +107,5 @@ with TestWithTempDir() as tmpdir: print(stat_res.st_nlink) print(stat_res.st_uid) print(stat_res.st_gid) + print(stat_res.st_size) + assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 49eff7dd2..796578bb1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -280,6 +280,7 @@ struct StatResult { st_nlink: u64, st_uid: u32, st_gid: u32, + st_size: u64, } impl PyValue for StatResult { @@ -314,6 +315,10 @@ impl StatResultRef { fn st_gid(self, _vm: &VirtualMachine) -> u32 { self.st_gid } + + fn st_size(self, _vm: &VirtualMachine) -> u64 { + self.st_size + } } #[cfg(unix)] @@ -327,6 +332,7 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { st_nlink: meta.st_nlink(), st_uid: meta.st_uid(), st_gid: meta.st_gid(), + st_size: meta.st_size(), } .into_ref(vm) .into_object()), @@ -345,6 +351,7 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. st_uid: 0, // 0 on windows st_gid: 0, // 0 on windows + st_size: meta.file_size(), } .into_ref(vm) .into_object()), @@ -387,6 +394,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "st_nlink" => ctx.new_property(StatResultRef::st_nlink), "st_uid" => ctx.new_property(StatResultRef::st_uid), "st_gid" => ctx.new_property(StatResultRef::st_gid), + "st_size" => ctx.new_property(StatResultRef::st_size), }); py_module!(vm, "_os", { From 1d5adc4353622d08daa63b46894a8cf97b83fe59 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 23 Apr 2019 00:17:42 +0900 Subject: [PATCH 354/658] Add built-in dict.pop --- tests/snippets/dict.py | 8 ++++++++ vm/src/dictdatatype.rs | 27 ++++++++++++++++++++++----- vm/src/obj/objdict.rs | 5 +++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 05cdd9928..774ca0fd7 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -166,3 +166,11 @@ assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} y.update(y) assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed + +x = {1: 'a', '1': None} +assert x.pop(1) == 'a' +assert x.pop('1') is None +assert x == {} + +with assertRaises(KeyError): + x.pop("not here") diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 1f4b4eecd..b0bd4894f 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -87,14 +87,18 @@ impl Dict { } } + fn unchecked_get(&self, index: usize) -> T { + if let Some(entry) = &self.entries[index] { + entry.value.clone() + } else { + panic!("Lookup returned invalid index into entries!"); + } + } + /// Retrieve a key pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { - if let Some(entry) = &self.entries[index] { - Ok(Some(entry.value.clone())) - } else { - panic!("Lookup returned invalid index into entries!"); - } + Ok(Some(self.unchecked_get(index))) } else { Ok(None) } @@ -190,6 +194,19 @@ impl Dict { // warn!("Perturb value: {}", i); } } + + /// Retrieve and delete a key + pub fn pop(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + if let LookupResult::Existing(index) = self.lookup(vm, key)? { + let value = self.unchecked_get(index); + self.entries[index] = None; + self.size -= 1; + Ok(value) + } else { + let key_repr = vm.to_pystr(key)?; + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + } + } } enum LookupResult { diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e5419e932..306040f39 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -228,6 +228,10 @@ impl PyDictRef { PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) } + fn pop(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow_mut().pop(vm, &key) + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -458,6 +462,7 @@ pub fn init(context: &PyContext) { "get" => context.new_rustfunc(PyDictRef::get), "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), + "pop" => context.new_rustfunc(PyDictRef::pop), }); PyDictKeys::extend_class(context, &context.dictkeys_type); From 2eb97aa1e815c852922e13f96771671fe7df01a5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 23 Apr 2019 01:23:27 +0900 Subject: [PATCH 355/658] Add __main__.py support --- src/main.rs | 33 +++++++++++++++++++++++++---- tests/snippets/dir_main/__main__.py | 1 + 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 tests/snippets/dir_main/__main__.py diff --git a/src/main.rs b/src/main.rs index 24b46821b..98fd5d221 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,7 +14,7 @@ use rustpython_vm::{ print_exception, pyobject::PyResult, util, VirtualMachine, }; use rustyline::{error::ReadlineError, Editor}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; fn main() { env_logger::init(); @@ -96,11 +96,36 @@ fn run_module(vm: &VirtualMachine, module: &str) -> PyResult { fn run_script(vm: &VirtualMachine, script_file: &str) -> PyResult { debug!("Running file {}", script_file); // Parse an ast from it: - let file_path = Path::new(script_file); - match util::read_file(file_path) { + let file_path = PathBuf::from(script_file); + let file_path = if file_path.is_file() { + file_path + } else if file_path.is_dir() { + let main_file_path = file_path.join("__main__.py"); + if main_file_path.is_file() { + main_file_path + } else { + error!( + "can't find '__main__' module in '{}'", + file_path.to_str().unwrap() + ); + std::process::exit(1); + } + } else { + error!( + "can't open file '{}': No such file or directory", + file_path.to_str().unwrap() + ); + std::process::exit(1); + }; + + match util::read_file(&file_path) { Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()), Err(err) => { - error!("Failed reading file: {:?}", err.kind()); + error!( + "Failed reading file '{}': {:?}", + file_path.to_str().unwrap(), + err.kind() + ); std::process::exit(1); } } diff --git a/tests/snippets/dir_main/__main__.py b/tests/snippets/dir_main/__main__.py new file mode 100644 index 000000000..c324c2e6e --- /dev/null +++ b/tests/snippets/dir_main/__main__.py @@ -0,0 +1 @@ +print('Hello') From 29c0d890334383bf7fd25b1a80f455758d7407b4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 24 Apr 2019 02:38:00 +0900 Subject: [PATCH 356/658] test_run_script for filr/dir --- src/main.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main.rs b/src/main.rs index 98fd5d221..5620d851e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -131,6 +131,19 @@ fn run_script(vm: &VirtualMachine, script_file: &str) -> PyResult { } } +#[test] +fn test_run_script() { + let vm = VirtualMachine::new(); + + // test file run + let r = run_script(&vm, "tests/snippets/dir_main/__main__.py"); + assert!(r.is_ok()); + + // test module run + let r = run_script(&vm, "tests/snippets/dir_main"); + assert!(r.is_ok()); +} + fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> { match compile::compile(vm, source, &compile::Mode::Single, "".to_string()) { Ok(code) => { From f9124f41a81cae25d3f63deffd85da60041d247d Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Wed, 24 Apr 2019 04:00:52 +0900 Subject: [PATCH 357/658] Test integer literals --- tests/snippets/literals.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/snippets/literals.py diff --git a/tests/snippets/literals.py b/tests/snippets/literals.py new file mode 100644 index 000000000..f481b33f5 --- /dev/null +++ b/tests/snippets/literals.py @@ -0,0 +1,8 @@ +# Integer literals +assert 0b101010 == 42 +assert 0B101010 == 42 +assert 0o777 == 511 +assert 0O777 == 511 +assert 0xcafebabe == 3405691582 +assert 0Xcafebabe == 3405691582 +assert 0xCAFEBABE == 3405691582 From fb62653c5c54a62b39de3ccb54039546e2dd66a9 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Wed, 24 Apr 2019 19:28:21 +0900 Subject: [PATCH 358/658] Test error cases of comparisons --- tests/snippets/comparisons.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/snippets/comparisons.py b/tests/snippets/comparisons.py index 1a4597ef6..d68dcaf3e 100644 --- a/tests/snippets/comparisons.py +++ b/tests/snippets/comparisons.py @@ -1,3 +1,4 @@ +from testutils import assert_raises assert 1 < 2 assert 1 < 2 < 3 @@ -11,3 +12,12 @@ assert not 5 == 5 != 5 == 5 assert not 1 < 2 < 3 > 4 assert not 1 < 2 > 3 < 4 assert not 1 > 2 < 3 < 4 + +def test_type_error(x, y): + assert_raises(TypeError, lambda: x < y) + assert_raises(TypeError, lambda: x <= y) + assert_raises(TypeError, lambda: x > y) + assert_raises(TypeError, lambda: x >= y) + +test_type_error([], 0) +test_type_error((), 0) From 8982f567b0b1fe81272b296f29b438430b1ba691 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 24 Apr 2019 15:19:52 +0200 Subject: [PATCH 359/658] Add nonlocal binding check. --- tests/snippets/global_nonlocal.py | 28 +++++++------ vm/src/symboltable.rs | 70 +++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py index 03e0d386e..ddf5e1ed2 100644 --- a/tests/snippets/global_nonlocal.py +++ b/tests/snippets/global_nonlocal.py @@ -1,3 +1,4 @@ +from testutils import assertRaises # Test global and nonlocal funkyness @@ -29,27 +30,28 @@ b = 2 global b """ -try: +with assertRaises(SyntaxError): exec(src) -except Exception as ex: - # print(ex) - pass -else: - raise RuntimeError('Must raise') # Invalid syntax: - src = """ nonlocal c """ -try: +with assertRaises(SyntaxError): + exec(src) + + +# Invalid syntax: +src = """ +def f(): + def x(): + nonlocal c +c = 2 +""" + +with assertRaises(SyntaxError): exec(src) -except Exception as ex: - # print(ex) - pass -else: - raise RuntimeError('Must raise') # class X: diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index ca10382e4..6bdec1e96 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -3,6 +3,8 @@ This ensures that global and nonlocal keywords are picked up. Then the compiler can use the symbol table to generate proper load and store instructions for names. + +Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c */ use crate::error::{CompileError, CompileErrorType}; @@ -15,7 +17,9 @@ pub fn make_symbol_table(program: &ast::Program) -> Result, +) -> SymbolTableResult { + // println!("Analyzing {:?}, parent={:?} symbols={:?}", symbol_scope, parent_symbol_scope, symbol_scope.symbols); + // Analyze sub scopes: + for sub_scope in &symbol_scope.sub_scopes { + analyze_symbol_table(&sub_scope, Some(symbol_scope))?; + } + + // Analyze symbols: + for (symbol_name, symbol_role) in &symbol_scope.symbols { + analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?; + } + + Ok(()) +} + +fn analyze_symbol( + symbol_name: &str, + symbol_role: &SymbolRole, + parent_symbol_scope: Option<&SymbolScope>, +) -> SymbolTableResult { + match symbol_role { + SymbolRole::Nonlocal => { + // check if name is defined in parent scope! + if let Some(parent_symbol_scope) = parent_symbol_scope { + if !parent_symbol_scope.symbols.contains_key(symbol_name) { + return Err(SymbolTableError { + error: format!("no binding for nonlocal '{}' found", symbol_name), + location: Default::default(), + }); + } + } else { + return Err(SymbolTableError { + error: format!( + "nonlocal {} defined at place without an enclosing scope", + symbol_name + ), + location: Default::default(), + }); + } + } + // TODO: add more checks for globals + _ => {} + } + Ok(()) +} + pub struct SymbolTableBuilder { // Scope stack. pub scopes: Vec, @@ -477,6 +535,12 @@ impl SymbolTableBuilder { location, }) } + SymbolRole::Nonlocal => { + return Err(SymbolTableError { + error: format!("name '{}' is used prior to nonlocal declaration", name), + location, + }) + } _ => { // Ok? } @@ -486,7 +550,7 @@ impl SymbolTableBuilder { SymbolRole::Nonlocal => { if scope_depth < 2 { return Err(SymbolTableError { - error: format!("name '{}' is used prior to global declaration", name), + error: format!("cannot define nonlocal '{}' at top level.", name), location, }); } From d84b8b52a5897b1e8fa2c2a03f43198c443447cd Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 25 Apr 2019 10:37:59 +1200 Subject: [PATCH 360/658] Fix the tuple from rpartition being in wrong order when sub isn't contained within the string --- tests/snippets/strings.py | 2 ++ vm/src/obj/objstr.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index adcc106bb..86fbe69f3 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -132,7 +132,9 @@ assert 'abc\t12345\txyz'.expandtabs() == 'abc 12345 xyz' assert '-'.join(['1', '2', '3']) == '1-2-3' assert 'HALLO'.isupper() assert "hello, my name is".partition("my ") == ('hello, ', 'my ', 'name is') +assert "hello".partition("is") == ('hello', '', '') assert "hello, my name is".rpartition("is") == ('hello, my name ', 'is', '') +assert "hello".rpartition("is") == ('', '', 'hello') assert not ''.isdecimal() assert '123'.isdecimal() assert not '\u00B2'.isdecimal() diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index b52a91409..dd610f4c8 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -635,9 +635,9 @@ impl PyString { new_tup.swap(0, 1); // so it's in the right order new_tup.insert(1, vm.ctx.new_str(sub.clone())); } else { + new_tup.push(vm.ctx.new_str("".to_string())); + new_tup.push(vm.ctx.new_str("".to_string())); new_tup.push(vm.ctx.new_str(value.clone())); - new_tup.push(vm.ctx.new_str("".to_string())); - new_tup.push(vm.ctx.new_str("".to_string())); } vm.ctx.new_tuple(new_tup) } From ceac014a6c4eb8e9b0f5d1c895e43b0eee821537 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 24 Apr 2019 13:32:29 +0100 Subject: [PATCH 361/658] Add dict.popitem. --- tests/snippets/dict.py | 6 ++++++ vm/src/obj/objdict.rs | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 774ca0fd7..488295ed0 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -174,3 +174,9 @@ assert x == {} with assertRaises(KeyError): x.pop("not here") + +x = {1: 'a'} +assert (1, 'a') == x.popitem() +with assertRaises(KeyError): + x.popitem() +assert x == {} diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 306040f39..c3737de15 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -232,6 +232,19 @@ impl PyDictRef { self.entries.borrow_mut().pop(vm, &key) } + fn popitem(self, vm: &VirtualMachine) -> PyResult { + let mut entries = self.entries.borrow_mut(); + let (key, value) = match entries.next_entry(&mut 0) { + Some((key, value)) => (key.clone(), value.clone()), + None => { + return Err(vm.new_key_error("popitem(): dictionary is empty".to_string())); + } + }; + + entries.delete(vm, &key)?; + Ok(vm.ctx.new_tuple(vec![key, value])) + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -463,6 +476,7 @@ pub fn init(context: &PyContext) { "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), "pop" => context.new_rustfunc(PyDictRef::pop), + "popitem" => context.new_rustfunc(PyDictRef::popitem), }); PyDictKeys::extend_class(context, &context.dictkeys_type); From 1c43a6bfbeb1fbbcf9e6f43f31499978ec9be55a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 24 Apr 2019 15:40:05 +0100 Subject: [PATCH 362/658] Add dict.setdefault. --- tests/snippets/dict.py | 8 ++++++++ vm/src/obj/objdict.rs | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 488295ed0..2b52c5f6d 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -180,3 +180,11 @@ assert (1, 'a') == x.popitem() with assertRaises(KeyError): x.popitem() assert x == {} + +x = {'a': 4} +assert 4 == x.setdefault('a', 0) +assert x['a'] == 4 +assert 0 == x.setdefault('b', 0) +assert x['b'] == 0 +assert None == x.setdefault('c') +assert x['c'] is None diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index c3737de15..a9cbcc41f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -213,6 +213,26 @@ impl PyDictRef { } } + fn setdefault( + self, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let mut entries = self.entries.borrow_mut(); + match entries.get(vm, &key)? { + Some(value) => Ok(value), + None => { + let set_value = match default { + OptionalArg::Present(value) => value, + OptionalArg::Missing => vm.ctx.none(), + }; + entries.insert(vm, &key, set_value.clone())?; + Ok(set_value) + } + } + } + fn copy(self, _vm: &VirtualMachine) -> PyDict { PyDict { entries: self.entries.clone(), @@ -473,6 +493,7 @@ pub fn init(context: &PyContext) { "items" => context.new_rustfunc(PyDictRef::items), "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), + "setdefault" => context.new_rustfunc(PyDictRef::setdefault), "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), "pop" => context.new_rustfunc(PyDictRef::pop), From cad4a13cf7fe86c5f0a9f904a54a6ab51e304160 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 25 Apr 2019 08:29:16 +0100 Subject: [PATCH 363/658] Add utility methods for default optional args. --- vm/src/function.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/vm/src/function.rs b/vm/src/function.rs index a72ead8f4..d3f26950f 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -361,6 +361,23 @@ impl OptionalArg { Missing => None, } } + + pub fn unwrap_or(self, default: T) -> T { + match self { + Present(value) => value, + Missing => default, + } + } + + pub fn unwrap_or_else(self, f: F) -> T + where + F: FnOnce() -> T, + { + match self { + Present(value) => value, + Missing => f(), + } + } } impl FromArgs for OptionalArg From 1a48fb17df7ba18e87c0371d32e225072051f177 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 25 Apr 2019 08:29:41 +0100 Subject: [PATCH 364/658] Add PyContext.new_classmethod --- vm/src/pyobject.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d172961a3..89e5c8c39 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -21,7 +21,7 @@ use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objbytearray; use crate::obj::objbytes; -use crate::obj::objclassmethod; +use crate::obj::objclassmethod::{self, PyClassMethod}; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::obj::objcomplex::{self, PyComplex}; @@ -668,6 +668,19 @@ impl PyContext { ) } + pub fn new_classmethod(&self, f: F) -> PyObjectRef + where + F: IntoPyNativeFunc, + { + PyObject::new( + PyClassMethod { + callable: self.new_rustfunc(f), + }, + self.classmethod_type(), + None, + ) + } + pub fn new_property(&self, f: F) -> PyObjectRef where F: IntoPyNativeFunc, From 4f0b17e84fbb56afca5d92e064c2a587270476bb Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 25 Apr 2019 08:30:32 +0100 Subject: [PATCH 365/658] Add dict.fromkeys. --- tests/snippets/dict.py | 3 +++ vm/src/obj/objdict.rs | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 2b52c5f6d..7d67668c9 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -188,3 +188,6 @@ assert 0 == x.setdefault('b', 0) assert x['b'] == 0 assert None == x.setdefault('c') assert x['c'] is None + +assert {1: None, "b": None} == dict.fromkeys([1, "b"]) +assert {1: 0, "b": 0} == dict.fromkeys([1, "b"], 0) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index a9cbcc41f..d36285c27 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,8 +3,8 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, - PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyIterable, PyObjectRef, + PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -93,6 +93,22 @@ impl PyDictRef { Ok(()) } + fn fromkeys( + class: PyClassRef, + iterable: PyIterable, + value: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let mut dict = DictContentType::default(); + let value = value.unwrap_or_else(|| vm.ctx.none()); + for elem in iterable.iter(vm)? { + let elem = elem?; + dict.insert(vm, &elem, value.clone())?; + } + let entries = RefCell::new(dict); + PyDict { entries }.into_ref_with_type(vm, class) + } + fn bool(self, _vm: &VirtualMachine) -> bool { !self.entries.borrow().is_empty() } @@ -492,6 +508,7 @@ pub fn init(context: &PyContext) { "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), "keys" => context.new_rustfunc(PyDictRef::keys), + "fromkeys" => context.new_classmethod(PyDictRef::fromkeys), "get" => context.new_rustfunc(PyDictRef::get), "setdefault" => context.new_rustfunc(PyDictRef::setdefault), "copy" => context.new_rustfunc(PyDictRef::copy), From d3f4c8f6e1d52b461849d95a791a49bf221fc1da Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 25 Apr 2019 08:36:38 +0100 Subject: [PATCH 366/658] Use unwrap_or_else for OptionalArg in dictionary. --- vm/src/obj/objdict.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index d36285c27..87cf3010b 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -222,10 +222,7 @@ impl PyDictRef { ) -> PyResult { match self.entries.borrow().get(vm, &key)? { Some(value) => Ok(value), - None => match default { - OptionalArg::Present(value) => Ok(value), - OptionalArg::Missing => Ok(vm.ctx.none()), - }, + None => Ok(default.unwrap_or_else(|| vm.ctx.none())), } } @@ -239,10 +236,7 @@ impl PyDictRef { match entries.get(vm, &key)? { Some(value) => Ok(value), None => { - let set_value = match default { - OptionalArg::Present(value) => value, - OptionalArg::Missing => vm.ctx.none(), - }; + let set_value = default.unwrap_or_else(|| vm.ctx.none()); entries.insert(vm, &key, set_value.clone())?; Ok(set_value) } From f38f263d939c555bb6197726f1459acd8320e6d1 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Thu, 25 Apr 2019 18:29:25 +0900 Subject: [PATCH 367/658] Implement stripping with arguments --- tests/snippets/strings.py | 5 +++++ vm/src/obj/objstr.rs | 28 ++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 86fbe69f3..3364a293c 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -52,6 +52,11 @@ assert b.strip() == 'hallo' assert b.lstrip() == 'hallo ' assert b.rstrip() == ' hallo' +s = '^*RustPython*^' +assert s.strip('^*') == 'RustPython' +assert s.lstrip('^*') == 'RustPython*^' +assert s.rstrip('^*') == '^*RustPython' + c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index dd610f4c8..56fc373c3 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -296,18 +296,34 @@ impl PyString { } #[pymethod] - fn strip(&self, _vm: &VirtualMachine) -> String { - self.value.trim().to_string() + fn strip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim().to_string(), + }; + self.value.trim_matches(|c| chars.contains(c)).to_string() } #[pymethod] - fn lstrip(&self, _vm: &VirtualMachine) -> String { - self.value.trim_start().to_string() + fn lstrip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim_start().to_string(), + }; + self.value + .trim_start_matches(|c| chars.contains(c)) + .to_string() } #[pymethod] - fn rstrip(&self, _vm: &VirtualMachine) -> String { - self.value.trim_end().to_string() + fn rstrip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim_end().to_string(), + }; + self.value + .trim_end_matches(|c| chars.contains(c)) + .to_string() } #[pymethod] From d48b59fd74c4078d3688cc0a2f7ad3f4db5239b9 Mon Sep 17 00:00:00 2001 From: Yan Feng Date: Thu, 25 Apr 2019 20:13:04 +0800 Subject: [PATCH 368/658] Fix os_stat for MacOS --- vm/src/stdlib/os.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 796578bb1..db5ba3bcb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -321,7 +321,7 @@ impl StatResultRef { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { use std::os::linux::fs::MetadataExt; match fs::metadata(&path.value) { @@ -340,6 +340,25 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +#[cfg(target_os = "macos")] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + use std::os::macos::fs::MetadataExt; + match fs::metadata(&path.value) { + Ok(meta) => Ok(StatResult { + st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), + st_size: meta.st_size(), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + #[cfg(windows)] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { use std::os::windows::fs::MetadataExt; @@ -359,7 +378,7 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -#[cfg(all(not(unix), not(windows)))] +#[cfg(all(not(target_os = "linux"), not(target_os = "macos"), not(windows)))] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { unimplemented!(); } From e6f08cf93edd2bb1f04bd754dc001e7f51edf4f9 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Fri, 26 Apr 2019 06:29:01 +0900 Subject: [PATCH 369/658] Enable dict round trip test for json --- tests/snippets/json_snippet.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/snippets/json_snippet.py b/tests/snippets/json_snippet.py index 917556a47..224fde8ca 100644 --- a/tests/snippets/json_snippet.py +++ b/tests/snippets/json_snippet.py @@ -25,8 +25,7 @@ assert '[[1]]' == json.dumps(((1,),)) assert [1, "string", 1.0, True] == json.loads(json.dumps((1, "string", 1.0, True))) assert '{}' == json.dumps({}) -# TODO: uncomment once dict comparison is implemented -# round_trip_test({'a': 'b'}) +round_trip_test({'a': 'b'}) assert 1 == json.loads("1") assert -1 == json.loads("-1") From e4698987bba0ced53c1d5e557fdfe493cd58e2bc Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Thu, 25 Apr 2019 15:14:43 -0500 Subject: [PATCH 370/658] Add a cfg for os.stat for android --- vm/src/stdlib/os.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index db5ba3bcb..6d81ca9c1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -359,6 +359,25 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +#[cfg(target_os = "android")] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + use std::os::android::fs::MetadataExt; + match fs::metadata(&path.value) { + Ok(meta) => Ok(StatResult { + st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), + st_size: meta.st_size(), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + #[cfg(windows)] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { use std::os::windows::fs::MetadataExt; @@ -378,7 +397,12 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -#[cfg(all(not(target_os = "linux"), not(target_os = "macos"), not(windows)))] +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "android", + windows +)))] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { unimplemented!(); } From 6e867edb6db3f73ffc6679e910b976e08dc9f5d6 Mon Sep 17 00:00:00 2001 From: Agent Date: Fri, 26 Apr 2019 12:17:16 +0800 Subject: [PATCH 371/658] Fix json.loads (#876) * Fix json.loads --- vm/src/stdlib/json.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index dceec8a7d..d57dd14a9 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -199,13 +199,12 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { // TODO: Support deserializing string sub-classes de.deserialize(&mut serde_json::Deserializer::from_str(s)) .map_err(|err| { - let json_decode_error = vm + let module = vm .get_attribute(vm.sys_module.clone(), "modules") .unwrap() .get_item("json", vm) - .unwrap() - .get_item("JSONDecodeError", vm) .unwrap(); + let json_decode_error = vm.get_attribute(module, "JSONDecodeError").unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); vm.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())) From 0e9acdc12b5a0ed98f85210451a596331304c909 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Fri, 26 Apr 2019 15:33:37 +0900 Subject: [PATCH 372/658] Fix justification --- tests/snippets/strings.py | 11 +++++++++++ vm/src/obj/objstr.rs | 12 ++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 3364a293c..0baa35d23 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -57,6 +57,17 @@ assert s.strip('^*') == 'RustPython' assert s.lstrip('^*') == 'RustPython*^' assert s.rstrip('^*') == '^*RustPython' +s = 'RustPython' +assert s.ljust(8) == 'RustPython' +assert s.rjust(8) == 'RustPython' +assert s.ljust(12) == 'RustPython ' +assert s.rjust(12) == ' RustPython' +assert s.ljust(12, '_') == 'RustPython__' +assert s.rjust(12, '_') == '__RustPython' +# The fill character must be exactly one character long +assert_raises(TypeError, lambda: s.ljust(12, '__')) +assert_raises(TypeError, lambda: s.rjust(12, '__')) + c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 56fc373c3..8ea438e35 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -740,7 +740,11 @@ impl PyString { ) -> PyResult { let value = &self.value; let rep_char = Self::get_fill_char(&rep, vm)?; - Ok(format!("{}{}", value, rep_char.repeat(len))) + if len <= value.len() { + Ok(value.to_string()) + } else { + Ok(format!("{}{}", value, rep_char.repeat(len - value.len()))) + } } #[pymethod] @@ -752,7 +756,11 @@ impl PyString { ) -> PyResult { let value = &self.value; let rep_char = Self::get_fill_char(&rep, vm)?; - Ok(format!("{}{}", rep_char.repeat(len), value)) + if len <= value.len() { + Ok(value.to_string()) + } else { + Ok(format!("{}{}", rep_char.repeat(len - value.len()), value)) + } } #[pymethod] From 16152a1d3290be1fe8cd2773010483bf39628c68 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 26 Apr 2019 23:49:51 +0900 Subject: [PATCH 373/658] Fix range() to support negative index --- tests/snippets/builtin_range.py | 2 ++ vm/src/obj/objrange.rs | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index 2ea71f7a7..1dd5e7028 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -54,3 +54,5 @@ assert list(reversed(range(1,10,5))) == [6, 1] # range retains the original int refs i = 2**64 assert range(i).stop is i + +assert range(10)[-1] == 9 diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 1a21a49ad..56b7c0396 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -1,5 +1,4 @@ use std::cell::Cell; -use std::ops::Mul; use num_bigint::{BigInt, Sign}; use num_integer::Integer; @@ -65,15 +64,18 @@ impl PyRange { } #[inline] - pub fn get<'a, T>(&'a self, index: T) -> Option - where - &'a BigInt: Mul, - { + pub fn get(&self, index: &BigInt) -> Option { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); - let result = start + step * index; + let index = if index < &BigInt::zero() { + stop + index + } else { + index.clone() + }; + + let result = start + step * &index; if (self.forward() && !self.is_empty() && &result < stop) || (!self.forward() && !self.is_empty() && &result > stop) @@ -274,7 +276,7 @@ impl PyRange { } RangeIndex::Slice(slice) => { let new_start = if let Some(int) = slice.start_index(vm)? { - if let Some(i) = self.get(int) { + if let Some(i) = self.get(&int) { PyInt::new(i).into_ref(vm) } else { self.start.clone() @@ -284,7 +286,7 @@ impl PyRange { }; let new_end = if let Some(int) = slice.stop_index(vm)? { - if let Some(i) = self.get(int) { + if let Some(i) = self.get(&int) { PyInt::new(i).into_ref(vm) } else { self.stop.clone() @@ -339,7 +341,8 @@ type PyRangeIteratorRef = PyRef; impl PyRangeIteratorRef { fn next(self, vm: &VirtualMachine) -> PyResult { - if let Some(int) = self.range.get(self.position.get()) { + let position = BigInt::from(self.position.get()); + if let Some(int) = self.range.get(&position) { self.position.set(self.position.get() + 1); Ok(int) } else { From 16353f4f546a8b6a5bb80cf85db759b8188055b4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 27 Apr 2019 00:06:07 +0900 Subject: [PATCH 374/658] range negative side bound error --- tests/snippets/builtin_range.py | 4 ++++ vm/src/obj/objrange.rs | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index 1dd5e7028..b087f7add 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -55,4 +55,8 @@ assert list(reversed(range(1,10,5))) == [6, 1] i = 2**64 assert range(i).stop is i +# negative index assert range(10)[-1] == 9 +assert_raises(IndexError, lambda: range(10)[-11], 'out of bound') +assert str(range(10)[-2:4]) == str(range(8, 4)) +assert str(range(10)[-6:-2]) == str(range(4, 8)) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 56b7c0396..abe82ec3b 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -70,7 +70,11 @@ impl PyRange { let step = self.step.as_bigint(); let index = if index < &BigInt::zero() { - stop + index + let index = stop + index; + if index < BigInt::zero() { + return None; + } + index } else { index.clone() }; From a7091ca88da3f9ee4fcd38ea033c2210c1e1dfb3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 27 Apr 2019 00:27:47 +0900 Subject: [PATCH 375/658] Add range.__eq__ --- tests/snippets/builtin_range.py | 4 ++-- vm/src/obj/objrange.rs | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index b087f7add..a8797f75a 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -58,5 +58,5 @@ assert range(i).stop is i # negative index assert range(10)[-1] == 9 assert_raises(IndexError, lambda: range(10)[-11], 'out of bound') -assert str(range(10)[-2:4]) == str(range(8, 4)) -assert str(range(10)[-6:-2]) == str(range(4, 8)) +assert range(10)[-2:4] == range(8, 4) +assert range(10)[-6:-2] == range(4, 8) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index abe82ec3b..d98e7491f 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -13,7 +13,7 @@ use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; use super::objiter; use super::objslice::{PySlice, PySliceRef}; -use super::objtype::PyClassRef; +use super::objtype::{self, PyClassRef}; #[derive(Debug, Clone)] pub struct PyRange { @@ -110,6 +110,7 @@ pub fn init(context: &PyContext) { "__bool__" => context.new_rustfunc(PyRange::bool), "__contains__" => context.new_rustfunc(PyRange::contains), "__doc__" => context.new_str(range_doc.to_string()), + "__eq__" => context.new_rustfunc(PyRange::eq), "__getitem__" => context.new_rustfunc(PyRange::getitem), "__iter__" => context.new_rustfunc(PyRange::iter), "__len__" => context.new_rustfunc(PyRange::len), @@ -246,6 +247,17 @@ impl PyRange { } } + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + if objtype::isinstance(&rhs, &vm.ctx.range_type()) { + let rhs = get_value(&rhs); + self.start.as_bigint() == rhs.start.as_bigint() + && self.stop.as_bigint() == rhs.stop.as_bigint() + && self.step.as_bigint() == rhs.step.as_bigint() + } else { + false + } + } + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { match self.index_of(int.as_bigint()) { From 462c8233f3f8d3a94f102b9bd13f70a43ea86343 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 27 Apr 2019 13:49:54 +0300 Subject: [PATCH 376/658] Add exceptions stack to VirtualMachine --- vm/src/bytecode.rs | 2 ++ vm/src/compile.rs | 12 ++++++++---- vm/src/frame.rs | 19 ++++++++++++------- vm/src/vm.rs | 10 ++++++++++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 73d69dc18..a6288cf50 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -183,6 +183,7 @@ pub enum Instruction { conversion: Option, spec: String, }, + PopException, } use self::Instruction::*; @@ -381,6 +382,7 @@ impl Instruction { UnpackEx { before, after } => w!(UnpackEx, before, after), Unpack => w!(Unpack), FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion + PopException => w!(PopException), } } } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index b7df81313..f16544006 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -627,16 +627,20 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { - // Duplicate exception for context: - self.emit(Instruction::Duplicate); self.store_name(alias); + } else { + // Drop exception from top of stack: + self.emit(Instruction::Pop); } + } else { + // Catch all! + // Drop exception from top of stack: + self.emit(Instruction::Pop); } // Handler code: self.compile_statements(&handler.body)?; - // Drop exception from top of stack: - self.emit(Instruction::Pop); + self.emit(Instruction::PopException); self.emit(Instruction::Jump { target: finally_label, }); diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 06c970de1..56d8c0d7b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -724,9 +724,9 @@ impl Frame { _ => vm.get_none(), }; let exception = match argc { - 0 => match in_exc { - true => self.get_exception(vm, false)?, - false => { + 0 => match vm.pop_exception() { + Some(exc) => exc, + None => { return Err(vm.new_exception( vm.ctx.exceptions.runtime_error.clone(), "No active exception to reraise".to_string(), @@ -739,9 +739,9 @@ impl Frame { }; let context = match argc { 0 => vm.get_none(), // We have already got the exception, - _ => match in_exc { - true => self.get_exception(vm, false)?, - false => vm.get_none(), + _ => match vm.pop_exception() { + Some(exc) => exc, + None => vm.get_none(), }, }; info!( @@ -860,6 +860,10 @@ impl Frame { self.push_value(formatted); Ok(None) } + bytecode::Instruction::PopException {} => { + assert!(!vm.pop_exception().is_none()); + Ok(None) + } } } @@ -966,7 +970,8 @@ impl Frame { while let Some(block) = self.pop_block() { match block.typ { BlockType::TryExcept { handler } => { - self.push_value(exc); + self.push_value(exc.clone()); + vm.push_exception(exc); self.jump(handler); return None; } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index ab15e14e9..a677d7341 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -50,6 +50,7 @@ pub struct VirtualMachine { pub ctx: PyContext, pub frames: RefCell>, pub wasm_id: Option, + pub exceptions: RefCell>, } impl VirtualMachine { @@ -69,6 +70,7 @@ impl VirtualMachine { ctx, frames: RefCell::new(vec![]), wasm_id: None, + exceptions: RefCell::new(vec![]), }; builtins::make_module(&vm, builtins.clone()); @@ -921,6 +923,14 @@ impl VirtualMachine { self._membership_iter_search(haystack, needle) } } + + pub fn push_exception(&self, exc: PyObjectRef) -> () { + self.exceptions.borrow_mut().push(exc) + } + + pub fn pop_exception(&self) -> Option { + self.exceptions.borrow_mut().pop() + } } impl Default for VirtualMachine { From 4b9bafc824cfd4430ae27dfb9aa7e6001abe4961 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 27 Apr 2019 14:33:24 +0300 Subject: [PATCH 377/658] Remove in_exc_handler for Raise --- vm/src/bytecode.rs | 3 +-- vm/src/compile.rs | 31 +++++-------------------------- vm/src/frame.rs | 2 +- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index a6288cf50..09bb7e70a 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -136,7 +136,6 @@ pub enum Instruction { PopBlock, Raise { argc: usize, - in_exc: bool, }, BuildString { size: usize, @@ -366,7 +365,7 @@ impl Instruction { SetupWith { end } => w!(SetupWith, end), CleanupWith { end } => w!(CleanupWith, end), PopBlock => w!(PopBlock), - Raise { argc, in_exc } => w!(Raise, argc, in_exc), + Raise { argc } => w!(Raise, argc), BuildString { size } => w!(BuildString, size), BuildTuple { size, unpack } => w!(BuildTuple, size, unpack), BuildList { size, unpack } => w!(BuildList, size, unpack), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index f16544006..44ea521f3 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -24,7 +24,6 @@ struct Compiler { current_qualified_path: Option, in_loop: bool, in_function_def: bool, - in_exc_handler: bool, } /// Compile a given sourcecode into a bytecode object. @@ -86,7 +85,6 @@ impl Compiler { current_qualified_path: None, in_loop: false, in_function_def: false, - in_exc_handler: false, } } @@ -330,24 +328,15 @@ impl Compiler { match cause { Some(cause) => { self.compile_expression(cause)?; - self.emit(Instruction::Raise { - argc: 2, - in_exc: self.in_exc_handler, - }); + self.emit(Instruction::Raise { argc: 2 }); } None => { - self.emit(Instruction::Raise { - argc: 1, - in_exc: self.in_exc_handler, - }); + self.emit(Instruction::Raise { argc: 1 }); } } } None => { - self.emit(Instruction::Raise { - argc: 0, - in_exc: self.in_exc_handler, - }); + self.emit(Instruction::Raise { argc: 0 }); } }, ast::Statement::Try { @@ -392,10 +381,7 @@ impl Compiler { }); } } - self.emit(Instruction::Raise { - argc: 1, - in_exc: self.in_exc_handler, - }); + self.emit(Instruction::Raise { argc: 1 }); self.set_label(end_label); } ast::Statement::Break => { @@ -597,8 +583,6 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: - let was_in_exc_handler = self.in_exc_handler; - self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now handler_label = self.new_label(); @@ -661,12 +645,7 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.emit(Instruction::Raise { - argc: 0, - in_exc: true, - }); - - self.in_exc_handler = was_in_exc_handler; + self.emit(Instruction::Raise { argc: 0 }); // We successfully ran the try block: // else: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 56d8c0d7b..00d843d2a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -718,7 +718,7 @@ impl Frame { Ok(None) } - bytecode::Instruction::Raise { argc, in_exc } => { + bytecode::Instruction::Raise { argc } => { let cause = match argc { 2 => self.get_exception(vm, true)?, _ => vm.get_none(), From 180ec22d4b67f26b3c0ef31bd0d80cf162158880 Mon Sep 17 00:00:00 2001 From: Agent Date: Sat, 27 Apr 2019 22:41:19 +0800 Subject: [PATCH 378/658] Add constants to os module (#883) --- Lib/os.py | 10 ++++++++++ tests/snippets/stdlib_os.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/Lib/os.py b/Lib/os.py index 3cadb5f61..a14702d5e 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,9 +1,19 @@ from _os import * +curdir = '.' +pardir = '..' +extsep = '.' + if name == 'nt': sep = '\\' + linesep = '\r\n' + altsep = '/' + pathsep = ';' else: sep = '/' + linesep = '\n' + altsep = None + pathsep = ':' # Change environ to automatically call putenv(), unsetenv if they exist. from _collections_abc import MutableMapping diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 0fb44cbb5..4b0344f12 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -33,11 +33,20 @@ if os.name == "posix": os.unsetenv(ENV_KEY) assert os.getenv(ENV_KEY) == None +assert os.curdir == "." +assert os.pardir == ".." +assert os.extsep == "." if os.name == "nt": assert os.sep == "\\" + assert os.linesep == "\r\n" + assert os.altsep == "/" + assert os.pathsep == ";" else: assert os.sep == "/" + assert os.linesep == "\n" + assert os.altsep == None + assert os.pathsep == ":" class TestWithTempDir(): def __enter__(self): From 9d40a4c9d8f2ab22c8f05c929321f4f26a141375 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Sat, 27 Apr 2019 17:44:21 +0200 Subject: [PATCH 379/658] Support list.__imul__ --- tests/snippets/list.py | 4 ++++ vm/src/obj/objlist.rs | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 12d961b61..e8e731de9 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -14,6 +14,10 @@ assert y == [2, 1, 2, 3, 1, 2, 3] assert x * 0 == [], "list __mul__ by 0 failed" assert x * -1 == [], "list __mul__ by -1 failed" assert x * 2 == [1, 2, 3, 1, 2, 3], "list __mul__ by 2 failed" +y = x +x *= 2 +assert y is x +assert x == [1, 2, 3] * 2 # index() assert ['a', 'b', 'c'].index('b') == 1 diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index bfc763d8c..5c1bad5af 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -395,6 +395,12 @@ impl PyListRef { vm.ctx.new_list(new_elements) } + fn imul(self, counter: isize, _vm: &VirtualMachine) -> Self { + let new_elements = seq_mul(&self.elements.borrow(), counter); + self.elements.replace(new_elements); + self + } + fn count(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut count: usize = 0; for element in self.elements.borrow().iter() { @@ -823,6 +829,7 @@ pub fn init(context: &PyContext) { "__iter__" => context.new_rustfunc(PyListRef::iter), "__setitem__" => context.new_rustfunc(PyListRef::setitem), "__mul__" => context.new_rustfunc(PyListRef::mul), + "__imul__" => context.new_rustfunc(PyListRef::imul), "__len__" => context.new_rustfunc(PyListRef::len), "__new__" => context.new_rustfunc(list_new), "__repr__" => context.new_rustfunc(PyListRef::repr), From d87468a1cbe7c65c87f193032485d4cc0ba6ee43 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 27 Apr 2019 18:48:41 +0900 Subject: [PATCH 380/658] Add type.__name__ --- tests/snippets/types_snippet.py | 1 + vm/src/obj/objtype.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests/snippets/types_snippet.py b/tests/snippets/types_snippet.py index f441552d1..075957418 100644 --- a/tests/snippets/types_snippet.py +++ b/tests/snippets/types_snippet.py @@ -49,3 +49,4 @@ assert C.__subclasses__() == [] assert type in object.__subclasses__() +assert cls.__name__ == 'Cls' diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 819f85088..c14e51230 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -99,6 +99,10 @@ impl PyClassRef { issubclass(&subclass, &self) } + fn name(self, _vm: &VirtualMachine) -> String { + self.name.clone() + } + fn repr(self, _vm: &VirtualMachine) -> String { format!("", self.name) } @@ -198,6 +202,7 @@ pub fn init(ctx: &PyContext) { .add_getter(PyClassRef::mro) .add_setter(PyClassRef::set_mro) .create(), + "__name__" => ctx.new_property(PyClassRef::name), "__repr__" => ctx.new_rustfunc(PyClassRef::repr), "__prepare__" => ctx.new_rustfunc(PyClassRef::prepare), "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), From 60afbb4bf5dd9c9b8a3f32e60f2be9bcbe2ff6f2 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 27 Apr 2019 21:53:08 +0300 Subject: [PATCH 381/658] Don't pop exeption for context --- vm/src/frame.rs | 4 ++-- vm/src/vm.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 00d843d2a..69b1d510a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -739,8 +739,8 @@ impl Frame { }; let context = match argc { 0 => vm.get_none(), // We have already got the exception, - _ => match vm.pop_exception() { - Some(exc) => exc, + _ => match vm.last_exception() { + Some(exc) => exc.clone(), None => vm.get_none(), }, }; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index a677d7341..9308c0caa 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -931,6 +931,17 @@ impl VirtualMachine { pub fn pop_exception(&self) -> Option { self.exceptions.borrow_mut().pop() } + + pub fn last_exception(&self) -> Option> { + let exceptions = self.exceptions.borrow(); + if exceptions.is_empty() { + None + } else { + Some(Ref::map(self.exceptions.borrow(), |exceptions| { + exceptions.last().unwrap() + })) + } + } } impl Default for VirtualMachine { From 7fb98c91d2ab4de624eb1622e46ecbf13ceb8703 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 05:05:33 +0900 Subject: [PATCH 382/658] PyFloat uses #[pyclass] #[pyimpl] #[pymethod] --- vm/src/obj/objfloat.rs | 66 +++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index e16edf5ce..5438d31c1 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -4,13 +4,14 @@ use super::objstr; use super::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; use num_bigint::ToBigInt; use num_rational::Ratio; use num_traits::ToPrimitive; +#[pyclass(name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyFloat { value: f64, @@ -40,7 +41,9 @@ impl From for PyFloat { } } +#[pyimpl] impl PyFloat { + #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let value = self.value; let result = if objtype::isinstance(&other, &vm.ctx.float_type()) { @@ -60,6 +63,7 @@ impl PyFloat { vm.ctx.new_bool(result) } + #[pymethod(name = "__lt__")] fn lt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { @@ -72,6 +76,7 @@ impl PyFloat { } } + #[pymethod(name = "__le__")] fn le(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { @@ -84,6 +89,7 @@ impl PyFloat { } } + #[pymethod(name = "__gt__")] fn gt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { @@ -96,6 +102,7 @@ impl PyFloat { } } + #[pymethod(name = "__ge__")] fn ge(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { @@ -108,10 +115,12 @@ impl PyFloat { } } + #[pymethod(name = "__abs__")] fn abs(&self, _vm: &VirtualMachine) -> f64 { self.value.abs() } + #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { @@ -124,10 +133,17 @@ impl PyFloat { } } + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.add(other, vm) + } + + #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { self.value != 0.0 } + #[pymethod(name = "__divmod__")] fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.float_type()) || objtype::isinstance(&other, &vm.ctx.int_type()) @@ -140,6 +156,7 @@ impl PyFloat { } } + #[pymethod(name = "__floordiv__")] fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { @@ -199,6 +216,7 @@ impl PyFloat { PyFloat { value }.into_ref_with_type(vm, cls) } + #[pymethod(name = "__mod__")] fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { @@ -218,10 +236,12 @@ impl PyFloat { } } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value } + #[pymethod(name = "__pow__")] fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { @@ -234,6 +254,7 @@ impl PyFloat { } } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { @@ -247,6 +268,7 @@ impl PyFloat { } } + #[pymethod(name = "__rsub__")] fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { @@ -260,10 +282,12 @@ impl PyFloat { } } + #[pymethod(name = "__repr__")] fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } + #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { @@ -283,6 +307,7 @@ impl PyFloat { } } + #[pymethod(name = "__rtruediv__")] fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { @@ -302,6 +327,7 @@ impl PyFloat { } } + #[pymethod(name = "__mul__")] fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type) { @@ -315,15 +341,23 @@ impl PyFloat { } } + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(other, vm) + } + + #[pyproperty(name = "real")] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { zelf } + #[pymethod(name = "is_integer")] fn is_integer(&self, _vm: &VirtualMachine) -> bool { let v = self.value; (v - v.round()).abs() < std::f64::EPSILON } + #[pymethod(name = "as_integer_ratio")] fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult { let value = self.value; if value.is_infinite() { @@ -362,36 +396,10 @@ pub fn make_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let float_type = &context.float_type; - + PyFloat::extend_class(context, &context.float_type); let float_doc = "Convert a string or number to a floating point number, if possible."; - - extend_class!(context, float_type, { - "__eq__" => context.new_rustfunc(PyFloat::eq), - "__lt__" => context.new_rustfunc(PyFloat::lt), - "__le__" => context.new_rustfunc(PyFloat::le), - "__gt__" => context.new_rustfunc(PyFloat::gt), - "__ge__" => context.new_rustfunc(PyFloat::ge), - "__abs__" => context.new_rustfunc(PyFloat::abs), - "__add__" => context.new_rustfunc(PyFloat::add), - "__radd__" => context.new_rustfunc(PyFloat::add), - "__bool__" => context.new_rustfunc(PyFloat::bool), - "__divmod__" => context.new_rustfunc(PyFloat::divmod), - "__floordiv__" => context.new_rustfunc(PyFloat::floordiv), + extend_class!(context, &context.float_type, { "__new__" => context.new_rustfunc(PyFloat::new_float), - "__mod__" => context.new_rustfunc(PyFloat::mod_), - "__neg__" => context.new_rustfunc(PyFloat::neg), - "__pow__" => context.new_rustfunc(PyFloat::pow), - "__sub__" => context.new_rustfunc(PyFloat::sub), - "__rsub__" => context.new_rustfunc(PyFloat::rsub), - "__repr__" => context.new_rustfunc(PyFloat::repr), "__doc__" => context.new_str(float_doc.to_string()), - "__truediv__" => context.new_rustfunc(PyFloat::truediv), - "__rtruediv__" => context.new_rustfunc(PyFloat::rtruediv), - "__mul__" => context.new_rustfunc(PyFloat::mul), - "__rmul__" => context.new_rustfunc(PyFloat::mul), - "real" => context.new_property(PyFloat::real), - "is_integer" => context.new_rustfunc(PyFloat::is_integer), - "as_integer_ratio" => context.new_rustfunc(PyFloat::as_integer_ratio) }); } From ed79ccc6b1e2c530c27f975cd9069d2be703d973 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 06:04:21 +0900 Subject: [PATCH 383/658] PyComplex uses PyClassImpl --- vm/src/obj/objcomplex.rs | 47 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 9f88dd6a4..5992c32d6 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -2,13 +2,14 @@ use num_complex::Complex64; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objfloat::{self, PyFloat}; use super::objint; use super::objtype::{self, PyClassRef}; +#[pyclass(name = "complex")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyComplex { value: Complex64, @@ -28,24 +29,14 @@ impl From for PyComplex { } pub fn init(context: &PyContext) { - let complex_type = &context.complex_type; - + PyComplex::extend_class(context, &context.complex_type); let complex_doc = "Create a complex number from a real part and an optional imaginary part.\n\n\ This is equivalent to (real + imag*1j) where imag defaults to 0."; - extend_class!(context, complex_type, { + extend_class!(context, &context.complex_type, { "__doc__" => context.new_str(complex_doc.to_string()), - "__abs__" => context.new_rustfunc(PyComplexRef::abs), - "__add__" => context.new_rustfunc(PyComplexRef::add), - "__eq__" => context.new_rustfunc(PyComplexRef::eq), - "__neg__" => context.new_rustfunc(PyComplexRef::neg), "__new__" => context.new_rustfunc(PyComplexRef::new), - "__radd__" => context.new_rustfunc(PyComplexRef::radd), - "__repr__" => context.new_rustfunc(PyComplexRef::repr), - "conjugate" => context.new_rustfunc(PyComplexRef::conjugate), - "imag" => context.new_property(PyComplexRef::imag), - "real" => context.new_property(PyComplexRef::real) }); } @@ -73,21 +64,28 @@ impl PyComplexRef { let value = Complex64::new(real, imag); PyComplex { value }.into_ref_with_type(vm, cls) } +} - fn real(self, _vm: &VirtualMachine) -> PyFloat { +#[pyimpl] +impl PyComplex { + #[pyproperty(name = "real")] + fn real(&self, _vm: &VirtualMachine) -> PyFloat { self.value.re.into() } - fn imag(self, _vm: &VirtualMachine) -> PyFloat { + #[pyproperty(name = "imag")] + fn imag(&self, _vm: &VirtualMachine) -> PyFloat { self.value.im.into() } - fn abs(self, _vm: &VirtualMachine) -> PyFloat { + #[pymethod(name = "__abs__")] + fn abs(&self, _vm: &VirtualMachine) -> PyFloat { let Complex64 { im, re } = self.value; re.hypot(im).into() } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.complex_type()) { vm.ctx.new_complex(self.value + get_value(&other)) } else if objtype::isinstance(&other, &vm.ctx.int_type()) { @@ -100,7 +98,8 @@ impl PyComplexRef { } } - fn radd(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_complex(Complex64::new( self.value.re + objint::get_value(&other).to_f64().unwrap(), @@ -111,11 +110,13 @@ impl PyComplexRef { } } - fn conjugate(self, _vm: &VirtualMachine) -> PyComplex { + #[pymethod(name = "conjugate")] + fn conjugate(&self, _vm: &VirtualMachine) -> PyComplex { self.value.conj().into() } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { self.value == get_value(&other) } else if objtype::isinstance(&other, &vm.ctx.int_type()) { @@ -132,11 +133,13 @@ impl PyComplexRef { vm.ctx.new_bool(result) } - fn neg(self, _vm: &VirtualMachine) -> PyComplex { + #[pymethod(name = "__neg__")] + fn neg(&self, _vm: &VirtualMachine) -> PyComplex { PyComplex::from(-self.value) } - fn repr(self, _vm: &VirtualMachine) -> String { + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { let Complex64 { re, im } = self.value; if re == 0.0 { format!("{}j", im) From 15e4e678cac0d0bf32bcaf0cf89a07012f0c4126 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 06:19:12 +0900 Subject: [PATCH 384/658] Add complex.__bool__ --- tests/snippets/builtin_complex.py | 6 ++++++ vm/src/obj/objcomplex.rs | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index c073f7433..1712533f0 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -27,6 +27,12 @@ assert complex(1, 2).__eq__('foo') == NotImplemented assert -complex(1, -1) == complex(-1, 1) assert -complex(0, 0) == complex(0, 0) +# __bool__ + +assert bool(complex(0, 0)) is False +assert bool(complex(0, 1)) is True +assert bool(complex(1, 0)) is True + # real a = complex(3, 4) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 5992c32d6..df3a6f015 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,5 +1,5 @@ use num_complex::Complex64; -use num_traits::ToPrimitive; +use num_traits::{ToPrimitive, Zero}; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; @@ -147,4 +147,9 @@ impl PyComplex { format!("({}+{}j)", re, im) } } + + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { + self.value != Complex64::zero() + } } From 2eae737f3d050bb06bec971d1b59e443a59f29e6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 06:52:54 +0900 Subject: [PATCH 385/658] complex.__add__ can raises OverflowError --- tests/snippets/builtin_complex.py | 10 +++++++++ vm/src/obj/objcomplex.rs | 35 +++++++++++++++++-------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 1712533f0..bfe43059e 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + # __abs__ assert abs(complex(3, 4)) == 5 @@ -50,3 +52,11 @@ assert 1 + 1j == complex(1, 1) assert 1j + 1 == complex(1, 1) assert (1j + 1) + 3 == complex(4, 1) assert 3 + (1j + 1) == complex(4, 1) + +# overflow +with assertRaises(OverflowError): + complex(10 ** 1000, 0) +with assertRaises(OverflowError): + complex(0, 10 ** 1000) +with assertRaises(OverflowError): + complex(0, 0) + 10 ** 1000 diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index df3a6f015..3be4e8239 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -66,6 +66,17 @@ impl PyComplexRef { } } +fn to_complex(value: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + if objtype::isinstance(&value, &vm.ctx.int_type()) { + match objint::get_value(&value).to_f64() { + Some(v) => Ok(Some(Complex64::new(v, 0.0))), + None => Err(vm.new_overflow_error("int too large to convert to float".to_string())), + } + } else { + Ok(None) + } +} + #[pyimpl] impl PyComplex { #[pyproperty(name = "real")] @@ -85,28 +96,20 @@ impl PyComplex { } #[pymethod(name = "__add__")] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.complex_type()) { - vm.ctx.new_complex(self.value + get_value(&other)) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx.new_complex(Complex64::new( - self.value.re + objint::get_value(&other).to_f64().unwrap(), - self.value.im, - )) + Ok(vm.ctx.new_complex(self.value + get_value(&other))) } else { - vm.ctx.not_implemented() + self.radd(other, vm) } } #[pymethod(name = "__radd__")] - fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx.new_complex(Complex64::new( - self.value.re + objint::get_value(&other).to_f64().unwrap(), - self.value.im, - )) - } else { - vm.ctx.not_implemented() + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match to_complex(other, vm) { + Ok(Some(other)) => Ok(vm.ctx.new_complex(self.value + other)), + Ok(None) => Ok(vm.ctx.not_implemented()), + Err(err) => Err(err), } } From a2625916f5fea651b6290980d9427a35f505153b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 12:01:54 +0900 Subject: [PATCH 386/658] Fix complex.__add__ to support float --- tests/snippets/builtin_complex.py | 6 ++++++ vm/src/obj/objcomplex.rs | 3 +++ 2 files changed, 9 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index bfe43059e..be688cc63 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -53,6 +53,12 @@ assert 1j + 1 == complex(1, 1) assert (1j + 1) + 3 == complex(4, 1) assert 3 + (1j + 1) == complex(4, 1) +# float and complex addition +assert 1.1 + 1.2j == complex(1.1, 1.2) +assert 1.3j + 1.4 == complex(1.4, 1.3) +assert (1.5j + 1.6) + 3 == complex(4.6, 1.5) +assert 3.5 + (1.1j + 1.2) == complex(4.7, 1.1) + # overflow with assertRaises(OverflowError): complex(10 ** 1000, 0) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 3be4e8239..af4880e06 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -72,6 +72,9 @@ fn to_complex(value: PyObjectRef, vm: &VirtualMachine) -> PyResult Ok(Some(Complex64::new(v, 0.0))), None => Err(vm.new_overflow_error("int too large to convert to float".to_string())), } + } else if objtype::isinstance(&value, &vm.ctx.float_type()) { + let v = objfloat::get_value(&value); + Ok(Some(Complex64::new(v, 0.0))) } else { Ok(None) } From dc05459459cd25a223399bb689c7a26939909ea9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 12:09:49 +0900 Subject: [PATCH 387/658] Add float.__sub__, float.__rsub__ --- tests/snippets/builtin_complex.py | 15 +++++++++++++++ vm/src/obj/objcomplex.rs | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index be688cc63..2df8e4df7 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -59,6 +59,21 @@ assert 1.3j + 1.4 == complex(1.4, 1.3) assert (1.5j + 1.6) + 3 == complex(4.6, 1.5) assert 3.5 + (1.1j + 1.2) == complex(4.7, 1.1) +# subtraction +assert 1 - 1j == complex(1, -1) +assert 1j - 1 == complex(-1, 1) +assert 2j - 1j == complex(0, 1) + +# type error addition +with assertRaises(TypeError): + assert 1j + 'str' +with assertRaises(TypeError): + assert 1j - 'str' +with assertRaises(TypeError): + assert 'str' + 1j +with assertRaises(TypeError): + assert 'str' - 1j + # overflow with assertRaises(OverflowError): complex(10 ** 1000, 0) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index af4880e06..93a0559aa 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -116,6 +116,28 @@ impl PyComplex { } } + #[pymethod(name = "__sub__")] + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&other, &vm.ctx.complex_type()) { + Ok(vm.ctx.new_complex(self.value - get_value(&other))) + } else { + match to_complex(other, vm) { + Ok(Some(other)) => Ok(vm.ctx.new_complex(self.value - other)), + Ok(None) => Ok(vm.ctx.not_implemented()), + Err(err) => Err(err), + } + } + } + + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match to_complex(other, vm) { + Ok(Some(other)) => Ok(vm.ctx.new_complex(other - self.value)), + Ok(None) => Ok(vm.ctx.not_implemented()), + Err(err) => Err(err), + } + } + #[pymethod(name = "conjugate")] fn conjugate(&self, _vm: &VirtualMachine) -> PyComplex { self.value.conj().into() From 9dda776e2541618ef6bd11f96f88e4da0d78c6d9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 05:24:33 +0900 Subject: [PATCH 388/658] Add float.__int__ / float.__float__ --- tests/snippets/floats.py | 3 +++ vm/src/obj/objfloat.rs | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 08e7d4b2b..801a4ccd7 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -84,6 +84,9 @@ assert 1.0.__rtruediv__(2) == 2.0 assert 2.0.__mul__(1) == 2.0 assert 2.0.__rsub__(1) == -1.0 +assert 1.2.__int__() == 1 +assert 1.2.__float__() == 1.2 + assert (1.7).real == 1.7 assert (1.3).is_integer() == False assert (1.0).is_integer() == True diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 5438d31c1..4b5f02d4e 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -7,7 +7,7 @@ use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; -use num_bigint::ToBigInt; +use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; use num_traits::ToPrimitive; @@ -346,6 +346,16 @@ impl PyFloat { self.mul(other, vm) } + #[pymethod(name = "__int__")] + fn int(&self, _vm: &VirtualMachine) -> BigInt { + self.value.to_bigint().unwrap() + } + + #[pymethod(name = "__float__")] + fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf + } + #[pyproperty(name = "real")] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { zelf From 4810a55c82dbb845df11b9bcf094863b69ee23fc Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 12:44:25 +0900 Subject: [PATCH 389/658] Add float.__trunc__ --- tests/snippets/floats.py | 4 ++++ vm/src/obj/objfloat.rs | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 801a4ccd7..f30aeddaf 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -86,6 +86,10 @@ assert 2.0.__rsub__(1) == -1.0 assert 1.2.__int__() == 1 assert 1.2.__float__() == 1.2 +assert 1.2.__trunc__() == 1 +assert int(1.2) == 1 +assert float(1.2) == 1.2 +# assert math.trunc(1.2) == 1 assert (1.7).real == 1.7 assert (1.3).is_integer() == False diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 4b5f02d4e..0fa06435a 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -346,11 +346,16 @@ impl PyFloat { self.mul(other, vm) } - #[pymethod(name = "__int__")] - fn int(&self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__trunc__")] + fn trunc(&self, _vm: &VirtualMachine) -> BigInt { self.value.to_bigint().unwrap() } + #[pymethod(name = "__int__")] + fn int(&self, vm: &VirtualMachine) -> BigInt { + self.trunc(vm) + } + #[pymethod(name = "__float__")] fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { zelf From 8047ab3d3efb5550471826235255d311a1648349 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 20:03:30 +0900 Subject: [PATCH 390/658] Add float tests --- tests/snippets/floats.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index f30aeddaf..f5877604e 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -7,10 +7,15 @@ from testutils import assert_raises a = 1.2 b = 1.3 c = 1.2 +z = 2 + +assert -a == -1.2 + assert a < b assert not b < a assert a <= b assert a <= c +assert a < z assert b > a assert not a > b @@ -18,13 +23,25 @@ assert not a > c assert b >= a assert c >= a assert not a >= b +assert z > a assert a + b == 2.5 assert a - c == 0 assert a / c == 1 +assert a % c == 0 +assert a + z == 3.2 +assert z + a == 3.2 +assert a - z == -0.8 +assert z - a == 0.8 +assert a / z == 0.6 +assert 6 / a == 5.0 +assert 2.0 % z == 0.0 +# assert z % 2.0 == 0.0 assert a < 5 assert a <= 5 +assert a < 5.5 +assert a <= 5.5 try: assert a < 'a' except TypeError: From 304cea89b8336b1f6f6ee53b08ba18e27b6ed14d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 20:31:07 +0900 Subject: [PATCH 391/658] objint::get_float_value --- vm/src/obj/objfloat.rs | 26 ++++++++------------------ vm/src/obj/objint.rs | 6 ++++++ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 0fa06435a..099d18bb3 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -162,9 +162,7 @@ impl PyFloat { let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? + objint::get_float_value(&other, vm)? } else { return Ok(vm.ctx.not_implemented()); }; @@ -180,12 +178,10 @@ impl PyFloat { let value = if objtype::isinstance(&arg, &vm.ctx.float_type()) { get_value(&arg) } else if objtype::isinstance(&arg, &vm.ctx.int_type()) { - match objint::get_value(&arg).to_f64() { - Some(f) => f, - None => { - return Err( - vm.new_overflow_error("int too large to convert to float".to_string()) - ); + match objint::get_float_value(&arg, vm) { + Ok(f) => f, + Err(e) => { + return Err(e); } } } else if objtype::isinstance(&arg, &vm.ctx.str_type()) { @@ -222,9 +218,7 @@ impl PyFloat { let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? + objint::get_float_value(&other, vm)? } else { return Ok(vm.ctx.not_implemented()); }; @@ -293,9 +287,7 @@ impl PyFloat { let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? + objint::get_float_value(&other, vm)? } else { return Ok(vm.ctx.not_implemented()); }; @@ -313,9 +305,7 @@ impl PyFloat { let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? + objint::get_float_value(&other, vm)? } else { return Ok(vm.ctx.not_implemented()); }; diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 20f5de4ca..c9a5c9a40 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -535,6 +535,12 @@ pub fn get_value(obj: &PyObjectRef) -> &BigInt { &obj.payload::().unwrap().value } +pub fn get_float_value(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + get_value(obj).to_f64().ok_or_else(|| { + vm.new_overflow_error("OverflowError: int too large to convert to float".to_string()) + }) +} + #[inline] fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { if i2.is_zero() { From 20b2f64cef8279fc59c20d6f5825bd2daf3e039b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 20:15:02 +0900 Subject: [PATCH 392/658] Add float.__rmod__ --- tests/snippets/floats.py | 7 ++++++- vm/src/obj/objfloat.rs | 24 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index f5877604e..c4f7bdb64 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -36,7 +36,7 @@ assert z - a == 0.8 assert a / z == 0.6 assert 6 / a == 5.0 assert 2.0 % z == 0.0 -# assert z % 2.0 == 0.0 +assert z % 2.0 == 0.0 assert a < 5 assert a <= 5 @@ -100,6 +100,11 @@ assert 1.0.__truediv__(2) == 0.5 assert 1.0.__rtruediv__(2) == 2.0 assert 2.0.__mul__(1) == 2.0 assert 2.0.__rsub__(1) == -1.0 +assert 2.0.__mod__(2) == 0.0 +assert 2.0.__rmod__(2) == 0.0 +assert_raises(ZeroDivisionError, lambda: 2.0 / 0) +assert_raises(ZeroDivisionError, lambda: 2.0 // 0) +assert_raises(ZeroDivisionError, lambda: 2.0 % 0) assert 1.2.__int__() == 1 assert 1.2.__float__() == 1.2 diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 099d18bb3..a588eee6a 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -41,6 +41,14 @@ impl From for PyFloat { } } +fn mod_(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok(vm.ctx.new_float(v1 % v2)) + } else { + Err(vm.new_zero_division_error("float mod by zero".to_string())) + } +} + #[pyimpl] impl PyFloat { #[pymethod(name = "__eq__")] @@ -223,11 +231,19 @@ impl PyFloat { return Ok(vm.ctx.not_implemented()); }; - if v2 != 0.0 { - Ok(vm.ctx.new_float(v1 % v2)) + mod_(v1, v2, vm) + } + + #[pymethod(name = "__rmod__")] + fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let v2 = self.value; + let v1 = if objtype::isinstance(&other, &vm.ctx.int_type) { + objint::get_float_value(&other, vm)? } else { - Err(vm.new_zero_division_error("float mod by zero".to_string())) - } + return Ok(vm.ctx.not_implemented()); + }; + + mod_(v1, v2, vm) } #[pymethod(name = "__neg__")] From f1551af0f5b4f31c14240f12d62313541c9fdf91 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 28 Apr 2019 18:52:52 +0300 Subject: [PATCH 393/658] Add tests - not working --- tests/snippets/try_exceptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 83e00c0fe..5f49a40d0 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -167,3 +167,22 @@ try: raise NameError except NameError as ex2: assert ex2.__context__ == None + +def f(): + raise + +with assertRaises(ZeroDivisionError): + try: + 1/0 + except: + f() + +with assertRaises(ZeroDivisionError): + try: + 1/0 + except ZeroDivisionError: + try: + raise + except NameError: + pass + raise From af7a89aa99cf3b66bf9103f46d4945aeb6c454d7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 28 Apr 2019 19:06:14 +0300 Subject: [PATCH 394/658] pop exception on context --- vm/src/frame.rs | 4 ++-- vm/src/vm.rs | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 69b1d510a..00d843d2a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -739,8 +739,8 @@ impl Frame { }; let context = match argc { 0 => vm.get_none(), // We have already got the exception, - _ => match vm.last_exception() { - Some(exc) => exc.clone(), + _ => match vm.pop_exception() { + Some(exc) => exc, None => vm.get_none(), }, }; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 9308c0caa..a677d7341 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -931,17 +931,6 @@ impl VirtualMachine { pub fn pop_exception(&self) -> Option { self.exceptions.borrow_mut().pop() } - - pub fn last_exception(&self) -> Option> { - let exceptions = self.exceptions.borrow(); - if exceptions.is_empty() { - None - } else { - Some(Ref::map(self.exceptions.borrow(), |exceptions| { - exceptions.last().unwrap() - })) - } - } } impl Default for VirtualMachine { From b72fc3bc9d57cc23f308c73b113d170a36cf8716 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 28 Apr 2019 19:20:16 +0300 Subject: [PATCH 395/658] Add test for #867 --- tests/snippets/try_exceptions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 5f49a40d0..c6a148e78 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -182,7 +182,14 @@ with assertRaises(ZeroDivisionError): 1/0 except ZeroDivisionError: try: - raise + raise except NameError: pass raise + +# Regression https://github.com/RustPython/RustPython/issues/867 +for _ in [1, 2]: + try: + raise ArithmeticError() + except ArithmeticError as e: + continue From c3714c2eca8a5f4416ec181188c43ba29b4fe714 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 19:22:03 +0900 Subject: [PATCH 396/658] Add math.trunc --- tests/snippets/math_basics.py | 25 ++++++++++++++++++++++++- vm/src/stdlib/math.rs | 19 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 09f3ed3b3..c707f0458 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -1,3 +1,6 @@ +import math +from testutils import assertRaises + a = 4 #print(a ** 3) @@ -17,7 +20,6 @@ assert a - 3 == 1 assert -a == -4 assert +a == 4 -# import math # assert(math.exp(2) == math.exp(2.0)) # assert(math.exp(True) == math.exp(1.0)) # @@ -27,3 +29,24 @@ assert +a == 4 # return 1.1111 # # assert math.log(1.1111) == math.log(Conversible()) + +# roundings +assert math.trunc(1) == 1 + +class A(object): + def __trunc__(self): + return 2 + +assert math.trunc(A()) == 2 + +class A(object): + def __trunc__(self): + return 2.0 + +assert math.trunc(A()) == 2.0 + +class A(object): + def __trunc__(self): + return 'str' + +assert math.trunc(A()) == 'str' diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 2ee584a79..82d8ac57d 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; // Helper macro: @@ -172,6 +172,20 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn math_trunc(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + const MAGIC_NAME: &str = "__trunc__"; + if let Ok(method) = vm.get_method(value.clone(), MAGIC_NAME) { + vm.invoke(method, vec![]) + } else { + Err(vm.new_type_error(format!( + "TypeError: type {} doesn't define {} method", + value.class().name, + MAGIC_NAME, + ))) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -219,6 +233,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "gamma" => ctx.new_rustfunc(math_gamma), "lgamma" => ctx.new_rustfunc(math_lgamma), + // Rounding functions: + "trunc" => ctx.new_rustfunc(math_trunc), + // Constants: "pi" => ctx.new_float(std::f64::consts::PI), // 3.14159... "e" => ctx.new_float(std::f64::consts::E), // 2.71.. From 6afe78c9a673a13e4ae277c98b9c3722e6afa6b8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 20:43:25 +0900 Subject: [PATCH 397/658] Add math.ceil / math.floor --- tests/snippets/math_basics.py | 32 -------------- tests/snippets/math_module.py | 80 +++++++++++++++++++++++++++++++++++ vm/src/stdlib/math.rs | 37 +++++++++++++--- 3 files changed, 111 insertions(+), 38 deletions(-) create mode 100644 tests/snippets/math_module.py diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index c707f0458..16d6e011c 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -1,4 +1,3 @@ -import math from testutils import assertRaises a = 4 @@ -19,34 +18,3 @@ assert 2 == a / 2 assert a - 3 == 1 assert -a == -4 assert +a == 4 - -# assert(math.exp(2) == math.exp(2.0)) -# assert(math.exp(True) == math.exp(1.0)) -# -# class Conversible(): -# def __float__(self): -# print("Converting to float now!") -# return 1.1111 -# -# assert math.log(1.1111) == math.log(Conversible()) - -# roundings -assert math.trunc(1) == 1 - -class A(object): - def __trunc__(self): - return 2 - -assert math.trunc(A()) == 2 - -class A(object): - def __trunc__(self): - return 2.0 - -assert math.trunc(A()) == 2.0 - -class A(object): - def __trunc__(self): - return 'str' - -assert math.trunc(A()) == 'str' diff --git a/tests/snippets/math_module.py b/tests/snippets/math_module.py new file mode 100644 index 000000000..1064610f1 --- /dev/null +++ b/tests/snippets/math_module.py @@ -0,0 +1,80 @@ +import math +from testutils import assertRaises + +# assert(math.exp(2) == math.exp(2.0)) +# assert(math.exp(True) == math.exp(1.0)) +# +# class Conversible(): +# def __float__(self): +# print("Converting to float now!") +# return 1.1111 +# +# assert math.log(1.1111) == math.log(Conversible()) + +# roundings +assert int.__trunc__ +assert int.__floor__ +assert int.__ceil__ + +# assert float.__trunc__ +with assertRaises(AttributeError): + assert float.__floor__ +with assertRaises(AttributeError): + assert float.__ceil__ + +assert math.trunc(2) == 2 +assert math.ceil(3) == 3 +assert math.floor(4) == 4 + +assert math.trunc(2.2) == 2 +assert math.ceil(3.3) == 4 +assert math.floor(4.4) == 4 + +class A(object): + def __trunc__(self): + return 2 + + def __ceil__(self): + return 3 + + def __floor__(self): + return 4 + +assert math.trunc(A()) == 2 +assert math.ceil(A()) == 3 +assert math.floor(A()) == 4 + +class A(object): + def __trunc__(self): + return 2.2 + + def __ceil__(self): + return 3.3 + + def __floor__(self): + return 4.4 + +assert math.trunc(A()) == 2.2 +assert math.ceil(A()) == 3.3 +assert math.floor(A()) == 4.4 + +class A(object): + def __trunc__(self): + return 'trunc' + + def __ceil__(self): + return 'ceil' + + def __floor__(self): + return 'floor' + +assert math.trunc(A()) == 'trunc' +assert math.ceil(A()) == 'ceil' +assert math.floor(A()) == 'floor' + +with assertRaises(TypeError): + math.trunc(object()) +with assertRaises(TypeError): + math.ceil(object()) +with assertRaises(TypeError): + math.floor(object()) diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 82d8ac57d..58b3ee953 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -7,7 +7,7 @@ use statrs::function::erf::{erf, erfc}; use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; -use crate::obj::objfloat; +use crate::obj::{objfloat, objtype}; use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -172,20 +172,43 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -fn math_trunc(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(value, None)]); - const MAGIC_NAME: &str = "__trunc__"; - if let Ok(method) = vm.get_method(value.clone(), MAGIC_NAME) { +fn try_magic_method(func_name: &str, vm: &VirtualMachine, value: &PyObjectRef) -> PyResult { + if let Ok(method) = vm.get_method(value.clone(), func_name) { vm.invoke(method, vec![]) } else { Err(vm.new_type_error(format!( "TypeError: type {} doesn't define {} method", value.class().name, - MAGIC_NAME, + func_name, ))) } } +fn math_trunc(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + try_magic_method("__trunc__", vm, value) +} + +fn math_ceil(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + if objtype::isinstance(value, &vm.ctx.float_type) { + let v = objfloat::get_value(value); + Ok(vm.ctx.new_float(v.ceil())) + } else { + try_magic_method("__ceil__", vm, value) + } +} + +fn math_floor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + if objtype::isinstance(value, &vm.ctx.float_type) { + let v = objfloat::get_value(value); + Ok(vm.ctx.new_float(v.floor())) + } else { + try_magic_method("__floor__", vm, value) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -235,6 +258,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { // Rounding functions: "trunc" => ctx.new_rustfunc(math_trunc), + "ceil" => ctx.new_rustfunc(math_ceil), + "floor" => ctx.new_rustfunc(math_floor), // Constants: "pi" => ctx.new_float(std::f64::consts::PI), // 3.14159... From 388b62aae20f378248c74057efa4490867df8133 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 28 Apr 2019 13:44:06 +0900 Subject: [PATCH 398/658] objint::to_int to call __int__ --- tests/snippets/ints.py | 34 ++++++++++++++++++++++++++++++++++ vm/src/obj/objint.rs | 26 +++++++++++++++++--------- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 1fbbc2bef..a072cc542 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -74,3 +74,37 @@ with assertRaises(TypeError): with assertRaises(TypeError): # check that first parameter is truly positional only int(val_options=1) + +class A(object): + def __int__(self): + return 10 + +assert int(A()) == 10 + +class B(object): + pass + +b = B() +b.__int__ = lambda: 20 + +with assertRaises(TypeError): + assert int(b) == 20 + +class C(object): + def __int__(self): + return 'str' + +with assertRaises(TypeError): + int(C()) + +class I(int): + def __int__(self): + return 3 + +assert int(I(1)) == 3 + +class F(float): + def __int__(self): + return 3 + +assert int(F(1.2)) == 3 diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index c9a5c9a40..3687ed2b0 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,12 +1,12 @@ use std::fmt; use std::hash::{Hash, Hasher}; -use num_bigint::{BigInt, ToBigInt}; +use num_bigint::BigInt; use num_integer::Integer; use num_traits::{Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; -use crate::function::OptionalArg; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -510,11 +510,8 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul } // Casting function: -// TODO: this should just call `__int__` on the object pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { match_class!(obj.clone(), - i @ PyInt => Ok(i.as_bigint().clone()), - f @ PyFloat => Ok(f.to_f64().to_bigint().unwrap()), s @ PyString => { i32::from_str_radix(s.as_str(), base) .map(BigInt::from) @@ -523,10 +520,21 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult Err(vm.new_type_error(format!( - "int() argument must be a string or a number, not '{}'", - obj.class().name - ))) + obj => { + if let Ok(f) = vm.get_method(obj.clone(), "__int__") { + let int_res = vm.invoke(f, PyFuncArgs::default())?; + match int_res.payload::() { + Some(i) => Ok(i.as_bigint().clone()), + None => Err(vm.new_type_error(format!( + "TypeError: __int__ returned non-int (type '{}')", int_res.class().name))), + } + } else { + Err(vm.new_type_error(format!( + "int() argument must be a string or a number, not '{}'", + obj.class().name + ))) + } + } ) } From 81dd5faa81bc65b10cdd0b540b939dc2318c436a Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 25 Apr 2019 11:13:51 +1200 Subject: [PATCH 399/658] Accept a tuple for the second argument of isinstance and issubclass --- tests/snippets/types_snippet.py | 6 +++++ vm/src/builtins.rs | 48 ++++++++++++++++++++++++--------- vm/src/vm.rs | 8 ++++-- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/tests/snippets/types_snippet.py b/tests/snippets/types_snippet.py index 075957418..13395cdbe 100644 --- a/tests/snippets/types_snippet.py +++ b/tests/snippets/types_snippet.py @@ -27,6 +27,12 @@ assert None.__class__ is type(None) assert isinstance(type, type) assert issubclass(type, type) +assert not isinstance(type, (int, float)) +assert isinstance(type, (int, object)) + +assert not issubclass(type, (int, float)) +assert issubclass(type, (int, type)) + class A: pass class B(A): pass class C(A): pass diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 9f393d333..360b37e9a 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -20,12 +20,13 @@ use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + Either, IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; use crate::obj::objcode::PyCodeRef; +use crate::obj::objtuple::PyTupleRef; #[cfg(not(target_arch = "wasm32"))] use crate::stdlib::io::io_open; @@ -312,19 +313,42 @@ fn builtin_id(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_input -fn builtin_isinstance(obj: PyObjectRef, typ: PyClassRef, vm: &VirtualMachine) -> PyResult { - vm.isinstance(&obj, &typ) +fn builtin_isinstance( + obj: PyObjectRef, + typ: Either, + vm: &VirtualMachine, +) -> PyResult { + match typ { + Either::A(ref cls) => vm.isinstance(&obj, cls), + Either::B(ref tuple) => { + for cls_obj in tuple.elements.borrow().iter() { + let cls = PyClassRef::try_from_object(vm, cls_obj.clone())?; + if vm.isinstance(&obj, &cls)? { + return Ok(true); + } + } + Ok(false) + } + } } -fn builtin_issubclass(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(subclass, Some(vm.get_type())), (cls, Some(vm.get_type()))] - ); - - let issubclass = vm.issubclass(subclass, cls)?; - Ok(vm.context().new_bool(issubclass)) +fn builtin_issubclass( + subclass: PyClassRef, + typ: Either, + vm: &VirtualMachine, +) -> PyResult { + match typ { + Either::A(ref cls) => vm.issubclass(&subclass, cls), + Either::B(ref tuple) => { + for cls_obj in tuple.elements.borrow().iter() { + let cls = PyClassRef::try_from_object(vm, cls_obj.clone())?; + if vm.issubclass(&subclass, &cls)? { + return Ok(true); + } + } + Ok(false) + } + } } fn builtin_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index ab15e14e9..068d7df19 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -304,8 +304,12 @@ impl VirtualMachine { /// Determines if `subclass` is a subclass of `cls`, either directly, indirectly or virtually /// via the __subclasscheck__ magic method. - pub fn issubclass(&self, subclass: &PyObjectRef, cls: &PyObjectRef) -> PyResult { - let ret = self.call_method(cls, "__subclasscheck__", vec![subclass.clone()])?; + pub fn issubclass(&self, subclass: &PyClassRef, cls: &PyClassRef) -> PyResult { + let ret = self.call_method( + cls.as_object(), + "__subclasscheck__", + vec![subclass.clone().into_object()], + )?; objbool::boolval(self, ret) } From 4a4631a622a3c73b15eb5f3c2bfbbdf7969350fa Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 28 Apr 2019 16:58:42 +1200 Subject: [PATCH 400/658] Use match_class instead of Either in isinstance/issubclass --- vm/src/builtins.rs | 47 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 360b37e9a..41082707e 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -11,22 +11,22 @@ use num_traits::Signed; use crate::compile; use crate::import::import_module; use crate::obj::objbool; +use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objint::{self, PyIntRef}; use crate::obj::objiter; use crate::obj::objstr::{self, PyString, PyStringRef}; -use crate::obj::objtype::{self, PyClassRef}; +use crate::obj::objtuple::PyTuple; +use crate::obj::objtype::{self, PyClass, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - Either, IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; -use crate::obj::objcode::PyCodeRef; -use crate::obj::objtuple::PyTupleRef; #[cfg(not(target_arch = "wasm32"))] use crate::stdlib::io::io_open; @@ -313,42 +313,37 @@ fn builtin_id(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_input -fn builtin_isinstance( - obj: PyObjectRef, - typ: Either, +fn type_test( vm: &VirtualMachine, + typ: PyObjectRef, + test: impl Fn(&PyClassRef) -> PyResult, + test_name: &str, ) -> PyResult { - match typ { - Either::A(ref cls) => vm.isinstance(&obj, cls), - Either::B(ref tuple) => { + match_class!(typ, + cls @ PyClass => test(&cls), + tuple @ PyTuple => { for cls_obj in tuple.elements.borrow().iter() { let cls = PyClassRef::try_from_object(vm, cls_obj.clone())?; - if vm.isinstance(&obj, &cls)? { + if test(&cls)? { return Ok(true); } } Ok(false) - } - } + }, + _ => Err(vm.new_type_error(format!("{}() arg 2 must be a type or tuple of types", test_name))) + ) +} + +fn builtin_isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult { + type_test(vm, typ, |cls| vm.isinstance(&obj, cls), "isinstance") } fn builtin_issubclass( subclass: PyClassRef, - typ: Either, + typ: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - match typ { - Either::A(ref cls) => vm.issubclass(&subclass, cls), - Either::B(ref tuple) => { - for cls_obj in tuple.elements.borrow().iter() { - let cls = PyClassRef::try_from_object(vm, cls_obj.clone())?; - if vm.issubclass(&subclass, &cls)? { - return Ok(true); - } - } - Ok(false) - } - } + type_test(vm, typ, |cls| vm.issubclass(&subclass, cls), "issubclass") } fn builtin_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { From f8373aa38a4da14d1cadebd5e5b068f8340357b6 Mon Sep 17 00:00:00 2001 From: Nicolas Trinquier Date: Sun, 28 Apr 2019 22:57:34 +0200 Subject: [PATCH 401/658] Add mul method for the complex type --- tests/snippets/builtin_complex.py | 5 +++++ vm/src/obj/objcomplex.rs | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 2df8e4df7..9c13e2d85 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -24,6 +24,11 @@ assert complex(1, 2) != complex(1, 1) assert complex(1, 2) != 'foo' assert complex(1, 2).__eq__('foo') == NotImplemented +# __mul__ + +assert complex(2, -3) * complex(-5, 7) == complex(-21, 29) +assert complex(2, -3) * 5 == complex(10, -15) + # __neg__ assert -complex(1, -1) == complex(-1, 1) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 93a0559aa..3dd48c3f9 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -161,6 +161,18 @@ impl PyComplex { vm.ctx.new_bool(result) } + #[pymethod(name = "__mul__")] + fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + match to_complex(other, vm) { + Ok(Some(other)) => Ok(vm.ctx.new_complex(Complex64::new( + self.value.re * other.re - self.value.im * other.im, + self.value.re * other.im + self.value.re * other.im, + ))), + Ok(None) => Ok(vm.ctx.not_implemented()), + Err(err) => Err(err), + } + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> PyComplex { PyComplex::from(-self.value) From eebdbfe679237a3efb0ec1605d9e41e875dbd5c1 Mon Sep 17 00:00:00 2001 From: Nicolas Trinquier Date: Sun, 28 Apr 2019 22:57:56 +0200 Subject: [PATCH 402/658] Add int and float methods for the complex type --- tests/snippets/builtin_complex.py | 2 +- vm/src/obj/objcomplex.rs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 9c13e2d85..4d0fa1202 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -26,7 +26,7 @@ assert complex(1, 2).__eq__('foo') == NotImplemented # __mul__ -assert complex(2, -3) * complex(-5, 7) == complex(-21, 29) +assert complex(2, -3) * complex(-5, 7) == complex(11, 29) assert complex(2, -3) * 5 == complex(10, -15) # __neg__ diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 3dd48c3f9..cd70c61c7 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -161,12 +161,22 @@ impl PyComplex { vm.ctx.new_bool(result) } + #[pymethod(name = "__float__")] + fn float(&self, vm: &VirtualMachine) -> PyResult { + return Err(vm.new_type_error(String::from("Can't convert complex to float"))); + } + + #[pymethod(name = "__int__")] + fn int(&self, vm: &VirtualMachine) -> PyResult { + return Err(vm.new_type_error(String::from("Can't convert complex to int"))); + } + #[pymethod(name = "__mul__")] - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match to_complex(other, vm) { Ok(Some(other)) => Ok(vm.ctx.new_complex(Complex64::new( self.value.re * other.re - self.value.im * other.im, - self.value.re * other.im + self.value.re * other.im, + self.value.re * other.im + self.value.im * other.re, ))), Ok(None) => Ok(vm.ctx.not_implemented()), Err(err) => Err(err), From 56ef6ec5f811cd3fc269bf083d58657c313fc45a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 29 Apr 2019 20:08:15 +0300 Subject: [PATCH 403/658] Change to is_some --- vm/src/frame.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 00d843d2a..a9e155969 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -861,7 +861,7 @@ impl Frame { Ok(None) } bytecode::Instruction::PopException {} => { - assert!(!vm.pop_exception().is_none()); + assert!(vm.pop_exception().is_some()); Ok(None) } } From 68e15a2bd88b4fc55f92647ef3cdeff455a71f9d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 29 Apr 2019 14:46:04 +0900 Subject: [PATCH 404/658] Add basic float.__round__ without ndigits support --- tests/snippets/math_basics.py | 7 +++++++ vm/src/obj/objfloat.rs | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 16d6e011c..a9092d58d 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -18,3 +18,10 @@ assert 2 == a / 2 assert a - 3 == 1 assert -a == -4 assert +a == 4 + +assert round(1.2) == 1 +assert round(1.8) == 2 +assert round(0.5) == 0 +assert round(1.5) == 2 +assert round(-0.5) == 0 +assert round(-1.5) == -2 diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index a588eee6a..f40c403a8 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -2,9 +2,11 @@ use super::objbytes; use super::objint; use super::objstr; use super::objtype; +use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, }; use crate::vm::VirtualMachine; use num_bigint::{BigInt, ToBigInt}; @@ -357,6 +359,36 @@ impl PyFloat { self.value.to_bigint().unwrap() } + #[pymethod(name = "__round__")] + fn round(&self, ndigits: OptionalArg, vm: &VirtualMachine) -> PyObjectRef { + let ndigits = match ndigits { + OptionalArg::Missing => None, + OptionalArg::Present(ref value) => { + if !vm.get_none().is(value) { + // retrive int to implement it + Some(vm.ctx.not_implemented()) + } else { + None + } + } + }; + if ndigits.is_none() { + let fract = self.value.fract(); + let value = if fract.abs() == 0.5 { + if self.value.trunc() % 2.0 == 0.0 { + self.value - fract + } else { + self.value + fract + } + } else { + self.value.round() + }; + vm.ctx.new_int(value.to_bigint().unwrap()) + } else { + vm.ctx.not_implemented() + } + } + #[pymethod(name = "__int__")] fn int(&self, vm: &VirtualMachine) -> BigInt { self.trunc(vm) From 490db0a84f8b6207392617adeadf7e42869a008e Mon Sep 17 00:00:00 2001 From: Nicolas Trinquier Date: Mon, 29 Apr 2019 20:13:01 +0200 Subject: [PATCH 405/658] Add support for complex type for the to_complex function --- vm/src/obj/objcomplex.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index cd70c61c7..f9ff9d9b0 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -67,7 +67,9 @@ impl PyComplexRef { } fn to_complex(value: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - if objtype::isinstance(&value, &vm.ctx.int_type()) { + if objtype::isinstance(&value, &vm.ctx.complex_type()) { + Ok(Some(get_value(&value))) + } else if objtype::isinstance(&value, &vm.ctx.int_type()) { match objint::get_value(&value).to_f64() { Some(v) => Ok(Some(Complex64::new(v, 0.0))), None => Err(vm.new_overflow_error("int too large to convert to float".to_string())), From 4c8f655d4ee6d0aef5ae65334a9cdc2666d9a04e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 01:37:47 +0900 Subject: [PATCH 406/658] float.__round__ inf/nan support --- tests/snippets/math_basics.py | 15 ++++++++++++++- vm/src/obj/objfloat.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index a9092d58d..51ea5c726 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -1,4 +1,4 @@ -from testutils import assertRaises +from testutils import assert_raises a = 4 @@ -25,3 +25,16 @@ assert round(0.5) == 0 assert round(1.5) == 2 assert round(-0.5) == 0 assert round(-1.5) == -2 + +assert_raises( + ValueError, + lambda: round(float('nan')), + 'ValueError: cannot convert float NaN to integer') +assert_raises( + OverflowError, + lambda: round(float('inf')), + 'OverflowError: cannot convert float NaN to integer') +assert_raises( + OverflowError, + lambda: round(-float('inf')), + 'OverflowError: cannot convert float NaN to integer') diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index f40c403a8..bb7a65975 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -51,6 +51,28 @@ fn mod_(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { } } +fn try_to_bigint(value: f64, vm: &VirtualMachine) -> PyResult { + match value.to_bigint() { + Some(int) => Ok(int), + None => { + if value.is_infinite() { + Err(vm.new_overflow_error( + "OverflowError: cannot convert float NaN to integer".to_string(), + )) + } else if value.is_nan() { + Err(vm + .new_value_error("ValueError: cannot convert float NaN to integer".to_string())) + } else { + // unreachable unless BigInt has a bug + unreachable!( + "A finite float value failed to be converted to bigint: {}", + value + ) + } + } + } +} + #[pyimpl] impl PyFloat { #[pymethod(name = "__eq__")] @@ -360,7 +382,7 @@ impl PyFloat { } #[pymethod(name = "__round__")] - fn round(&self, ndigits: OptionalArg, vm: &VirtualMachine) -> PyObjectRef { + fn round(&self, ndigits: OptionalArg, vm: &VirtualMachine) -> PyResult { let ndigits = match ndigits { OptionalArg::Missing => None, OptionalArg::Present(ref value) => { @@ -383,9 +405,10 @@ impl PyFloat { } else { self.value.round() }; - vm.ctx.new_int(value.to_bigint().unwrap()) + let int = try_to_bigint(value, vm)?; + Ok(vm.ctx.new_int(int)) } else { - vm.ctx.not_implemented() + Ok(vm.ctx.not_implemented()) } } From b7c7591639617e845d0d7e8c141887a1e2bf91e2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 01:48:03 +0900 Subject: [PATCH 407/658] Fix float.__trunc__ to handle abnormal values --- tests/snippets/floats.py | 4 +++- vm/src/obj/objfloat.rs | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index c4f7bdb64..8820813f1 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -111,7 +111,9 @@ assert 1.2.__float__() == 1.2 assert 1.2.__trunc__() == 1 assert int(1.2) == 1 assert float(1.2) == 1.2 -# assert math.trunc(1.2) == 1 +assert math.trunc(1.2) == 1 +assert_raises(OverflowError, float('inf').__trunc__) +assert_raises(ValueError, float('nan').__trunc__) assert (1.7).real == 1.7 assert (1.3).is_integer() == False diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index bb7a65975..80c325a33 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -377,8 +377,8 @@ impl PyFloat { } #[pymethod(name = "__trunc__")] - fn trunc(&self, _vm: &VirtualMachine) -> BigInt { - self.value.to_bigint().unwrap() + fn trunc(&self, vm: &VirtualMachine) -> PyResult { + try_to_bigint(self.value, vm) } #[pymethod(name = "__round__")] @@ -413,7 +413,7 @@ impl PyFloat { } #[pymethod(name = "__int__")] - fn int(&self, vm: &VirtualMachine) -> BigInt { + fn int(&self, vm: &VirtualMachine) -> PyResult { self.trunc(vm) } From 132d34fc67c908356c81435666d2ac805566cec6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 27 Apr 2019 00:27:47 +0900 Subject: [PATCH 408/658] Add range.__eq__ --- tests/snippets/builtin_range.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index a8797f75a..55d4f5194 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -27,6 +27,11 @@ assert range(4, 10, 2).count(4) == 1 assert range(4, 10, 2).count(7) == 0 assert range(10).count("foo") == 0 +# __eq__ +assert range(1, 2, 3) == range(1, 2, 3) +assert range(1, 2, 1) == range(1, 2) +assert range(2) == range(0, 2) + # __bool__ assert bool(range(1)) assert bool(range(1, 2)) From c951cb3cfd693536968dee2c675862b01761210e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 03:27:06 +0900 Subject: [PATCH 409/658] float.__round__ for ndigits == 0 --- tests/snippets/floats.py | 8 ++++++++ vm/src/obj/objfloat.rs | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 8820813f1..176ce927e 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -114,6 +114,14 @@ assert float(1.2) == 1.2 assert math.trunc(1.2) == 1 assert_raises(OverflowError, float('inf').__trunc__) assert_raises(ValueError, float('nan').__trunc__) +assert 0.5.__round__() == 0.0 +assert 1.5.__round__() == 2.0 +assert 0.5.__round__(0) == 0.0 +assert 1.5.__round__(0) == 2.0 +assert 0.5.__round__(None) == 0.0 +assert 1.5.__round__(None) == 2.0 +assert_raises(OverflowError, float('inf').__round__) +assert_raises(ValueError, float('nan').__round__) assert (1.7).real == 1.7 assert (1.3).is_integer() == False diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 80c325a33..42810da8e 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -11,7 +11,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; -use num_traits::ToPrimitive; +use num_traits::{ToPrimitive, Zero}; #[pyclass(name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] @@ -387,8 +387,19 @@ impl PyFloat { OptionalArg::Missing => None, OptionalArg::Present(ref value) => { if !vm.get_none().is(value) { - // retrive int to implement it - Some(vm.ctx.not_implemented()) + let ndigits = if objtype::isinstance(value, &vm.ctx.int_type()) { + objint::get_value(value) + } else { + return Err(vm.new_type_error(format!( + "TypeError: '{}' object cannot be interpreted as an integer", + value.class().name + ))); + }; + if ndigits.is_zero() { + None + } else { + Some(ndigits) + } } else { None } From ece66018512e47f766646614f0ec53fd9b764327 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Tue, 30 Apr 2019 15:47:41 +0200 Subject: [PATCH 410/658] add PyByteInner as fn argument / use Either --- vm/src/obj/objbyteinner.rs | 273 +++++++++++++++++-------------------- vm/src/obj/objbytes.rs | 30 ++-- 2 files changed, 139 insertions(+), 164 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 46f55ec8d..2f0e6b903 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,5 +1,11 @@ use crate::obj::objint::PyIntRef; -use crate::obj::objslice::PySlice; +use crate::obj::objnone::PyNoneRef; +use crate::obj::objslice::PySliceRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyobject::Either; +use crate::pyobject::PyRef; +use crate::pyobject::PyValue; +use crate::pyobject::TryFromObject; use crate::pyobject::{PyIterable, PyObjectRef}; use core::ops::Range; use num_bigint::BigInt; @@ -16,7 +22,7 @@ use std::hash::{Hash, Hasher}; use super::objint; use super::objsequence::{is_valid_slice_arg, PySliceableSequence}; -use super::objtype; + use crate::obj::objint::PyInt; use num_integer::Integer; use num_traits::ToPrimitive; @@ -24,7 +30,7 @@ use num_traits::ToPrimitive; use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; use super::objbytes::PyBytes; use super::objmemory::PyMemoryView; -use super::objnone::PyNone; + use super::objsequence; #[derive(Debug, Default, Clone)] @@ -32,6 +38,37 @@ pub struct PyByteInner { pub elements: Vec, } +impl TryFromObject for PyByteInner { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + + i @ PyBytes => Ok(PyByteInner{elements: i.get_value().to_vec()}), + j @ PyByteArray => Ok(PyByteInner{elements: get_value_bytearray(&j.as_object()).to_vec()}), + k @ PyMemoryView => Ok(PyByteInner{elements: k.get_obj_value().unwrap()}), + obj => Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj.class() + ))) + ) + } +} + +impl TryFromObject for Either> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match PyByteInner::try_from_object(vm, obj.clone()) { + Ok(a) => Ok(Either::A(a)), + Err(_) => match obj.clone().downcast::() { + Ok(b) => Ok(Either::B(b)), + Err(_) => Err(vm.new_type_error(format!( + "a bytes-like object or {} is required, not {}", + B::class(vm), + obj.class() + ))), + }, + } + } +} + #[derive(FromArgs)] pub struct ByteInnerNewOptions { #[pyarg(positional_only, optional = true)] @@ -103,11 +140,11 @@ impl ByteInnerNewOptions { #[derive(FromArgs)] pub struct ByteInnerFindOptions { #[pyarg(positional_only, optional = false)] - sub: PyObjectRef, + sub: Either, #[pyarg(positional_only, optional = true)] - start: OptionalArg, + start: OptionalArg>, #[pyarg(positional_only, optional = true)] - end: OptionalArg, + end: OptionalArg>, } impl ByteInnerFindOptions { @@ -116,29 +153,19 @@ impl ByteInnerFindOptions { elements: &[u8], vm: &VirtualMachine, ) -> PyResult<(Vec, Range)> { - let sub = match try_as_bytes_like(&self.sub) { - Some(value) => value, - None => match_class!(self.sub, - i @ PyInt => vec![i.as_bigint().byte_or(vm)?], - obj => {return Err(vm.new_type_error(format!("argument should be integer or bytes-like object, not {}", obj)));}), + let sub = match self.sub { + Either::A(v) => v.elements.to_vec(), + Either::B(int) => vec![int.as_bigint().byte_or(vm)?], }; - let start = if let OptionalArg::Present(st) = self.start { - match_class!(st, - i @ PyInt => {Some(i.as_bigint().clone())}, - _obj @ PyNone => None, - _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} - ) - } else { - None + + let start = match self.start { + OptionalArg::Present(Some(int)) => Some(int.as_bigint().clone()), + _ => None, }; - let end = if let OptionalArg::Present(e) = self.end { - match_class!(e, - i @ PyInt => {Some(i.as_bigint().clone())}, - _obj @ PyNone => None, - _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} - ) - } else { - None + + let end = match self.end { + OptionalArg::Present(Some(int)) => Some(int.as_bigint().clone()), + _ => None, }; let range = elements.to_vec().get_slice_range(&start, &end); @@ -199,20 +226,16 @@ impl ByteInnerPaddingOptions { #[derive(FromArgs)] pub struct ByteInnerTranslateOptions { #[pyarg(positional_only, optional = false)] - table: PyObjectRef, + table: Either, #[pyarg(positional_or_keyword, optional = true)] - delete: OptionalArg, + delete: OptionalArg, } impl ByteInnerTranslateOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec, Vec)> { - let table = match try_as_bytes_like(&self.table) { - Some(value) => value, - None => match_class!(self.table, - - _n @ PyNone => (0..=255).collect::>(), - obj => {return Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)));}, - ), + let table = match self.table { + Either::A(v) => v.elements.to_vec(), + Either::B(_) => (0..=255).collect::>(), }; if table.len() != 256 { @@ -221,18 +244,9 @@ impl ByteInnerTranslateOptions { ); } - let delete = if let OptionalArg::Present(value) = &self.delete { - match try_as_bytes_like(&value) { - Some(value) => value, - None => { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - value - ))); - } - } - } else { - vec![] + let delete = match self.delete { + OptionalArg::Present(byte) => byte.elements, + _ => vec![], }; Ok((table, delete)) @@ -319,54 +333,45 @@ impl PyByteInner { elements } - pub fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match try_as_bytes_like(&needle) { - Some(value) => self.contains_bytes(&value, vm), - None => match_class!(needle, - i @ PyInt => self.contains_int(&i, vm), - obj => {Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))}), - } - } - - fn contains_bytes(&self, other: &[u8], vm: &VirtualMachine) -> PyResult { - for (n, i) in self.elements.iter().enumerate() { - if n + other.len() <= self.len() - && *i == other[0] - && &self.elements[n..n + other.len()] == other - { - return Ok(vm.new_bool(true)); + pub fn contains(&self, needle: Either, vm: &VirtualMachine) -> PyResult { + match needle { + Either::A(byte) => { + let other = &byte.elements[..]; + for (n, i) in self.elements.iter().enumerate() { + if n + other.len() <= self.len() + && *i == other[0] + && &self.elements[n..n + other.len()] == other + { + return Ok(vm.new_bool(true)); + } + } + Ok(vm.new_bool(false)) + } + Either::B(int) => { + if self.elements.contains(&int.as_bigint().byte_or(vm)?) { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } } } - Ok(vm.new_bool(false)) } - fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { - if self.elements.contains(&int.as_bigint().byte_or(vm)?) { - Ok(vm.new_bool(true)) - } else { - Ok(vm.new_bool(false)) + pub fn getitem(&self, needle: Either, vm: &VirtualMachine) -> PyResult { + match needle { + Either::A(int) => { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + Ok(vm.new_int(self.elements[idx])) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + Either::B(slice) => Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)), } } - pub fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(needle, - int @ PyInt => //self.inner.getitem_int(&int, vm), - { - if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { - Ok(vm.new_int(self.elements[idx])) - } else { - Err(vm.new_index_error("index out of range".to_string())) - } - }, - slice @ PySlice => //self.inner.getitem_slice(slice.as_object(), vm), - { - Ok(vm - .ctx - .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) - }, - obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) - } - pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { Ok(vm.new_bool( !self.elements.is_empty() @@ -475,7 +480,7 @@ impl PyByteInner { } pub fn swapcase(&self, _vm: &VirtualMachine) -> Vec { - let mut new: Vec = Vec::new(); + let mut new: Vec = Vec::with_capacity(self.elements.len()); for w in &self.elements { match w { 65..=90 => new.push(w.to_ascii_lowercase()), @@ -532,7 +537,6 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - // let fn_name = "center".to_string(); let (fillbyte, diff) = options.get_value("center", self.len(), vm)?; let mut ln: usize = diff / 2; @@ -559,7 +563,6 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - // let fn_name = "ljust".to_string(); let (fillbyte, diff) = options.get_value("ljust", self.len(), vm)?; // merge all @@ -575,7 +578,6 @@ impl PyByteInner { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult> { - // let fn_name = "rjust".to_string(); let (fillbyte, diff) = options.get_value("rjust", self.len(), vm)?; // merge all @@ -629,35 +631,28 @@ impl PyByteInner { pub fn startsendswith( &self, - arg: PyObjectRef, + arg: Either, start: OptionalArg, end: OptionalArg, endswith: bool, // true for endswith, false for startswith vm: &VirtualMachine, ) -> PyResult { - let suff = if objtype::isinstance(&arg, &vm.ctx.tuple_type()) { - let mut flatten = vec![]; - for v in objsequence::get_elements(&arg).to_vec() { - match try_as_bytes_like(&v) { - None => { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - &v.class().name, - ))); + let suff = match arg { + Either::A(byte) => byte.elements, + Either::B(tuple) => { + let mut flatten = vec![]; + for v in objsequence::get_elements(tuple.as_object()).to_vec() { + match try_as_bytes_like(&v) { + None => { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + &v.class().name, + ))); + } + Some(value) => flatten.extend(value), } - Some(value) => flatten.extend(value), - } - } - flatten - } else { - match try_as_bytes_like(&arg) { - Some(value) => value, - None => { - return Err(vm.new_type_error(format!( - "endswith first arg must be bytes or a tuple of bytes, not {}", - arg - ))); } + flatten } }; @@ -715,33 +710,17 @@ impl PyByteInner { Ok(-1isize) } - pub fn maketrans(from: PyObjectRef, to: PyObjectRef, vm: &VirtualMachine) -> PyResult { + pub fn maketrans(from: PyByteInner, to: PyByteInner, vm: &VirtualMachine) -> PyResult { let mut res = vec![]; - let from = match try_as_bytes_like(&from) { - Some(value) => value, - None => { - return Err( - vm.new_type_error(format!("a bytes-like object is required, not {}", from)) - ); - } - }; - - let to = match try_as_bytes_like(&to) { - Some(value) => value, - None => { - return Err( - vm.new_type_error(format!("a bytes-like object is required, not {}", to)) - ); - } - }; - for i in 0..=255 { - res.push(if let Some(position) = from.iter().position(|&x| x == i) { - to[position] - } else { - i - }); + res.push( + if let Some(position) = from.elements.iter().position(|&x| x == i) { + to.elements[position] + } else { + i + }, + ); } Ok(vm.ctx.new_bytes(res)) @@ -763,20 +742,12 @@ impl PyByteInner { pub fn strip( &self, - chars: OptionalArg, + chars: OptionalArg, position: ByteInnerPosition, - vm: &VirtualMachine, + _vm: &VirtualMachine, ) -> PyResult> { - let chars = if let OptionalArg::Present(content) = chars { - match try_as_bytes_like(&content) { - Some(value) => value, - None => { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - content - ))); - } - } + let chars = if let OptionalArg::Present(bytes) = chars { + bytes.elements } else { vec![b' '] }; diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index bd483be18..c16fd123f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,9 @@ +use crate::obj::objint::PyIntRef; +use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; + +use crate::pyobject::Either; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -35,7 +40,6 @@ impl PyBytes { inner: PyByteInner { elements }, } } - pub fn get_value(&self) -> &[u8] { &self.inner.elements } @@ -150,12 +154,12 @@ impl PyBytesRef { } #[pymethod(name = "__contains__")] - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(self, needle: Either, vm: &VirtualMachine) -> PyResult { self.inner.contains(needle, vm) } #[pymethod(name = "__getitem__")] - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn getitem(self, needle: Either, vm: &VirtualMachine) -> PyResult { self.inner.getitem(needle, vm) } @@ -256,7 +260,7 @@ impl PyBytesRef { #[pymethod(name = "endswith")] fn endswith( self, - suffix: PyObjectRef, + suffix: Either, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, @@ -267,12 +271,12 @@ impl PyBytesRef { #[pymethod(name = "startswith")] fn startswith( self, - suffix: PyObjectRef, + prefix: Either, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.inner.startsendswith(suffix, start, end, false, vm) + self.inner.startsendswith(prefix, start, end, false, vm) } #[pymethod(name = "find")] @@ -281,12 +285,12 @@ impl PyBytesRef { } #[pymethod(name = "index")] - fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { let res = self.inner.find(options, false, vm)?; if res == -1 { return Err(vm.new_value_error("substring not found".to_string())); } - Ok(vm.new_int(res)) + Ok(res) } #[pymethod(name = "rfind")] @@ -295,12 +299,12 @@ impl PyBytesRef { } #[pymethod(name = "rindex")] - fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { let res = self.inner.find(options, true, vm)?; if res == -1 { return Err(vm.new_value_error("substring not found".to_string())); } - Ok(vm.new_int(res)) + Ok(res) } #[pymethod(name = "translate")] @@ -309,21 +313,21 @@ impl PyBytesRef { } #[pymethod(name = "strip")] - fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { Ok(vm .ctx .new_bytes(self.inner.strip(chars, ByteInnerPosition::All, vm)?)) } #[pymethod(name = "lstrip")] - fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { Ok(vm .ctx .new_bytes(self.inner.strip(chars, ByteInnerPosition::Left, vm)?)) } #[pymethod(name = "rstrip")] - fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { Ok(vm .ctx .new_bytes(self.inner.strip(chars, ByteInnerPosition::Right, vm)?)) From 599f1f678f8b4318ab35a87d91d91132e3ee77f3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 06:03:41 +0900 Subject: [PATCH 411/658] extend_class for PyRange --- vm/src/obj/objrange.rs | 74 ++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index d98e7491f..a22788119 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,7 +6,7 @@ use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -15,6 +15,15 @@ use super::objiter; use super::objslice::{PySlice, PySliceRef}; use super::objtype::{self, PyClassRef}; +/// range(stop) -> range object +/// range(start, stop[, step]) -> range object +/// +/// Return an object that produces a sequence of integers from start (inclusive) +/// to stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1. +/// start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3. +/// These are exactly the valid indices for a list of 4 elements. +/// When step is given, it specifies the increment (or decrement). +#[pyclass] #[derive(Debug, Clone)] pub struct PyRange { pub start: PyIntRef, @@ -96,33 +105,7 @@ pub fn get_value(obj: &PyObjectRef) -> PyRange { } pub fn init(context: &PyContext) { - let range_type = &context.range_type; - - let range_doc = "range(stop) -> range object\n\ - range(start, stop[, step]) -> range object\n\n\ - Return an object that produces a sequence of integers from start (inclusive)\n\ - to stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1.\n\ - start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3.\n\ - These are exactly the valid indices for a list of 4 elements.\n\ - When step is given, it specifies the increment (or decrement)."; - - extend_class!(context, range_type, { - "__bool__" => context.new_rustfunc(PyRange::bool), - "__contains__" => context.new_rustfunc(PyRange::contains), - "__doc__" => context.new_str(range_doc.to_string()), - "__eq__" => context.new_rustfunc(PyRange::eq), - "__getitem__" => context.new_rustfunc(PyRange::getitem), - "__iter__" => context.new_rustfunc(PyRange::iter), - "__len__" => context.new_rustfunc(PyRange::len), - "__new__" => context.new_rustfunc(range_new), - "__repr__" => context.new_rustfunc(PyRange::repr), - "__reversed__" => context.new_rustfunc(PyRange::reversed), - "count" => context.new_rustfunc(PyRange::count), - "index" => context.new_rustfunc(PyRange::index), - "start" => context.new_property(PyRange::start), - "stop" => context.new_property(PyRange::stop), - "step" => context.new_property(PyRange::step), - }); + PyRange::extend_class(context, &context.range_type); let rangeiterator_type = &context.rangeiterator_type; extend_class!(context, rangeiterator_type, { @@ -133,6 +116,7 @@ pub fn init(context: &PyContext) { type PyRangeRef = PyRef; +#[pyimpl] impl PyRange { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { @@ -160,18 +144,22 @@ impl PyRange { .into_ref_with_type(vm, cls) } + #[pyproperty(name = "start")] fn start(&self, _vm: &VirtualMachine) -> PyIntRef { self.start.clone() } + #[pyproperty(name = "stop")] fn stop(&self, _vm: &VirtualMachine) -> PyIntRef { self.stop.clone() } + #[pyproperty(name = "step")] fn step(&self, _vm: &VirtualMachine) -> PyIntRef { self.step.clone() } + #[pymethod(name = "__iter__")] fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIterator { PyRangeIterator { position: Cell::new(0), @@ -179,6 +167,7 @@ impl PyRange { } } + #[pymethod(name = "__reversed__")] fn reversed(&self, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); @@ -211,6 +200,7 @@ impl PyRange { } } + #[pymethod(name = "__len__")] fn len(&self, _vm: &VirtualMachine) -> PyInt { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); @@ -224,6 +214,7 @@ impl PyRange { } } + #[pymethod(name = "__repr__")] fn repr(&self, _vm: &VirtualMachine) -> String { if self.step.as_bigint().is_one() { format!("range({}, {})", self.start, self.stop) @@ -232,10 +223,12 @@ impl PyRange { } } + #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { !self.is_empty() } + #[pymethod(name = "__contains__")] fn contains(&self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { match self.offset(int.as_bigint()) { @@ -247,6 +240,7 @@ impl PyRange { } } + #[pymethod(name = "__eq__")] fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.range_type()) { let rhs = get_value(&rhs); @@ -258,6 +252,7 @@ impl PyRange { } } + #[pymethod(name = "index")] fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { match self.index_of(int.as_bigint()) { @@ -269,6 +264,7 @@ impl PyRange { } } + #[pymethod(name = "count")] fn count(&self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { if let Ok(int) = item.downcast::() { if self.index_of(int.as_bigint()).is_some() { @@ -281,6 +277,7 @@ impl PyRange { } } + #[pymethod(name = "__getitem__")] fn getitem(&self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { RangeIndex::Int(index) => { @@ -327,18 +324,19 @@ impl PyRange { } } } -} -fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let range = if args.args.len() <= 2 { - let (cls, stop) = args.bind(vm)?; - PyRange::new(cls, stop, vm) - } else { - let (cls, start, stop, step) = args.bind(vm)?; - PyRange::new_from(cls, start, stop, step, vm) - }?; + #[pymethod(name = "__new__")] + fn range_new(args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let range = if args.args.len() <= 2 { + let (cls, stop) = args.bind(vm)?; + PyRange::new(cls, stop, vm) + } else { + let (cls, start, stop, step) = args.bind(vm)?; + PyRange::new_from(cls, start, stop, step, vm) + }?; - Ok(range.into_object()) + Ok(range.into_object()) + } } #[derive(Debug)] From 5d13daef207852ecde3feef346981bd1ab8d1f5d Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 30 Apr 2019 18:01:40 +0200 Subject: [PATCH 412/658] remove all try_as_bytes_like --- vm/src/obj/objbyteinner.rs | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 2f0e6b903..b75f8be69 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -612,18 +612,9 @@ impl PyByteInner { pub fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { let mut refs = vec![]; - for (index, v) in iter.iter(vm)?.enumerate() { + for v in iter.iter(vm)? { let v = v?; - match try_as_bytes_like(&v) { - None => { - return Err(vm.new_type_error(format!( - "sequence item {}: expected a bytes-like object, {} found", - index, - &v.class().name, - ))); - } - Some(value) => refs.extend(value), - } + refs.extend(PyByteInner::try_from_object(vm, v)?.elements) } Ok(vm.ctx.new_bytes(refs)) @@ -642,15 +633,7 @@ impl PyByteInner { Either::B(tuple) => { let mut flatten = vec![]; for v in objsequence::get_elements(tuple.as_object()).to_vec() { - match try_as_bytes_like(&v) { - None => { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - &v.class().name, - ))); - } - Some(value) => flatten.extend(value), - } + flatten.extend(PyByteInner::try_from_object(vm, v)?.elements) } flatten } @@ -784,15 +767,6 @@ pub fn try_as_byte(obj: &PyObjectRef) -> Option> { _ => None) } -pub fn try_as_bytes_like(obj: &PyObjectRef) -> Option> { - match_class!(obj.clone(), - - i @ PyBytes => Some(i.get_value().to_vec()), - j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), - k @ PyMemoryView => Some(k.get_obj_value().unwrap()), - _ => None) -} - pub trait ByteOr: ToPrimitive { fn byte_or(&self, vm: &VirtualMachine) -> Result { match self.to_u8() { From a318e4e09e062affcf97f12540a87724b849a1b4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:19:53 +0900 Subject: [PATCH 413/658] objfloat.rs try_float for common operand handling --- vm/src/obj/objfloat.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index a588eee6a..87ac3199d 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -41,6 +41,16 @@ impl From for PyFloat { } } +fn try_float(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { + Ok(if objtype::isinstance(&value, &vm.ctx.float_type()) { + Some(get_value(&value)) + } else if objtype::isinstance(&value, &vm.ctx.int_type()) { + Some(objint::get_float_value(&value, vm)?) + } else { + None + }) +} + fn mod_(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v2 != 0.0 { Ok(vm.ctx.new_float(v1 % v2)) From a0ad4368c618cd2f49a4425897b1b7a4181261f7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 01:59:38 +0900 Subject: [PATCH 414/658] Add float.pow OverflowError handling --- tests/snippets/floats.py | 3 +++ vm/src/obj/objfloat.rs | 15 +++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index c4f7bdb64..58855acc9 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -113,6 +113,9 @@ assert int(1.2) == 1 assert float(1.2) == 1.2 # assert math.trunc(1.2) == 1 +assert 1.2 ** 2 == 1.44 +assert_raises(OverflowError, lambda: 1.2 ** (10 ** 1000)) + assert (1.7).real == 1.7 assert (1.3).is_integer() == False assert (1.0).is_integer() == True diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 87ac3199d..23636cef8 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -262,16 +262,11 @@ impl PyFloat { } #[pymethod(name = "__pow__")] - fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - vm.ctx.new_float(v1.powf(get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - let result = v1.powf(objint::get_value(&other).to_f64().unwrap()); - vm.ctx.new_float(result) - } else { - vm.ctx.not_implemented() - } + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| self.value.powf(other).into_pyobject(vm), + ) } #[pymethod(name = "__sub__")] From 813307f8f94b89cd78165b34a469a2f247ea3417 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 30 Apr 2019 02:58:50 +0900 Subject: [PATCH 415/658] Refactor PyFloat using try_float() --- tests/snippets/floats.py | 15 +++ vm/src/obj/objfloat.rs | 191 +++++++++++++++------------------------ 2 files changed, 89 insertions(+), 117 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 58855acc9..580c5920f 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -8,6 +8,7 @@ a = 1.2 b = 1.3 c = 1.2 z = 2 +ov = 10 ** 1000 assert -a == -1.2 @@ -37,6 +38,20 @@ assert a / z == 0.6 assert 6 / a == 5.0 assert 2.0 % z == 0.0 assert z % 2.0 == 0.0 +assert_raises(OverflowError, lambda: a + ov) +assert_raises(OverflowError, lambda: a - ov) +assert_raises(OverflowError, lambda: a * ov) +assert_raises(OverflowError, lambda: a / ov) +assert_raises(OverflowError, lambda: a // ov) +assert_raises(OverflowError, lambda: a % ov) +assert_raises(OverflowError, lambda: a ** ov) +assert_raises(OverflowError, lambda: ov + a) +assert_raises(OverflowError, lambda: ov - a) +assert_raises(OverflowError, lambda: ov * a) +assert_raises(OverflowError, lambda: ov / a) +assert_raises(OverflowError, lambda: ov // a) +assert_raises(OverflowError, lambda: ov % a) +# assert_raises(OverflowError, lambda: ov ** a) assert a < 5 assert a <= 5 diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 23636cef8..426483f4c 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -51,14 +51,30 @@ fn try_float(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> }) } -fn mod_(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { +fn inner_div(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v2 != 0.0 { - Ok(vm.ctx.new_float(v1 % v2)) + Ok(v1 / v2) + } else { + Err(vm.new_zero_division_error("float division by zero".to_string())) + } +} + +fn inner_mod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok(v1 % v2) } else { Err(vm.new_zero_division_error("float mod by zero".to_string())) } } +fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok((v1 / v2).floor()) + } else { + Err(vm.new_zero_division_error("float floordiv by zero".to_string())) + } +} + #[pyimpl] impl PyFloat { #[pymethod(name = "__eq__")] @@ -139,20 +155,15 @@ impl PyFloat { } #[pymethod(name = "__add__")] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - vm.ctx.new_float(v1 + get_value(&other)) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx - .new_float(v1 + objint::get_value(&other).to_f64().unwrap()) - } else { - vm.ctx.not_implemented() - } + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value + other).into_pyobject(vm), + ) } #[pymethod(name = "__radd__")] - fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.add(other, vm) } @@ -163,45 +174,39 @@ impl PyFloat { #[pymethod(name = "__divmod__")] fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&other, &vm.ctx.float_type()) - || objtype::isinstance(&other, &vm.ctx.int_type()) - { - let r1 = self.floordiv(other.clone(), vm)?; - let r2 = self.mod_(other, vm)?; - Ok(vm.ctx.new_tuple(vec![r1, r2])) - } else { - Ok(vm.ctx.not_implemented()) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| { + let r1 = inner_floordiv(self.value, other, vm)?; + let r2 = inner_mod(self.value, other, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(r1), vm.ctx.new_float(r2)])) + }, + ) } #[pymethod(name = "__floordiv__")] fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_float_value(&other, vm)? - } else { - return Ok(vm.ctx.not_implemented()); - }; + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_floordiv(self.value, other, vm)?.into_pyobject(vm), + ) + } - if v2 != 0.0 { - Ok(vm.ctx.new_float((v1 / v2).floor())) - } else { - Err(vm.new_zero_division_error("float floordiv by zero".to_string())) - } + #[pymethod(name = "__rfloordiv__")] + fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_floordiv(other, self.value, vm)?.into_pyobject(vm), + ) } fn new_float(cls: PyClassRef, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { let value = if objtype::isinstance(&arg, &vm.ctx.float_type()) { get_value(&arg) } else if objtype::isinstance(&arg, &vm.ctx.int_type()) { - match objint::get_float_value(&arg, vm) { - Ok(f) => f, - Err(e) => { - return Err(e); - } - } + objint::get_float_value(&arg, vm)? } else if objtype::isinstance(&arg, &vm.ctx.str_type()) { match lexical::try_parse(objstr::get_value(&arg)) { Ok(f) => f, @@ -232,28 +237,18 @@ impl PyFloat { #[pymethod(name = "__mod__")] fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_float_value(&other, vm)? - } else { - return Ok(vm.ctx.not_implemented()); - }; - - mod_(v1, v2, vm) + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_mod(self.value, other, vm)?.into_pyobject(vm), + ) } #[pymethod(name = "__rmod__")] fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v2 = self.value; - let v1 = if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_float_value(&other, vm)? - } else { - return Ok(vm.ctx.not_implemented()); - }; - - mod_(v1, v2, vm) + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_mod(other, self.value, vm)?.into_pyobject(vm), + ) } #[pymethod(name = "__neg__")] @@ -271,30 +266,18 @@ impl PyFloat { #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - Ok(vm.ctx.new_float(v1 - get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - Ok(vm - .ctx - .new_float(v1 - objint::get_value(&other).to_f64().unwrap())) - } else { - Ok(vm.ctx.not_implemented()) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value - other).into_pyobject(vm), + ) } #[pymethod(name = "__rsub__")] fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - Ok(vm.ctx.new_float(get_value(&other) - v1)) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - Ok(vm - .ctx - .new_float(objint::get_value(&other).to_f64().unwrap() - v1)) - } else { - Ok(vm.ctx.not_implemented()) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other - self.value).into_pyobject(vm), + ) } #[pymethod(name = "__repr__")] @@ -304,52 +287,26 @@ impl PyFloat { #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_float_value(&other, vm)? - } else { - return Ok(vm.ctx.not_implemented()); - }; - - if v2 != 0.0 { - Ok(vm.ctx.new_float(v1 / v2)) - } else { - Err(vm.new_zero_division_error("float division by zero".to_string())) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_div(self.value, other, vm)?.into_pyobject(vm), + ) } #[pymethod(name = "__rtruediv__")] fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_float_value(&other, vm)? - } else { - return Ok(vm.ctx.not_implemented()); - }; - - if v1 != 0.0 { - Ok(vm.ctx.new_float(v2 / v1)) - } else { - Err(vm.new_zero_division_error("float division by zero".to_string())) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_div(other, self.value, vm)?.into_pyobject(vm), + ) } #[pymethod(name = "__mul__")] fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type) { - Ok(vm.ctx.new_float(v1 * get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - Ok(vm - .ctx - .new_float(v1 * objint::get_value(&other).to_f64().unwrap())) - } else { - Ok(vm.ctx.not_implemented()) - } + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value * other).into_pyobject(vm), + ) } #[pymethod(name = "__rmul__")] From cfff7cbd63d00d14ca4ce93d56164a65656985b9 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Fri, 19 Apr 2019 08:17:16 +0200 Subject: [PATCH 416/658] add split and rsplit --- tests/snippets/bytes.py | 178 ++++++++++++ vm/src/obj/objbyteinner.rs | 539 +++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 24 +- 3 files changed, 740 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2733533d8..2b4af2dac 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -345,3 +345,181 @@ assert b" spacious ".lstrip() == b"spacious " assert b"www.example.com".lstrip(b"cmowz.") == b"example.com" assert b" spacious ".rstrip() == b" spacious" assert b"mississippi".rstrip(b"ipz") == b"mississ" + + +# split +assert b"1,2,3".split(b",") == [b"1", b"2", b"3"] +assert b"1,2,3".split(b",", maxsplit=1) == [b"1", b"2,3"] +assert b"1,2,,3,".split(b",") == [b"1", b"2", b"", b"3", b""] +assert b"1 2 3".split() == [b"1", b"2", b"3"] +assert b"1 2 3".split(maxsplit=1) == [b"1", b"2 3"] +assert b" 1 2 3 ".split() == [b"1", b"2", b"3"] +assert b"k\ruh\nfz e f".split() == [b"k", b"uh", b"fz", b"e", b"f"] + +SPLIT_FIXTURES = [ + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + -1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + -1, + ], + # maxsplit + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], []], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], [3]], + 1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 4, 5], [1, 2, 3], [1, 2, 3]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]], + 0, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122], [101, 102]], + 1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102, 9, 9]], + [[13, 13, 13, 117, 104], [102, 122], [101, 102]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [[49, 44, 50, 44, 51], [44], [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 44, 50, 44, 51], [44], [[49], [50, 44, 51]], [[49, 44, 50], [51]], 1], + [ + [49, 44, 50, 44, 44, 51, 44], + [44], + [[49], [50], [], [51], []], + [[49], [50], [], [51], []], + -1, + ], + [[49, 32, 50, 32, 51], None, [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 32, 50, 32, 51], None, [[49], [50, 32, 51]], [[49, 32, 50], [51]], 1], + [ + [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32], + None, + [[49], [50], [51]], + [[49], [50], [51]], + -1, + ], +] + + +# for i in SPLIT_FIXTURES: # for not yet implemented : TypeError: Unsupported method: __next__ +n_sp = 0 +while n_sp < len(SPLIT_FIXTURES): + i = SPLIT_FIXTURES[n_sp] + sep = None if i[1] == None else bytes(i[1]) + try: + assert bytes(i[0]).split(sep=sep, maxsplit=i[4]) == [bytes(j) for j in i[2]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected : ", [list(x) for x in bytes(i[0]).split(sep=sep, maxsplit=i[4])] + ) + break + + try: + assert bytes(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytes(j) for j in i[3]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected Rev : ", + [list(x) for x in bytes(i[0]).rsplit(sep=sep, maxsplit=i[4])], + ) + break + + n_sp += 1 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b75f8be69..7b4037b57 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -253,6 +253,41 @@ impl ByteInnerTranslateOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerSplitOptions { + #[pyarg(positional_or_keyword, optional = true)] + sep: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + maxsplit: OptionalArg, +} + +impl ByteInnerSplitOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec, i32)> { + let sep = if let OptionalArg::Present(value) = self.sep { + match try_as_bytes_like(&value) { + Some(value) => value, + None => match_class!(value, + _n @ PyNone=> vec![], + obj => {return Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj + )));} + ), + } + } else { + vec![] + }; + + let maxsplit = if let OptionalArg::Present(value) = self.maxsplit { + value.as_bigint().to_i32().unwrap() + } else { + -1 + }; + + Ok((sep.clone(), maxsplit)) + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -757,6 +792,30 @@ impl PyByteInner { } Ok(self.elements[start..end].to_vec()) } + + pub fn split( + &self, + options: ByteInnerSplitOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let (sep, maxsplit) = options.get_value(vm)?; + + let splitted = split_slice(&self.elements, &sep, maxsplit); + + Ok(splitted) + } + + pub fn rsplit( + &self, + options: ByteInnerSplitOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let (sep, maxsplit) = options.get_value(vm)?; + + let splitted = split_slice_reverse(&self.elements, &sep, maxsplit); + + Ok(splitted) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { @@ -783,3 +842,483 @@ pub enum ByteInnerPosition { Right, All, } + +fn split_slice<'a>(slice: &'a [u8], sep: &[u8], maxsplit: i32) -> Vec<&'a [u8]> { + let mut splitted: Vec<&[u8]> = vec![]; + let mut prev_index = 0; + let mut index = 0; + let mut count = 0; + let mut in_string = false; + + // No sep given, will split for any \t \n \r and space = [9, 10, 13, 32] + if sep.is_empty() { + // split wihtout sep always trim left spaces for any maxsplit + // so we have to ignore left spaces. + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + index += 1 + } else { + prev_index = index; + break; + } + } + + // most simple case + if maxsplit == 0 { + splitted.push(&slice[index..slice.len()]); + return splitted; + } + + // main loop. in_string means previous char is ascii char(true) or space(false) + // loop from left to right + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + if in_string { + splitted.push(&slice[prev_index..index]); + in_string = false; + count += 1; + if count == maxsplit { + // while index < slice.len() + splitted.push(&slice[index + 1..slice.len()]); + break; + } + } + } else if !in_string { + prev_index = index; + in_string = true; + } + + index += 1; + + // handle last item in slice + if index == slice.len() { + if in_string { + if [9, 10, 13, 32].contains(&slice[index - 1]) { + splitted.push(&slice[prev_index..index - 1]); + } else { + splitted.push(&slice[prev_index..index]); + } + } + break; + } + } + } else { + // sep is given, we match exact slice + while index != slice.len() { + if index + sep.len() >= slice.len() { + if &slice[index..slice.len()] == sep { + splitted.push(&slice[prev_index..index]); + splitted.push(&[]); + break; + } + splitted.push(&slice[prev_index..slice.len()]); + break; + } + + if &slice[index..index + sep.len()] == sep { + splitted.push(&slice[prev_index..index]); + index += sep.len(); + prev_index = index; + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[prev_index..slice.len()]); + break; + } + continue; + } + + index += 1; + } + } + splitted +} + +fn split_slice_reverse<'a>(slice: &'a [u8], sep: &[u8], maxsplit: i32) -> Vec<&'a [u8]> { + let mut splitted: Vec<&[u8]> = vec![]; + let mut prev_index = slice.len(); + let mut index = slice.len(); + let mut count = 0; + + // No sep given, will split for any \t \n \r and space = [9, 10, 13, 32] + if sep.is_empty() { + //adjust index + index -= 1; + + // rsplit without sep always trim right spaces for any maxsplit + // so we have to ignore right spaces. + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + index -= 1 + } else { + break; + } + } + prev_index = index + 1; + + // most simple case + if maxsplit == 0 { + splitted.push(&slice[0..=index]); + return splitted; + } + + // main loop. in_string means previous char is ascii char(true) or space(false) + // loop from right to left and reverse result the end + let mut in_string = true; + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + if in_string { + splitted.push(&slice[index + 1..prev_index]); + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[0..index]); + break; + } + in_string = false; + index -= 1; + continue; + } + } else if !in_string { + in_string = true; + if index == 0 { + splitted.push(&slice[0..1]); + break; + } + prev_index = index + 1; + } + if index == 0 { + break; + } + index -= 1; + } + } else { + // sep is give, we match exact slice going backwards + while index != 0 { + if index <= sep.len() { + if &slice[0..index] == sep { + splitted.push(&slice[index..prev_index]); + splitted.push(&[]); + break; + } + splitted.push(&slice[0..prev_index]); + break; + } + if &slice[(index - sep.len())..index] == sep { + splitted.push(&slice[index..prev_index]); + index -= sep.len(); + prev_index = index; + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[0..prev_index]); + break; + } + continue; + } + + index -= 1; + } + } + splitted.reverse(); + splitted +} + +#[cfg(test)] + +// needed for dev. Same as python tests in bytes.py. should it be kept ? + +mod tests { + use super::*; + + #[test] + fn no_end() { + assert_eq!( + split_slice(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[4, 5], -1), + vec![[1, 2, 3], [1, 2, 3], [1, 2, 3]] + ); + assert_eq!( + split_slice_reverse(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[4, 5], -1), + vec![[1, 2, 3], [1, 2, 3], [1, 2, 3]] + ) + } + + #[test] + fn needle_end() { + let v: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3], &[1, 2, 3], &[]]; + assert_eq!( + split_slice(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], &[4, 5], -1), + v + ); + assert_eq!( + split_slice_reverse(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], &[4, 5], -1), + v + ) + } + + #[test] + fn needle_end_minus_one() { + let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3], &[1, 2, 3], &[3]]; + + assert_eq!(split_slice(&v, &n, -1), res); + assert_eq!(split_slice_reverse(&v, &n, -1), res) + } + + #[test] + fn needle_start() { + let v = [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[], &[2, 3], &[1, 2, 3], &[1, 2, 3]]; + + assert_eq!(split_slice(&v, &n, -1), res); + assert_eq!(split_slice_reverse(&v, &n, -1), res) + } + + #[test] + fn needle_start_plus_one() { + let v = [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1], &[2, 3], &[1, 2, 3], &[1, 2, 3]]; + + assert_eq!(split_slice(&v, &n, -1), res); + assert_eq!(split_slice_reverse(&v, &n, -1), res) + } + #[test] + fn needles_next_to() { + let v = [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[], &[], &[1, 2, 3], &[1, 2, 3]]; + + assert_eq!(split_slice(&v, &n, -1), res); + assert_eq!(split_slice_reverse(&v, &n, -1), res) + } + #[test] + fn no_end_max() { + let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3]]; + let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn needle_end_max() { + let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]]; + let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + #[test] + fn needle_end_minus_one_max() { + let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]]; + let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[3]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn needle_start_max() { + let v = [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[], &[2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; + let res_rev: Vec<&[u8]> = vec![&[4, 5, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + #[test] + fn needle_start_minus_one_max() { + let v = [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1], &[2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; + let res_rev: Vec<&[u8]> = vec![&[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn needle_next_to() { + let v = [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; + let n = [4, 5]; + let res: Vec<&[u8]> = vec![&[1, 2, 3], &[], &[4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; + let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 4, 5], &[1, 2, 3], &[1, 2, 3]]; + let max = 2; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle() { + let v = [107, 13, 117, 104, 10, 102, 122, 32, 101, 9, 102]; + let n = []; + let res: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; + let res_rev: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle_end_nostring() { + let v = [107, 13, 117, 104, 10, 102, 122, 32, 101, 9, 102, 9]; + let n = []; + let res: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; + let res_rev: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle_sides_no_max() { + let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; + let n = []; + let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; + let res_rev: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle_sides_max_zero() { + let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; + let n = []; + let res: Vec<&[u8]> = vec![&[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]]; + let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]]; + let max = 0; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle_sides_max_one() { + let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; + let n = []; + let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122, 32, 101, 102, 9, 9]]; + let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104, 10, 102, 122], &[101, 102]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn no_needle_sides_max_two() { + let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; + let n = []; + let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102, 9, 9]]; + let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104], &[102, 122], &[101, 102]]; + let max = 2; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + #[test] + fn no_needle_no_max_big_spaces() { + let v = [ + 13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9, + ]; + let n = []; + let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; + let res_rev: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_needle() { + let v = [49, 44, 50, 44, 51]; + let n = [44]; + let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_needle_max_one() { + let v = [49, 44, 50, 44, 51]; + let n = [44]; + let res: Vec<&[u8]> = vec![&[49], &[50, 44, 51]]; + let res_rev: Vec<&[u8]> = vec![&[49, 44, 50], &[51]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_nearneedle() { + let v = [49, 44, 50, 44, 44, 51, 44]; + let n = [44]; + let res: Vec<&[u8]> = vec![&[49], &[50], &[], &[51], &[]]; + let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[], &[51], &[]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_space_no_sep() { + let v = [49, 32, 50, 32, 51]; + let n = []; + let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_space_no_sep_max_one() { + let v = [49, 32, 50, 32, 51]; + let n = []; + let res: Vec<&[u8]> = vec![&[49], &[50, 32, 51]]; + let res_rev: Vec<&[u8]> = vec![&[49, 32, 50], &[51]]; + let max = 1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + + #[test] + fn cpython_bigspace() { + let v = [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32]; + let n = []; + let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; + let max = -1; + + assert_eq!(split_slice(&v, &n, max), res); + assert_eq!(split_slice_reverse(&v, &n, max), res_rev) + } + +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index c16fd123f..c3b80a5c9 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -13,7 +13,7 @@ use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, Py use super::objbyteinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, - ByteInnerTranslateOptions, PyByteInner, + ByteInnerSplitOptions, ByteInnerTranslateOptions, PyByteInner, }; use super::objiter; @@ -332,6 +332,28 @@ impl PyBytesRef { .ctx .new_bytes(self.inner.strip(chars, ByteInnerPosition::Right, vm)?)) } + + #[pymethod(name = "split")] + fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .split(options, vm)? + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "rsplit")] + fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .rsplit(options, vm)? + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } } #[derive(Debug)] From 0af8ad05eb36694cb11224cd72d9b8fd1f0682f3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 20 Apr 2019 17:03:12 +0200 Subject: [PATCH 417/658] add expandstab --- tests/snippets/bytes.py | 12 +++++++++ vm/src/obj/objbyteinner.rs | 52 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 9 +++++-- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2b4af2dac..a3ecc6a94 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -523,3 +523,15 @@ while n_sp < len(SPLIT_FIXTURES): break n_sp += 1 + + +# expandtabs +a = b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14" +assert ( + a.expandtabs() == b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert a.expandtabs(5) == b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +assert b"01\t012\t0123\t01234".expandtabs() == b"01 012 0123 01234" +assert b"01\t012\t0123\t01234".expandtabs(4) == b"01 012 0123 01234" +assert b"123\t123".expandtabs(-5) == b"123123" +assert b"123\t123".expandtabs(0) == b"123123" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 7b4037b57..228c97248 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -288,6 +288,26 @@ impl ByteInnerSplitOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerExpandtabsOptions { + #[pyarg(positional_or_keyword, optional = true)] + tabsize: OptionalArg, +} + +impl ByteInnerExpandtabsOptions { + pub fn get_value(self) -> usize { + if let OptionalArg::Present(value) = self.tabsize { + if let Some(v) = objint::get_value(&value).to_usize() { + v + } else { + 0 + } + } else { + 8 + } + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -816,6 +836,38 @@ impl PyByteInner { Ok(splitted) } + + pub fn expandtabs(&self, options: ByteInnerExpandtabsOptions) -> Vec { + let tabsize = options.get_value(); + let mut counter: usize = 0; + let mut res = vec![]; + + if tabsize == 0 { + return self + .elements + .iter() + .cloned() + .filter(|x| *x != b'\t') + .collect::>(); + } + + for i in &self.elements { + if *i == b'\t' { + let len = tabsize - counter % tabsize; + res.extend_from_slice(&vec![b' '; len]); + counter += len; + } else { + res.push(*i); + if *i == b'\r' || *i == b'\n' { + counter = 0; + } else { + counter += 1; + } + } + } + + res + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index c3b80a5c9..28218e597 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -12,8 +12,8 @@ use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, - ByteInnerSplitOptions, ByteInnerTranslateOptions, PyByteInner, + ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerPosition, ByteInnerSplitOptions, ByteInnerTranslateOptions, PyByteInner, }; use super::objiter; @@ -354,6 +354,11 @@ impl PyBytesRef { .collect::>(); Ok(vm.ctx.new_list(as_bytes)) } + + #[pymethod(name = "expandtabs")] + fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.expandtabs(options))) + } } #[derive(Debug)] From 434985d6d07689ad84595774f11208cdc45b6387 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 20 Apr 2019 21:04:13 +0200 Subject: [PATCH 418/658] add partition/reparition refactor split/rsplit --- tests/snippets/bytes.py | 17 +++++++++++++++++ vm/src/obj/objbyteinner.rs | 35 ++++++++++++++++++++++++----------- vm/src/obj/objbytes.rs | 24 ++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a3ecc6a94..76efed6c2 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -535,3 +535,20 @@ assert b"01\t012\t0123\t01234".expandtabs() == b"01 012 0123 01234" assert b"01\t012\t0123\t01234".expandtabs(4) == b"01 012 0123 01234" assert b"123\t123".expandtabs(-5) == b"123123" assert b"123\t123".expandtabs(0) == b"123123" + + +# partition +assert b"123456789".partition(b"45") == (b"123", b"45", b"6789") +assert b"14523456789".partition(b"45") == (b"1", b"45", b"23456789") +a = b"14523456789".partition(bytearray(b"45")) +assert isinstance(a[1], bytearray) +a = b"14523456789".partition(memoryview(b"45")) +assert isinstance(a[1], memoryview) + +# partition +assert b"123456789".rpartition(b"45") == (b"123", b"45", b"6789") +assert b"14523456789".rpartition(b"45") == (b"14523", b"45", b"6789") +a = b"14523456789".rpartition(bytearray(b"45")) +assert isinstance(a[1], bytearray) +a = b"14523456789".rpartition(memoryview(b"45")) +assert isinstance(a[1], memoryview) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 228c97248..3bc7cba69 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -816,25 +816,38 @@ impl PyByteInner { pub fn split( &self, options: ByteInnerSplitOptions, + reverse: bool, vm: &VirtualMachine, ) -> PyResult> { let (sep, maxsplit) = options.get_value(vm)?; - let splitted = split_slice(&self.elements, &sep, maxsplit); - - Ok(splitted) + if reverse { + Ok(split_slice_reverse(&self.elements, &sep, maxsplit)) + } else { + Ok(split_slice(&self.elements, &sep, maxsplit)) + } } - pub fn rsplit( + pub fn partition( &self, - options: ByteInnerSplitOptions, + sep: &PyObjectRef, + reverse: bool, vm: &VirtualMachine, - ) -> PyResult> { - let (sep, maxsplit) = options.get_value(vm)?; - - let splitted = split_slice_reverse(&self.elements, &sep, maxsplit); - - Ok(splitted) + ) -> PyResult<(Vec, Vec)> { + let sep = match try_as_bytes_like(&sep) { + Some(value) => value, + None => { + return Err( + vm.new_type_error(format!("a bytes-like object is required, not {}", sep)) + ); + } + }; + let splitted = if reverse { + split_slice_reverse(&self.elements, &sep, 1) + } else { + split_slice(&self.elements, &sep, 1) + }; + Ok((splitted[0].to_vec(), splitted[1].to_vec())) } pub fn expandtabs(&self, options: ByteInnerExpandtabsOptions) -> Vec { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 28218e597..d84d971f4 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -18,6 +18,7 @@ use super::objbyteinner::{ use super::objiter; use super::objtype::PyClassRef; + /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -337,7 +338,7 @@ impl PyBytesRef { fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { let as_bytes = self .inner - .split(options, vm)? + .split(options, false, vm)? .iter() .map(|x| vm.ctx.new_bytes(x.to_vec())) .collect::>(); @@ -348,13 +349,32 @@ impl PyBytesRef { fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { let as_bytes = self .inner - .rsplit(options, vm)? + .split(options, true, vm)? .iter() .map(|x| vm.ctx.new_bytes(x.to_vec())) .collect::>(); Ok(vm.ctx.new_list(as_bytes)) } + #[pymethod(name = "partition")] + fn partition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray + // even it's bytes or memoryview + let (left, right) = self.inner.partition(&sep, false, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) + } + #[pymethod(name = "rpartition")] + fn rpartition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray + // even it's bytes or memoryview + let (left, right) = self.inner.partition(&sep, true, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) + } + #[pymethod(name = "expandtabs")] fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.expandtabs(options))) From d89ca3c3e49d80feaeb53d68a9c6364d2904174f Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 21 Apr 2019 00:38:27 +0200 Subject: [PATCH 419/658] add spitlines --- tests/snippets/bytes.py | 15 +++++++-- vm/src/obj/objbyteinner.rs | 64 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 14 ++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 76efed6c2..e7dce5f4e 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -541,7 +541,7 @@ assert b"123\t123".expandtabs(0) == b"123123" assert b"123456789".partition(b"45") == (b"123", b"45", b"6789") assert b"14523456789".partition(b"45") == (b"1", b"45", b"23456789") a = b"14523456789".partition(bytearray(b"45")) -assert isinstance(a[1], bytearray) +assert isinstance(a[1], bytearray) a = b"14523456789".partition(memoryview(b"45")) assert isinstance(a[1], memoryview) @@ -549,6 +549,17 @@ assert isinstance(a[1], memoryview) assert b"123456789".rpartition(b"45") == (b"123", b"45", b"6789") assert b"14523456789".rpartition(b"45") == (b"14523", b"45", b"6789") a = b"14523456789".rpartition(bytearray(b"45")) -assert isinstance(a[1], bytearray) +assert isinstance(a[1], bytearray) a = b"14523456789".rpartition(memoryview(b"45")) assert isinstance(a[1], memoryview) + +# splitlines +assert b"ab c\n\nde fg\rkl\r\n".splitlines() == [b"ab c", b"", b"de fg", b"kl"] +assert b"ab c\n\nde fg\rkl\r\n".splitlines(keepends=True) == [ + b"ab c\n", + b"\n", + b"de fg\r", + b"kl\r\n", +] +assert b"".splitlines() == [] +assert b"One line\n".splitlines() == [b"One line"] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 3bc7cba69..d32c99d2c 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -308,6 +308,22 @@ impl ByteInnerExpandtabsOptions { } } +#[derive(FromArgs)] +pub struct ByteInnerSplitlinesOptions { + #[pyarg(positional_or_keyword, optional = true)] + keepends: OptionalArg, +} + +impl ByteInnerSplitlinesOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(value) = self.keepends { + Ok(bool::try_from_object(vm, value)?) + } else { + Ok(false) + } + } +} + impl PyByteInner { pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); @@ -881,6 +897,54 @@ impl PyByteInner { res } + + pub fn splitlines( + &self, + options: ByteInnerSplitlinesOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let keepends = options.get_value(vm)?; + + let mut res = vec![]; + + if self.elements.is_empty() { + return Ok(vec![]); + } + + let mut prev_index = 0; + let mut index = 0; + let keep = if keepends { 1 } else { 0 }; + let slice = &self.elements; + + while index < slice.len() { + match slice[index] { + b'\n' => { + res.push(&slice[prev_index..index + keep]); + index += 1; + prev_index = index; + } + b'\r' => { + if index + 2 <= slice.len() && slice[index + 1] == b'\n' { + res.push(&slice[prev_index..index + keep + keep]); + index += 2; + } else { + res.push(&slice[prev_index..index + keep]); + index += 1; + } + prev_index = index; + } + _x => { + if index == slice.len() - 1 { + res.push(&slice[prev_index..=index]); + break; + } + index += 1 + } + } + } + + Ok(res) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index d84d971f4..31396c11c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -13,7 +13,8 @@ use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, Py use super::objbyteinner::{ ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerPosition, ByteInnerSplitOptions, ByteInnerTranslateOptions, PyByteInner, + ByteInnerPosition, ByteInnerSplitOptions, ByteInnerSplitlinesOptions, + ByteInnerTranslateOptions, PyByteInner, }; use super::objiter; @@ -379,6 +380,17 @@ impl PyBytesRef { fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.expandtabs(options))) } + + #[pymethod(name = "splitlines")] + fn splitlines(self, options: ByteInnerSplitlinesOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .splitlines(options, vm)? + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } } #[derive(Debug)] From 29f674fa1dbb7650524e2e736489b15e0e54f037 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 21 Apr 2019 01:01:51 +0200 Subject: [PATCH 420/658] fix split with empty bytes --- tests/snippets/bytes.py | 4 ++++ vm/src/obj/objbyteinner.rs | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index e7dce5f4e..944467349 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -355,6 +355,10 @@ assert b"1 2 3".split() == [b"1", b"2", b"3"] assert b"1 2 3".split(maxsplit=1) == [b"1", b"2 3"] assert b" 1 2 3 ".split() == [b"1", b"2", b"3"] assert b"k\ruh\nfz e f".split() == [b"k", b"uh", b"fz", b"e", b"f"] +assert b"Two lines\n".split(b'\n') == [b'Two lines', b''] +assert b"".split() == [] +assert b"".split(b"\n") == [b''] +assert b"\n".split(b"\n") == [b'', b''] SPLIT_FIXTURES = [ [ diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index d32c99d2c..bb0f0118a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -837,6 +837,13 @@ impl PyByteInner { ) -> PyResult> { let (sep, maxsplit) = options.get_value(vm)?; + if self.elements.is_empty() { + if !sep.is_empty() { + return Ok(vec![&[]]); + } + return Ok(vec![]); + } + if reverse { Ok(split_slice_reverse(&self.elements, &sep, maxsplit)) } else { From b60d7413b9a2b4476329548e499715b007224fb4 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Sun, 21 Apr 2019 11:26:12 +0200 Subject: [PATCH 421/658] add zfill --- tests/snippets/bytes.py | 13 ++++++++++--- vm/src/obj/objbyteinner.rs | 20 ++++++++++++++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 944467349..17f9520fa 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -355,10 +355,10 @@ assert b"1 2 3".split() == [b"1", b"2", b"3"] assert b"1 2 3".split(maxsplit=1) == [b"1", b"2 3"] assert b" 1 2 3 ".split() == [b"1", b"2", b"3"] assert b"k\ruh\nfz e f".split() == [b"k", b"uh", b"fz", b"e", b"f"] -assert b"Two lines\n".split(b'\n') == [b'Two lines', b''] +assert b"Two lines\n".split(b"\n") == [b"Two lines", b""] assert b"".split() == [] -assert b"".split(b"\n") == [b''] -assert b"\n".split(b"\n") == [b'', b''] +assert b"".split(b"\n") == [b""] +assert b"\n".split(b"\n") == [b"", b""] SPLIT_FIXTURES = [ [ @@ -567,3 +567,10 @@ assert b"ab c\n\nde fg\rkl\r\n".splitlines(keepends=True) == [ ] assert b"".splitlines() == [] assert b"One line\n".splitlines() == [b"One line"] + +# zfill + +assert b"42".zfill(5) == b"00042" +assert b"-42".zfill(5) == b"-0042" +assert b"42".zfill(1) == b"42" +assert b"42".zfill(-1) == b"42" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bb0f0118a..45333f3b6 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -952,6 +952,26 @@ impl PyByteInner { Ok(res) } + + pub fn zfill(&self, width: PyIntRef) -> Vec { + if let Some(value) = width.as_bigint().to_usize() { + if value < self.elements.len() { + return self.elements.to_vec(); + } + let mut res = vec![]; + if self.elements.starts_with(&[b'-']) { + res.push(b'-'); + res.extend_from_slice(&vec![b'0'; value - self.elements.len()]); + res.extend_from_slice(&self.elements[1..]); + } else { + res.extend_from_slice(&vec![b'0'; value - self.elements.len()]); + res.extend_from_slice(&self.elements[0..]); + } + res + } else { + self.elements.to_vec() + } + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 31396c11c..2f9a7d86e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -391,6 +391,11 @@ impl PyBytesRef { .collect::>(); Ok(vm.ctx.new_list(as_bytes)) } + + #[pymethod(name = "zfill")] + fn zfill(self, width: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.zfill(width))) + } } #[derive(Debug)] From cf7032347545afd1874b6ff6faa0726d71df881b Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Sun, 21 Apr 2019 12:40:21 +0200 Subject: [PATCH 422/658] add replace --- tests/snippets/bytes.py | 8 ++++++ vm/src/obj/objbyteinner.rs | 58 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 11 ++++++++ 3 files changed, 77 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 17f9520fa..9c1eb63f4 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -574,3 +574,11 @@ assert b"42".zfill(5) == b"00042" assert b"-42".zfill(5) == b"-0042" assert b"42".zfill(1) == b"42" assert b"42".zfill(-1) == b"42" + +# replace +assert b"123456789123".replace(b"23", b"XX") == b'1XX4567891XX' +assert b"123456789123".replace(b"23", b"XX", 1) == b'1XX456789123' +assert b"123456789123".replace(b"23", b"XX", 0) == b"123456789123" +assert b"123456789123".replace(b"23", b"XX", -1) == b'1XX4567891XX' +assert b"123456789123".replace(b"23", b"") == b"14567891" + diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 45333f3b6..bb6b0380d 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -972,6 +972,64 @@ impl PyByteInner { self.elements.to_vec() } } + + pub fn replace( + &self, + old: PyObjectRef, + new: PyObjectRef, + count: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + let old = match try_as_bytes_like(&old) { + Some(value) => value, + None => { + return Err( + vm.new_type_error(format!("a bytes-like object is required, not {}", old)) + ); + } + }; + + let new = match try_as_bytes_like(&new) { + Some(value) => value, + None => { + return Err( + vm.new_type_error(format!("a bytes-like object is required, not {}", new)) + ); + } + }; + + let count = if let OptionalArg::Present(int) = count { + if let Some(value) = int.as_bigint().to_u32() { + value + } else { + self.elements.len() as u32 + } + } else { + self.elements.len() as u32 + }; + + let mut res = vec![]; + let mut index = 0; + let mut done = 0; + + let slice = &self.elements; + while index <= slice.len() - old.len() { + if done == count { + res.extend_from_slice(&slice[index..]); + break; + } + if &slice[index..index + old.len()] == old.as_slice() { + res.extend_from_slice(&new); + index += old.len(); + done += 1; + } else { + res.push(slice[index]); + index += 1 + } + } + + Ok(res) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2f9a7d86e..77c8f7ed8 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -396,6 +396,17 @@ impl PyBytesRef { fn zfill(self, width: PyIntRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.zfill(width))) } + + #[pymethod(name = "replace")] + fn replace( + self, + old: PyObjectRef, + new: PyObjectRef, + count: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.replace(old, new, count, vm)?)) + } } #[derive(Debug)] From fd614b7a757a11679d0431caeca3cafbb00ab70f Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 30 Apr 2019 17:12:36 +0200 Subject: [PATCH 423/658] refactor split options --- vm/src/obj/objbyteinner.rs | 29 +++++++---------------------- vm/src/obj/objbytes.rs | 4 ++-- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bb6b0380d..31a04e0e1 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -256,26 +256,16 @@ impl ByteInnerTranslateOptions { #[derive(FromArgs)] pub struct ByteInnerSplitOptions { #[pyarg(positional_or_keyword, optional = true)] - sep: OptionalArg, + sep: OptionalArg>, #[pyarg(positional_or_keyword, optional = true)] maxsplit: OptionalArg, } impl ByteInnerSplitOptions { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec, i32)> { - let sep = if let OptionalArg::Present(value) = self.sep { - match try_as_bytes_like(&value) { - Some(value) => value, - None => match_class!(value, - _n @ PyNone=> vec![], - obj => {return Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - obj - )));} - ), - } - } else { - vec![] + pub fn get_value(self) -> PyResult<(Vec, i32)> { + let sep = match self.sep.into_option() { + Some(Some(bytes)) => bytes.elements, + _ => vec![], }; let maxsplit = if let OptionalArg::Present(value) = self.maxsplit { @@ -829,13 +819,8 @@ impl PyByteInner { Ok(self.elements[start..end].to_vec()) } - pub fn split( - &self, - options: ByteInnerSplitOptions, - reverse: bool, - vm: &VirtualMachine, - ) -> PyResult> { - let (sep, maxsplit) = options.get_value(vm)?; + pub fn split(&self, options: ByteInnerSplitOptions, reverse: bool) -> PyResult> { + let (sep, maxsplit) = options.get_value()?; if self.elements.is_empty() { if !sep.is_empty() { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 77c8f7ed8..0eb228a41 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -339,7 +339,7 @@ impl PyBytesRef { fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { let as_bytes = self .inner - .split(options, false, vm)? + .split(options, false)? .iter() .map(|x| vm.ctx.new_bytes(x.to_vec())) .collect::>(); @@ -350,7 +350,7 @@ impl PyBytesRef { fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { let as_bytes = self .inner - .split(options, true, vm)? + .split(options, true)? .iter() .map(|x| vm.ctx.new_bytes(x.to_vec())) .collect::>(); From 09710b52226a9f8cfc5ee3b70cc88fb2ada06723 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 03:15:28 +0900 Subject: [PATCH 424/658] Add float.__rdivmod__ and div/mod tests --- tests/snippets/floats.py | 7 +++++++ vm/src/obj/objfloat.rs | 24 ++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 580c5920f..45f25c100 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -106,6 +106,8 @@ assert 2.0.__sub__(1.0) == 1.0 assert 2.0.__rmul__(1.0) == 2.0 assert 1.0.__truediv__(2.0) == 0.5 assert 1.0.__rtruediv__(2.0) == 2.0 +assert 2.5.__divmod__(2.0) == (1.0, 0.5) +assert 2.0.__rdivmod__(2.5) == (1.0, 0.5) assert 1.0.__add__(1) == 2.0 assert 1.0.__radd__(1) == 2.0 @@ -120,6 +122,11 @@ assert 2.0.__rmod__(2) == 0.0 assert_raises(ZeroDivisionError, lambda: 2.0 / 0) assert_raises(ZeroDivisionError, lambda: 2.0 // 0) assert_raises(ZeroDivisionError, lambda: 2.0 % 0) +assert_raises(ZeroDivisionError, lambda: divmod(2.0, 0)) +assert_raises(ZeroDivisionError, lambda: 2 / 0.0) +assert_raises(ZeroDivisionError, lambda: 2 // 0.0) +assert_raises(ZeroDivisionError, lambda: 2 % 0.0) +# assert_raises(ZeroDivisionError, lambda: divmod(2, 0.0)) assert 1.2.__int__() == 1 assert 1.2.__float__() == 1.2 diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 426483f4c..2845e5ef4 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -75,6 +75,14 @@ fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { } } +fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + if v2 != 0.0 { + Ok(((v1 / v2).floor(), v1 % v2)) + } else { + Err(vm.new_zero_division_error("float divmod()".to_string())) + } +} + #[pyimpl] impl PyFloat { #[pymethod(name = "__eq__")] @@ -177,8 +185,20 @@ impl PyFloat { try_float(&other, vm)?.map_or_else( || Ok(vm.ctx.not_implemented()), |other| { - let r1 = inner_floordiv(self.value, other, vm)?; - let r2 = inner_mod(self.value, other, vm)?; + let (r1, r2) = inner_divmod(self.value, other, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(r1), vm.ctx.new_float(r2)])) + }, + ) + } + + #[pymethod(name = "__rdivmod__")] + fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| { + let (r1, r2) = inner_divmod(other, self.value, vm)?; Ok(vm .ctx .new_tuple(vec![vm.ctx.new_float(r1), vm.ctx.new_float(r2)])) From 1c814ff2b5b9a53ebef4d09f7f0d96b63e544d60 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 04:02:34 +0900 Subject: [PATCH 425/658] Add float.__rpow__ --- tests/snippets/floats.py | 1 + vm/src/obj/objfloat.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 45f25c100..d409463ee 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -137,6 +137,7 @@ assert float(1.2) == 1.2 assert 1.2 ** 2 == 1.44 assert_raises(OverflowError, lambda: 1.2 ** (10 ** 1000)) +assert 3 ** 2.0 == 9.0 assert (1.7).real == 1.7 assert (1.3).is_integer() == False diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 2845e5ef4..baf043d6c 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -284,6 +284,14 @@ impl PyFloat { ) } + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| other.powf(self.value).into_pyobject(vm), + ) + } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { try_float(&other, vm)?.map_or_else( From 0d505adbb5135676d03357b40d88861fd06bafa8 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Tue, 30 Apr 2019 22:01:45 +0200 Subject: [PATCH 426/658] use PyBytinner as arg --- vm/src/obj/objbyteinner.rs | 101 ++++++++++++------------------------- vm/src/obj/objbytes.rs | 25 +++++---- 2 files changed, 47 insertions(+), 79 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 31a04e0e1..d6c47c2f0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -281,19 +281,14 @@ impl ByteInnerSplitOptions { #[derive(FromArgs)] pub struct ByteInnerExpandtabsOptions { #[pyarg(positional_or_keyword, optional = true)] - tabsize: OptionalArg, + tabsize: OptionalArg, } impl ByteInnerExpandtabsOptions { pub fn get_value(self) -> usize { - if let OptionalArg::Present(value) = self.tabsize { - if let Some(v) = objint::get_value(&value).to_usize() { - v - } else { - 0 - } - } else { - 8 + match self.tabsize.into_option() { + Some(int) => int.as_bigint().to_usize().unwrap_or(0), + None => 8, } } } @@ -301,16 +296,20 @@ impl ByteInnerExpandtabsOptions { #[derive(FromArgs)] pub struct ByteInnerSplitlinesOptions { #[pyarg(positional_or_keyword, optional = true)] - keepends: OptionalArg, + keepends: OptionalArg, } impl ByteInnerSplitlinesOptions { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult { - if let OptionalArg::Present(value) = self.keepends { - Ok(bool::try_from_object(vm, value)?) - } else { - Ok(false) + pub fn get_value(self) -> bool { + match self.keepends.into_option() { + Some(x) => x, + None => false, } + // if let OptionalArg::Present(value) = self.keepends { + // Ok(bool::try_from_object(vm, value)?) + // } else { + // Ok(false) + // } } } @@ -836,24 +835,11 @@ impl PyByteInner { } } - pub fn partition( - &self, - sep: &PyObjectRef, - reverse: bool, - vm: &VirtualMachine, - ) -> PyResult<(Vec, Vec)> { - let sep = match try_as_bytes_like(&sep) { - Some(value) => value, - None => { - return Err( - vm.new_type_error(format!("a bytes-like object is required, not {}", sep)) - ); - } - }; + pub fn partition(&self, sep: &PyByteInner, reverse: bool) -> PyResult<(Vec, Vec)> { let splitted = if reverse { - split_slice_reverse(&self.elements, &sep, 1) + split_slice_reverse(&self.elements, &sep.elements, 1) } else { - split_slice(&self.elements, &sep, 1) + split_slice(&self.elements, &sep.elements, 1) }; Ok((splitted[0].to_vec(), splitted[1].to_vec())) } @@ -890,17 +876,13 @@ impl PyByteInner { res } - pub fn splitlines( - &self, - options: ByteInnerSplitlinesOptions, - vm: &VirtualMachine, - ) -> PyResult> { - let keepends = options.get_value(vm)?; + pub fn splitlines(&self, options: ByteInnerSplitlinesOptions) -> Vec<&[u8]> { + let keepends = options.get_value(); let mut res = vec![]; if self.elements.is_empty() { - return Ok(vec![]); + return vec![]; } let mut prev_index = 0; @@ -935,7 +917,7 @@ impl PyByteInner { } } - Ok(res) + res } pub fn zfill(&self, width: PyIntRef) -> Vec { @@ -960,37 +942,16 @@ impl PyByteInner { pub fn replace( &self, - old: PyObjectRef, - new: PyObjectRef, + old: PyByteInner, + new: PyByteInner, count: OptionalArg, - vm: &VirtualMachine, ) -> PyResult> { - let old = match try_as_bytes_like(&old) { - Some(value) => value, - None => { - return Err( - vm.new_type_error(format!("a bytes-like object is required, not {}", old)) - ); - } - }; - - let new = match try_as_bytes_like(&new) { - Some(value) => value, - None => { - return Err( - vm.new_type_error(format!("a bytes-like object is required, not {}", new)) - ); - } - }; - - let count = if let OptionalArg::Present(int) = count { - if let Some(value) = int.as_bigint().to_u32() { - value - } else { - self.elements.len() as u32 - } - } else { - self.elements.len() as u32 + let count = match count.into_option() { + Some(int) => int + .as_bigint() + .to_u32() + .unwrap_or(self.elements.len() as u32), + None => self.elements.len() as u32, }; let mut res = vec![]; @@ -1003,8 +964,8 @@ impl PyByteInner { res.extend_from_slice(&slice[index..]); break; } - if &slice[index..index + old.len()] == old.as_slice() { - res.extend_from_slice(&new); + if &slice[index..index + old.len()] == old.elements.as_slice() { + res.extend_from_slice(&new.elements); index += old.len(); done += 1; } else { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0eb228a41..54bccf6f0 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,5 @@ use crate::obj::objint::PyIntRef; + use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStringRef; use crate::obj::objtuple::PyTupleRef; @@ -9,7 +10,9 @@ use core::cell::Cell; use std::ops::Deref; use crate::function::OptionalArg; -use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, +}; use super::objbyteinner::{ ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, @@ -360,8 +363,10 @@ impl PyBytesRef { #[pymethod(name = "partition")] fn partition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray - // even it's bytes or memoryview - let (left, right) = self.inner.partition(&sep, false, vm)?; + // even it's bytes or memoryview so PyByteInner wiil be ok for args + let sepa = PyByteInner::try_from_object(vm, sep.clone())?; + + let (left, right) = self.inner.partition(&sepa, false)?; Ok(vm .ctx .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) @@ -369,8 +374,10 @@ impl PyBytesRef { #[pymethod(name = "rpartition")] fn rpartition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray - // even it's bytes or memoryview - let (left, right) = self.inner.partition(&sep, true, vm)?; + // even it's bytes or memoryview so PyByteInner wiil be ok for args + let sepa = PyByteInner::try_from_object(vm, sep.clone())?; + + let (left, right) = self.inner.partition(&sepa, true)?; Ok(vm .ctx .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) @@ -385,7 +392,7 @@ impl PyBytesRef { fn splitlines(self, options: ByteInnerSplitlinesOptions, vm: &VirtualMachine) -> PyResult { let as_bytes = self .inner - .splitlines(options, vm)? + .splitlines(options) .iter() .map(|x| vm.ctx.new_bytes(x.to_vec())) .collect::>(); @@ -400,12 +407,12 @@ impl PyBytesRef { #[pymethod(name = "replace")] fn replace( self, - old: PyObjectRef, - new: PyObjectRef, + old: PyByteInner, + new: PyByteInner, count: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - Ok(vm.ctx.new_bytes(self.inner.replace(old, new, count, vm)?)) + Ok(vm.ctx.new_bytes(self.inner.replace(old, new, count)?)) } } From 22a35c9b90c6796fc8de97f08932257224c1c56e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:00:25 +0900 Subject: [PATCH 427/658] extend_class for PyRangeIterator --- vm/src/obj/objrange.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index a22788119..ac129f754 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -106,12 +106,7 @@ pub fn get_value(obj: &PyObjectRef) -> PyRange { pub fn init(context: &PyContext) { PyRange::extend_class(context, &context.range_type); - - let rangeiterator_type = &context.rangeiterator_type; - extend_class!(context, rangeiterator_type, { - "__next__" => context.new_rustfunc(PyRangeIteratorRef::next), - "__iter__" => context.new_rustfunc(PyRangeIteratorRef::iter), - }); + PyRangeIterator::extend_class(context, &context.rangeiterator_type); } type PyRangeRef = PyRef; @@ -339,6 +334,7 @@ impl PyRange { } } +#[pyclass] #[derive(Debug)] pub struct PyRangeIterator { position: Cell, @@ -353,8 +349,10 @@ impl PyValue for PyRangeIterator { type PyRangeIteratorRef = PyRef; -impl PyRangeIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyRangeIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let position = BigInt::from(self.position.get()); if let Some(int) = self.range.get(&position) { self.position.set(self.position.get() + 1); @@ -364,8 +362,9 @@ impl PyRangeIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIteratorRef { + zelf } } From 14dd85a596f3b5e5e002f4899f5fd737230b363d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 03:00:54 +0900 Subject: [PATCH 428/658] Fix __builtins__.divmod to reflect __rdivmod__ --- vm/src/builtins.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 41082707e..300d11b31 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -123,11 +123,14 @@ fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(x, None), (y, None)]); - match vm.get_method(x.clone(), "__divmod__") { - Ok(attrib) => vm.invoke(attrib, vec![y.clone()]), - Err(..) => Err(vm.new_type_error("unsupported operand type(s) for divmod".to_string())), - } + arg_check!(vm, args, required = [(a, None), (b, None)]); + vm.call_or_reflection( + a.clone(), + b.clone(), + "__divmod__", + "__rdivmod__", + |vm, a, b| Err(vm.new_unsupported_operand_error(a, b, "divmod")), + ) } /// Implements `eval`. From 0d891cba2c90b56cb11ee1b20ef3458acf86686c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:59:43 +0900 Subject: [PATCH 429/658] Temporary fix to assertRaises and assert_raises prints errors --- tests/snippets/testutils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/snippets/testutils.py b/tests/snippets/testutils.py index 7dda57b17..49b7fedc1 100644 --- a/tests/snippets/testutils.py +++ b/tests/snippets/testutils.py @@ -14,9 +14,9 @@ def assert_raises(exc_type, expr, msg=None): except exc_type: pass else: - failmsg = '{!s} was not raised'.format(exc_type.__name__) + failmsg = '{} was not raised'.format(exc_type.__name__) if msg is not None: - failmsg += ': {!s}'.format(msg) + failmsg += ': {}'.format(msg) assert False, failmsg @@ -29,8 +29,8 @@ class assertRaises: def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - failmsg = '{!s} was not raised'.format(self.expected.__name__) - assert False, failmsg + failmsg = '{} was not raised'.format(self.expected.__name__) + assert False, failmsg if not issubclass(exc_type, self.expected): return False return True From 4ef65d8ac303a3574f5be78482afc0cfa942ae4d Mon Sep 17 00:00:00 2001 From: Tzu-Yin Hong Date: Wed, 1 May 2019 15:53:28 +0800 Subject: [PATCH 430/658] Add bytearray.append --- tests/snippets/bytearray.py | 6 ++++++ vm/src/obj/objbytearray.rs | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 563da2c53..4286f8171 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -65,3 +65,9 @@ except IndexError: pass else: assert False + +a = bytearray(b'appen') +assert len(a) == 5 +a.append(100) +assert len(a) == 6 +assert a.pop() == 100 diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index f6244399b..16758a438 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -79,6 +79,7 @@ pub fn init(context: &PyContext) { "istitle" =>context.new_rustfunc(PyByteArrayRef::istitle), "isupper" => context.new_rustfunc(PyByteArrayRef::isupper), "lower" => context.new_rustfunc(PyByteArrayRef::lower), + "append" => context.new_rustfunc(PyByteArrayRef::append), "pop" => context.new_rustfunc(PyByteArrayRef::pop), "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); @@ -213,6 +214,10 @@ impl PyByteArrayRef { self.value.borrow_mut().clear(); } + fn append(self, x: u8, _vm: &VirtualMachine) { + self.value.borrow_mut().push(x); + } + fn pop(self, vm: &VirtualMachine) -> PyResult { let mut bytes = self.value.borrow_mut(); bytes From bb878a34027112e19c06189662ae756ff8afe264 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:25:04 +0900 Subject: [PATCH 431/658] PyFloat to use extend_class for __new__ and __doc__ --- vm/src/obj/objfloat.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index baf043d6c..fabe99246 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -11,6 +11,7 @@ use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; use num_traits::ToPrimitive; +/// Convert a string or number to a floating point number, if possible. #[pyclass(name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyFloat { @@ -222,7 +223,8 @@ impl PyFloat { ) } - fn new_float(cls: PyClassRef, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__new__")] + fn float_new(cls: PyClassRef, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { let value = if objtype::isinstance(&arg, &vm.ctx.float_type()) { get_value(&arg) } else if objtype::isinstance(&arg, &vm.ctx.int_type()) { @@ -408,9 +410,4 @@ pub fn make_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { PyFloat::extend_class(context, &context.float_type); - let float_doc = "Convert a string or number to a floating point number, if possible."; - extend_class!(context, &context.float_type, { - "__new__" => context.new_rustfunc(PyFloat::new_float), - "__doc__" => context.new_str(float_doc.to_string()), - }); } From 4a598d03e28536fb40c6dbd87be03c9b85bdf8ff Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 03:37:01 +0900 Subject: [PATCH 432/658] Add float.imag, float.conjugate --- tests/snippets/floats.py | 2 ++ vm/src/obj/objfloat.rs | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index d409463ee..0ba3460a9 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -140,6 +140,8 @@ assert_raises(OverflowError, lambda: 1.2 ** (10 ** 1000)) assert 3 ** 2.0 == 9.0 assert (1.7).real == 1.7 +assert (1.7).imag == 0.0 +assert (1.7).conjugate() == 1.7 assert (1.3).is_integer() == False assert (1.0).is_integer() == True diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index fabe99246..ea58fed58 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -364,6 +364,16 @@ impl PyFloat { zelf } + #[pyproperty(name = "imag")] + fn imag(&self, _vm: &VirtualMachine) -> f64 { + 0.0f64 + } + + #[pymethod(name = "conjugate")] + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf + } + #[pymethod(name = "is_integer")] fn is_integer(&self, _vm: &VirtualMachine) -> bool { let v = self.value; From 1247a35be94ee11cbee909bf753bb5b5bfc913a9 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 1 May 2019 21:05:21 +0200 Subject: [PATCH 433/658] Improve handling of lexical errors. --- parser/src/fstring.rs | 7 +- parser/src/lexer.rs | 167 +++++++++++++++++++++++-------- tests/snippets/invalid_syntax.py | 4 +- 3 files changed, 133 insertions(+), 45 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index cad885531..e1e758907 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -5,7 +5,7 @@ use std::str; use lalrpop_util::ParseError as LalrpopError; use crate::ast::{ConversionFlag, StringGroup}; -use crate::lexer::{LexicalError, Location, Tok}; +use crate::lexer::{LexicalError, LexicalErrorType, Location, Tok}; use crate::parser::parse_expression; use self::FStringError::*; @@ -25,7 +25,10 @@ pub enum FStringError { impl From for LalrpopError { fn from(_err: FStringError) -> Self { lalrpop_util::ParseError::User { - error: LexicalError::StringError, + error: LexicalError { + error: LexicalErrorType::StringError, + location: Default::default(), + }, } } } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 11d30cbee..6ea29ea28 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -50,17 +50,25 @@ pub struct Lexer> { at_begin_of_line: bool, nesting: usize, // Amount of parenthesis indentation_stack: Vec, - pending: Vec>, + pending: Vec, chr0: Option, chr1: Option, location: Location, + keywords: HashMap, } #[derive(Debug)] -pub enum LexicalError { +pub struct LexicalError { + pub error: LexicalErrorType, + pub location: Location, +} + +#[derive(Debug)] +pub enum LexicalErrorType { StringError, NestingError, UnrecognizedToken { tok: char }, + OtherError(String), } #[derive(Clone, Debug, Default, PartialEq)] @@ -125,9 +133,10 @@ pub fn get_keywords() -> HashMap { keywords } -pub type Spanned = Result<(Location, Tok, Location), LexicalError>; +pub type Spanned = (Location, Tok, Location); +pub type LexResult = Result; -pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator> + 'a { +pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator + 'a { let nlh = NewlineHandler::new(source.chars()); let lch = LineContinationHandler::new(nlh); Lexer::new(lch) @@ -259,6 +268,7 @@ where chr0: None, location: Location::new(0, 0), chr1: None, + keywords: get_keywords(), }; lxr.next_char(); lxr.next_char(); @@ -269,7 +279,7 @@ where } // Lexer helper functions: - fn lex_identifier(&mut self) -> Spanned { + fn lex_identifier(&mut self) -> LexResult { let mut name = String::new(); let start_pos = self.get_pos(); @@ -310,16 +320,14 @@ where } let end_pos = self.get_pos(); - let mut keywords = get_keywords(); - - if keywords.contains_key(&name) { - Ok((start_pos, keywords.remove(&name).unwrap(), end_pos)) + if self.keywords.contains_key(&name) { + Ok((start_pos, self.keywords[&name].clone(), end_pos)) } else { Ok((start_pos, Tok::Name { name }, end_pos)) } } - fn lex_number(&mut self) -> Spanned { + fn lex_number(&mut self) -> LexResult { let start_pos = self.get_pos(); if self.chr0 == Some('0') { if self.chr1 == Some('x') || self.chr1 == Some('X') { @@ -345,12 +353,12 @@ where } } - fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> Spanned { + fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult { let mut value_text = String::new(); loop { - if self.is_number(radix) { - value_text.push(self.next_char().unwrap()); + if let Some(c) = self.take_number(radix) { + value_text.push(c); } else if self.chr0 == Some('_') { self.next_char(); } else { @@ -359,18 +367,21 @@ where } let end_pos = self.get_pos(); - let value = BigInt::from_str_radix(&value_text, radix).unwrap(); + let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError { + error: LexicalErrorType::OtherError(format!("{:?}", e)), + location: start_pos.clone(), + })?; Ok((start_pos, Tok::Int { value }, end_pos)) } - fn lex_normal_number(&mut self) -> Spanned { + fn lex_normal_number(&mut self) -> LexResult { let start_pos = self.get_pos(); let mut value_text = String::new(); // Normal number: - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } // If float: @@ -378,8 +389,8 @@ where // Take '.': if self.chr0 == Some('.') { value_text.push(self.next_char().unwrap()); - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } } @@ -392,8 +403,8 @@ where value_text.push(self.next_char().unwrap()); } - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } } @@ -448,7 +459,7 @@ where is_raw: bool, _is_unicode: bool, is_fstring: bool, - ) -> Spanned { + ) -> LexResult { let quote_char = self.next_char().unwrap(); let mut string_content = String::new(); let start_pos = self.get_pos(); @@ -474,7 +485,10 @@ where if let Some(c) = self.next_char() { string_content.push(c) } else { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } else { match self.next_char() { @@ -502,7 +516,10 @@ where string_content.push(c); } None => { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } } @@ -525,7 +542,10 @@ where } else { if c == '\n' { if !triple_quoted { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } self.new_line(); } @@ -533,7 +553,10 @@ where } } None => { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } } @@ -545,7 +568,10 @@ where value: lex_byte(string_content)?, } } else { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } else { Tok::String { @@ -575,8 +601,8 @@ where } } - fn is_number(&self, radix: u32) -> bool { - match radix { + fn take_number(&mut self, radix: u32) -> Option { + let take_char = match radix { 2 => match self.chr0 { Some('0'..='1') => true, _ => false, @@ -594,6 +620,12 @@ where _ => false, }, x => unimplemented!("Radix not implemented: {}", x), + }; + + if take_char { + Some(self.next_char().unwrap()) + } else { + None } } @@ -615,7 +647,7 @@ where self.location.column = 1; } - fn inner_next(&mut self) -> Option> { + fn inner_next(&mut self) -> Option { if !self.pending.is_empty() { return Some(self.pending.remove(0)); } @@ -631,6 +663,17 @@ where loop { match self.chr0 { Some(' ') => { + /* + if tabs != 0 { + // Don't allow spaces after tabs as part of indentation. + // This is technically stricter than python3 but spaces after + // tabs is even more insane than mixing spaces and tabs. + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError("Spaces not allowed as part of indentation after tabs".to_string()), + location: self.get_pos(), + })); + } + */ self.next_char(); spaces += 1; } @@ -639,7 +682,13 @@ where // Don't allow tabs after spaces as part of indentation. // This is technically stricter than python3 but spaces before // tabs is even more insane than mixing spaces and tabs. - panic!("Tabs not allowed as part of indentation after spaces"); + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError( + "Tabs not allowed as part of indentation after spaces" + .to_string(), + ), + location: self.get_pos(), + })); } self.next_char(); tabs += 1; @@ -694,7 +743,10 @@ where self.pending.push(Ok((tok_start, Tok::Dedent, tok_end))); } None => { - panic!("inconsistent use of tabs and spaces in indentation") + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError("inconsistent use of tabs and spaces in indentation".to_string()), + location: self.get_pos(), + })); } _ => { break; @@ -704,12 +756,25 @@ where if indentation_level != *self.indentation_stack.last().unwrap() { // TODO: handle wrong indentations - panic!("Non matching indentation levels!"); + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError( + "Non matching indentation levels!".to_string(), + ), + location: self.get_pos(), + })); } return Some(self.pending.remove(0)); } - None => panic!("inconsistent use of tabs and spaces in indentation"), + None => { + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError( + "inconsistent use of tabs and spaces in indentation" + .to_string(), + ), + location: self.get_pos(), + })); + } } } } @@ -928,7 +993,10 @@ where let tok_end = self.get_pos(); return Some(Ok((tok_start, Tok::NotEqual, tok_end))); } else { - return Some(Err(LexicalError::UnrecognizedToken { tok: '!' })); + return Some(Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: '!' }, + location: tok_start, + })); } } '~' => { @@ -942,7 +1010,10 @@ where ')' => { let result = self.eat_single_char(Tok::Rpar); if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); + return Some(Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + })); } self.nesting -= 1; return Some(result); @@ -955,7 +1026,10 @@ where ']' => { let result = self.eat_single_char(Tok::Rsqb); if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); + return Some(Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + })); } self.nesting -= 1; return Some(result); @@ -968,7 +1042,10 @@ where '}' => { let result = self.eat_single_char(Tok::Rbrace); if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); + return Some(Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + })); } self.nesting -= 1; return Some(result); @@ -1089,7 +1166,10 @@ where } _ => { let c = self.next_char(); - return Some(Err(LexicalError::UnrecognizedToken { tok: c.unwrap() })); + return Some(Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, + location: self.get_pos(), + })); } // Ignore all the rest.. } } @@ -1099,7 +1179,7 @@ where } } - fn eat_single_char(&mut self, ty: Tok) -> Spanned { + fn eat_single_char(&mut self, ty: Tok) -> LexResult { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); @@ -1116,7 +1196,7 @@ impl Iterator for Lexer where T: Iterator, { - type Item = Spanned; + type Item = LexResult; fn next(&mut self) -> Option { // Idea: create some sort of hash map for single char tokens: @@ -1152,7 +1232,10 @@ fn lex_byte(s: String) -> Result, LexicalError> { hex_value.clear(); } } else { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: Default::default(), + }); } } else { match (c, escape) { diff --git a/tests/snippets/invalid_syntax.py b/tests/snippets/invalid_syntax.py index bbbd08a41..3c9c6d46c 100644 --- a/tests/snippets/invalid_syntax.py +++ b/tests/snippets/invalid_syntax.py @@ -1,4 +1,4 @@ - +from testutils import assertRaises src = """ def valid_func(): @@ -14,3 +14,5 @@ except SyntaxError as ex: else: raise AssertionError("Must throw syntax error") +with assertRaises(SyntaxError): + compile('0xX', 'test.py', 'exec') From c7fd54e809d542320c97977ee7734ef36f153068 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Sun, 28 Apr 2019 13:25:57 +0200 Subject: [PATCH 434/658] Convert iterators to pyclass macros --- vm/src/obj/objbytearray.rs | 22 ++++++++++------------ vm/src/obj/objbytes.rs | 20 +++++++++----------- vm/src/obj/objenumerate.rs | 20 +++++++++++--------- vm/src/obj/objfilter.rs | 31 +++++++++++++++---------------- vm/src/obj/objiter.rs | 25 ++++++++++++------------- vm/src/obj/objlist.rs | 23 +++++++++++------------ vm/src/obj/objmap.rs | 30 +++++++++++++++--------------- vm/src/obj/objrange.rs | 22 ++++++++++------------ vm/src/obj/objtuple.rs | 22 ++++++++++------------ vm/src/obj/objzip.rs | 20 +++++++++++--------- vm/src/stdlib/os.rs | 22 +++++++++++----------- 11 files changed, 125 insertions(+), 132 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 16758a438..6f78ecf48 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -7,7 +7,7 @@ use std::ops::{Deref, DerefMut}; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; @@ -84,11 +84,7 @@ pub fn init(context: &PyContext) { "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); - let bytearrayiterator_type = &context.bytearrayiterator_type; - extend_class!(context, bytearrayiterator_type, { - "__next__" => context.new_rustfunc(PyByteArrayIteratorRef::next), - "__iter__" => context.new_rustfunc(PyByteArrayIteratorRef::iter), - }); + PyByteArrayIterator::extend_class(context, &context.bytearrayiterator_type); } fn bytearray_new( @@ -287,6 +283,7 @@ mod tests { } } +#[pyclass] #[derive(Debug)] pub struct PyByteArrayIterator { position: Cell, @@ -299,10 +296,10 @@ impl PyValue for PyByteArrayIterator { } } -type PyByteArrayIteratorRef = PyRef; - -impl PyByteArrayIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyByteArrayIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { if self.position.get() < self.bytearray.value.borrow().len() { let ret = self.bytearray.value.borrow()[self.position.get()]; self.position.set(self.position.get() + 1); @@ -312,7 +309,8 @@ impl PyByteArrayIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0556de053..df9795902 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -64,11 +64,7 @@ pub fn init(context: &PyContext) { extend_class!(context, bytes_type, { "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), }); - let bytesiterator_type = &context.bytesiterator_type; - extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), - }); + PyBytesIterator::extend_class(context, &context.bytesiterator_type); } #[pyimpl] @@ -271,6 +267,7 @@ impl PyBytesRef { } } +#[pyclass] #[derive(Debug)] pub struct PyBytesIterator { position: Cell, @@ -283,10 +280,10 @@ impl PyValue for PyBytesIterator { } } -type PyBytesIteratorRef = PyRef; - -impl PyBytesIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyBytesIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { if self.position.get() < self.bytes.inner.len() { let ret = self.bytes[self.position.get()]; self.position.set(self.position.get() + 1); @@ -296,7 +293,8 @@ impl PyBytesIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index c94aeabbd..56d232272 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -5,13 +5,14 @@ use num_bigint::BigInt; use num_traits::Zero; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::PyIntRef; use super::objiter; use super::objtype::PyClassRef; +#[pyclass] #[derive(Debug)] pub struct PyEnumerate { counter: RefCell, @@ -44,8 +45,10 @@ fn enumerate_new( .into_ref_with_type(vm, cls) } -impl PyEnumerateRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyEnumerate { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let iterator = &self.iterator; let counter = &self.counter; let next_obj = objiter::call_next(vm, iterator)?; @@ -58,16 +61,15 @@ impl PyEnumerateRef { Ok(result) } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let enumerate_type = &context.enumerate_type; - extend_class!(context, enumerate_type, { + PyEnumerate::extend_class(context, &context.enumerate_type); + extend_class!(context, &context.enumerate_type, { "__new__" => context.new_rustfunc(enumerate_new), - "__next__" => context.new_rustfunc(PyEnumerateRef::next), - "__iter__" => context.new_rustfunc(PyEnumerateRef::iter), }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index 858a0531a..24800d6aa 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -1,4 +1,4 @@ -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; // Required for arg_check! to use isinstance use super::objbool; @@ -7,6 +7,11 @@ use crate::obj::objtype::PyClassRef; pub type PyFilterRef = PyRef; +/// filter(function or None, iterable) --> filter object +/// +/// Return an iterator yielding those items of iterable for which function(item) +/// is true. If function is None, return the items that are true. +#[pyclass] #[derive(Debug)] pub struct PyFilter { predicate: PyObjectRef, @@ -34,8 +39,10 @@ fn filter_new( .into_ref_with_type(vm, cls) } -impl PyFilterRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyFilter { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let predicate = &self.predicate; let iterator = &self.iterator; loop { @@ -53,23 +60,15 @@ impl PyFilterRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let filter_type = &context.filter_type; - - let filter_doc = - "filter(function or None, iterable) --> filter object\n\n\ - Return an iterator yielding those items of iterable for which function(item)\n\ - is true. If function is None, return the items that are true."; - - extend_class!(context, filter_type, { + PyFilter::extend_class(context, &context.filter_type); + extend_class!(context, &context.filter_type, { "__new__" => context.new_rustfunc(filter_new), - "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(PyFilterRef::next), - "__iter__" => context.new_rustfunc(PyFilterRef::iter), }); } diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 6e1dc1d06..efe672d62 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -4,7 +4,9 @@ use std::cell::Cell; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{ + PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, +}; use crate::vm::VirtualMachine; use super::objtype; @@ -75,6 +77,7 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } +#[pyclass] #[derive(Debug)] pub struct PySequenceIterator { pub position: Cell, @@ -87,10 +90,10 @@ impl PyValue for PySequenceIterator { } } -type PySequenceIteratorRef = PyRef; - -impl PySequenceIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PySequenceIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let number = vm.ctx.new_int(self.position.get()); match vm.call_method(&self.obj, "__getitem__", vec![number]) { Ok(val) => { @@ -105,16 +108,12 @@ impl PySequenceIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let iter_type = &context.iter_type; - - extend_class!(context, iter_type, { - "__next__" => context.new_rustfunc(PySequenceIteratorRef::next), - "__iter__" => context.new_rustfunc(PySequenceIteratorRef::iter), - }); + PySequenceIterator::extend_class(context, &context.iter_type); } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 5c1bad5af..6b00b6828 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -8,7 +8,8 @@ use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + IdProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -776,6 +777,7 @@ fn list_sort(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[pyclass] #[derive(Debug)] pub struct PyListIterator { pub position: Cell, @@ -788,10 +790,10 @@ impl PyValue for PyListIterator { } } -type PyListIteratorRef = PyRef; - -impl PyListIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyListIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { if self.position.get() < self.list.elements.borrow().len() { let ret = self.list.elements.borrow()[self.position.get()].clone(); self.position.set(self.position.get() + 1); @@ -801,8 +803,9 @@ impl PyListIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } @@ -848,9 +851,5 @@ pub fn init(context: &PyContext) { "remove" => context.new_rustfunc(PyListRef::remove) }); - let listiterator_type = &context.listiterator_type; - extend_class!(context, listiterator_type, { - "__next__" => context.new_rustfunc(PyListIteratorRef::next), - "__iter__" => context.new_rustfunc(PyListIteratorRef::iter), - }); + PyListIterator::extend_class(context, &context.listiterator_type); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index 061f4179f..61fe67c0d 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -1,10 +1,15 @@ use crate::function::Args; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; use super::objtype::PyClassRef; +/// map(func, *iterables) --> map object +/// +/// Make an iterator that computes the function using arguments from +/// each of the iterables. Stops when the shortest iterable is exhausted. +#[pyclass] #[derive(Debug)] pub struct PyMap { mapper: PyObjectRef, @@ -35,8 +40,10 @@ fn map_new( .into_ref_with_type(vm, cls.clone()) } -impl PyMapRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyMap { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let next_objs = self .iterators .iter() @@ -47,22 +54,15 @@ impl PyMapRef { vm.invoke(self.mapper.clone(), next_objs) } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let map_type = &context.map_type; - - let map_doc = "map(func, *iterables) --> map object\n\n\ - Make an iterator that computes the function using arguments from\n\ - each of the iterables. Stops when the shortest iterable is exhausted."; - - extend_class!(context, map_type, { + PyMap::extend_class(context, &context.map_type); + extend_class!(context, &context.map_type, { "__new__" => context.new_rustfunc(map_new), - "__next__" => context.new_rustfunc(PyMapRef::next), - "__iter__" => context.new_rustfunc(PyMapRef::iter), - "__doc__" => context.new_str(map_doc.to_string()) }); } diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index d98e7491f..2a43579c5 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,7 +6,7 @@ use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -124,11 +124,7 @@ pub fn init(context: &PyContext) { "step" => context.new_property(PyRange::step), }); - let rangeiterator_type = &context.rangeiterator_type; - extend_class!(context, rangeiterator_type, { - "__next__" => context.new_rustfunc(PyRangeIteratorRef::next), - "__iter__" => context.new_rustfunc(PyRangeIteratorRef::iter), - }); + PyRangeIterator::extend_class(context, &context.rangeiterator_type); } type PyRangeRef = PyRef; @@ -341,6 +337,7 @@ fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(range.into_object()) } +#[pyclass] #[derive(Debug)] pub struct PyRangeIterator { position: Cell, @@ -353,10 +350,10 @@ impl PyValue for PyRangeIterator { } } -type PyRangeIteratorRef = PyRef; - -impl PyRangeIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyRangeIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { let position = BigInt::from(self.position.get()); if let Some(int) = self.range.get(&position) { self.position.set(self.position.get() + 1); @@ -366,8 +363,9 @@ impl PyRangeIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 216945284..42999189f 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -3,7 +3,7 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -224,6 +224,7 @@ fn tuple_new( PyTuple::from(elements).into_ref_with_type(vm, cls) } +#[pyclass] #[derive(Debug)] pub struct PyTupleIterator { position: Cell, @@ -236,10 +237,10 @@ impl PyValue for PyTupleIterator { } } -type PyTupleIteratorRef = PyRef; - -impl PyTupleIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyTupleIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { if self.position.get() < self.tuple.elements.borrow().len() { let ret = self.tuple.elements.borrow()[self.position.get()].clone(); self.position.set(self.position.get() + 1); @@ -249,8 +250,9 @@ impl PyTupleIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } @@ -282,9 +284,5 @@ If the argument is a tuple, the return value is the same object."; "index" => context.new_rustfunc(PyTupleRef::index) }); - let tupleiterator_type = &context.tupleiterator_type; - extend_class!(context, tupleiterator_type, { - "__next__" => context.new_rustfunc(PyTupleIteratorRef::next), - "__iter__" => context.new_rustfunc(PyTupleIteratorRef::iter), - }); + PyTupleIterator::extend_class(context, &context.tupleiterator_type); } diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index 81640a724..d9c577aa0 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -1,5 +1,5 @@ use crate::function::Args; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; @@ -7,6 +7,7 @@ use crate::obj::objtype::PyClassRef; pub type PyZipRef = PyRef; +#[pyclass] #[derive(Debug)] pub struct PyZip { iterators: Vec, @@ -26,8 +27,10 @@ fn zip_new(cls: PyClassRef, iterables: Args, vm: &VirtualMachine) -> PyResult PyResult { +#[pyimpl] +impl PyZip { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { if self.iterators.is_empty() { Err(objiter::new_stop_iteration(vm)) } else { @@ -41,16 +44,15 @@ impl PyZipRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let zip_type = &context.zip_type; - extend_class!(context, zip_type, { + PyZip::extend_class(context, &context.zip_type); + extend_class!(context, &context.zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(PyZipRef::next), - "__iter__" => context.new_rustfunc(PyZipRef::iter), }); } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6d81ca9c1..2b6c859eb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -15,7 +15,7 @@ use crate::obj::objiter; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{ItemProtocol, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -232,6 +232,7 @@ impl DirEntryRef { } } +#[pyclass] #[derive(Debug)] struct ScandirIterator { entries: RefCell, @@ -243,10 +244,10 @@ impl PyValue for ScandirIterator { } } -type ScandirIteratorRef = PyRef; - -impl ScandirIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl ScandirIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { match self.entries.borrow_mut().next() { Some(entry) => match entry { Ok(entry) => Ok(DirEntry { entry }.into_ref(vm).into_object()), @@ -256,8 +257,9 @@ impl ScandirIteratorRef { } } - fn iter(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } @@ -418,10 +420,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let environ = _os_environ(vm); - let scandir_iter = py_class!(ctx, "ScandirIter", ctx.object(), { - "__iter__" => ctx.new_rustfunc(ScandirIteratorRef::iter), - "__next__" => ctx.new_rustfunc(ScandirIteratorRef::next), - }); + let scandir_iter = ctx.new_class("ScandirIter", ctx.object()); + ScandirIterator::extend_class(ctx, &scandir_iter); let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { "name" => ctx.new_property(DirEntryRef::name), From c138dea9926fe5fe4148e33485ffb14bb02c7e08 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 04:03:18 +0900 Subject: [PATCH 435/658] Remove int.__pow__ float handling --- vm/src/obj/objint.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 3687ed2b0..98fc39a1b 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -13,7 +13,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objfloat::{self, PyFloat}; +use super::objfloat::PyFloat; use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -327,9 +327,6 @@ impl PyInt { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); vm.ctx.new_int(self.value.pow(v2)) - } else if objtype::isinstance(&other, &vm.ctx.float_type()) { - let v2 = objfloat::get_value(&other); - vm.ctx.new_float((self.value.to_f64().unwrap()).powf(v2)) } else { vm.ctx.not_implemented() } From 614378d7202821c070563ac5a773ecd39ab036d9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 04:56:51 +0900 Subject: [PATCH 436/658] Add int.__pow__ negative exp impl --- tests/snippets/ints.py | 3 +++ vm/src/obj/objint.rs | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index a072cc542..3aed1144d 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -37,6 +37,8 @@ assert (2).__mul__(1) == 2 assert (2).__rmul__(1) == 2 assert (2).__truediv__(1) == 2.0 assert (2).__rtruediv__(1) == 0.5 +assert (2).__pow__(3) == 8 +assert (10).__pow__(-1) == 0.1 # real/imag attributes assert (1).real == 1 @@ -58,6 +60,7 @@ assert (2).__mul__(1.0) == NotImplemented assert (2).__rmul__(1.0) == NotImplemented assert (2).__truediv__(1.0) == NotImplemented assert (2).__rtruediv__(1.0) == NotImplemented +assert (2).__pow__(3.0) == NotImplemented assert int() == 0 diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 98fc39a1b..12caa060a 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -323,13 +323,21 @@ impl PyInt { } #[pymethod(name = "__pow__")] - fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if objtype::isinstance(&other, &vm.ctx.int_type()) { - let v2 = get_value(&other).to_u32().unwrap(); - vm.ctx.new_int(self.value.pow(v2)) + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(if objtype::isinstance(&other, &vm.ctx.int_type()) { + let other = other.payload::().unwrap(); + if other.value.is_negative() { + let v1 = self.float(vm)?; + let v2 = other.float(vm)?; + vm.ctx.new_float(v1.pow(v2.to_f64().unwrap())) + } else if let Some(v2) = other.value.to_u32() { + vm.ctx.new_int(self.value.pow(v2)) + } else { + vm.ctx.not_implemented() + } } else { vm.ctx.not_implemented() - } + }) } #[pymethod(name = "__mod__")] @@ -400,10 +408,9 @@ impl PyInt { } #[pymethod(name = "__float__")] - fn float(&self, vm: &VirtualMachine) -> PyResult { + fn float(&self, vm: &VirtualMachine) -> PyResult { self.value .to_f64() - .map(PyFloat::from) .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) } @@ -541,9 +548,7 @@ pub fn get_value(obj: &PyObjectRef) -> &BigInt { } pub fn get_float_value(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - get_value(obj).to_f64().ok_or_else(|| { - vm.new_overflow_error("OverflowError: int too large to convert to float".to_string()) - }) + obj.payload::().unwrap().float(vm) } #[inline] From eb9ca06ee721eebd991cde1b7d42bdf49837819d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 05:09:11 +0900 Subject: [PATCH 437/658] Add int.__rpow__ --- tests/snippets/ints.py | 2 ++ vm/src/obj/objint.rs | 39 +++++++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 3aed1144d..40b4c2c6d 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -39,6 +39,7 @@ assert (2).__truediv__(1) == 2.0 assert (2).__rtruediv__(1) == 0.5 assert (2).__pow__(3) == 8 assert (10).__pow__(-1) == 0.1 +assert (2).__rpow__(3) == 9 # real/imag attributes assert (1).real == 1 @@ -61,6 +62,7 @@ assert (2).__rmul__(1.0) == NotImplemented assert (2).__truediv__(1.0) == NotImplemented assert (2).__rtruediv__(1.0) == NotImplemented assert (2).__pow__(3.0) == NotImplemented +assert (2).__rpow__(3.0) == NotImplemented assert int() == 0 diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 12caa060a..fe977ca0b 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -111,6 +111,19 @@ impl_try_from_object_int!( (u64, to_u64), ); +fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { + Ok(if int2.value.is_negative() { + let v1 = int1.float(vm)?; + let v2 = int2.float(vm)?; + vm.ctx.new_float(v1.pow(v2)) + } else if let Some(v2) = int2.value.to_u64() { + vm.ctx.new_int(int1.value.pow(v2)) + } else { + // missing feature: BigInt exp + vm.ctx.not_implemented() + }) +} + #[pyimpl] impl PyInt { #[pymethod(name = "__eq__")] @@ -324,20 +337,22 @@ impl PyInt { #[pymethod(name = "__pow__")] fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Ok(if objtype::isinstance(&other, &vm.ctx.int_type()) { + if objtype::isinstance(&other, &vm.ctx.int_type()) { let other = other.payload::().unwrap(); - if other.value.is_negative() { - let v1 = self.float(vm)?; - let v2 = other.float(vm)?; - vm.ctx.new_float(v1.pow(v2.to_f64().unwrap())) - } else if let Some(v2) = other.value.to_u32() { - vm.ctx.new_int(self.value.pow(v2)) - } else { - vm.ctx.not_implemented() - } + inner_pow(self, &other, vm) } else { - vm.ctx.not_implemented() - }) + Ok(vm.ctx.not_implemented()) + } + } + + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + let other = other.payload::().unwrap(); + inner_pow(&other, self, vm) + } else { + Ok(vm.ctx.not_implemented()) + } } #[pymethod(name = "__mod__")] From 8d4562b0bbb08179c5218f7bfcc48e8ec9a28e8c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 22:32:45 +0900 Subject: [PATCH 438/658] Fix __builtins__.pow to reflect __rpow__ --- tests/snippets/math_basics.py | 6 +++++- vm/src/builtins.rs | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 51ea5c726..28c145e3f 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -14,7 +14,7 @@ assert a ** 3 == 64 assert a * 3 == 12 assert a / 2 == 2 assert 2 == a / 2 -# assert a % 3 == 1 +assert a % 3 == 1 assert a - 3 == 1 assert -a == -4 assert +a == 4 @@ -38,3 +38,7 @@ assert_raises( OverflowError, lambda: round(-float('inf')), 'OverflowError: cannot convert float NaN to integer') + +assert pow(2, 2) == 4 +assert pow(1, 2.0) == 1.0 +assert pow(2.0, 1) == 2.0 diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 300d11b31..27a53dc92 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -529,11 +529,11 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(x, None), (y, None)], optional = [(mod_value, Some(vm.ctx.int_type()))] ); - let pow_method_name = "__pow__"; - let result = match vm.get_method(x.clone(), pow_method_name) { - Ok(attrib) => vm.invoke(attrib, vec![y.clone()]), - Err(..) => Err(vm.new_type_error("unsupported operand type(s) for pow".to_string())), - }; + + let result = vm.call_or_reflection(x.clone(), y.clone(), "__pow__", "__rpow__", |vm, x, y| { + Err(vm.new_unsupported_operand_error(x, y, "pow")) + }); + //Check if the 3rd argument is defined and perform modulus on the result //this should be optimized in the future to perform a "power-mod" algorithm in //order to improve performance From c18615b6e5d5011a2330790d169dcbffede06369 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 23:34:11 +0900 Subject: [PATCH 439/658] Fix __builtins__.pow to perform modpow for 3rd argument --- tests/snippets/math_basics.py | 22 +++++++++++++++++++ vm/src/builtins.rs | 41 +++++++++++++++++++++-------------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 28c145e3f..2cef18b74 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -42,3 +42,25 @@ assert_raises( assert pow(2, 2) == 4 assert pow(1, 2.0) == 1.0 assert pow(2.0, 1) == 2.0 + +assert pow(2, 4, 5) == 1 +assert_raises( + TypeError, + lambda: pow(2, 4, 5.0), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + TypeError, + lambda: pow(2, 4.0, 5), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + TypeError, + lambda: pow(2.0, 4, 5), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + ValueError, + lambda: pow(2, -1, 5), + 'pow() 2nd argument cannot be negative when 3rd argument specified') +assert_raises( + ValueError, + lambda: pow(2, 2, 0), + 'pow() 3rd argument cannot be 0') diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 27a53dc92..79c276888 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -6,7 +6,8 @@ use std::char; use std::io::{self, Write}; use std::path::PathBuf; -use num_traits::Signed; +use num_bigint::Sign; +use num_traits::{Signed, Zero}; use crate::compile; use crate::import::import_module; @@ -530,24 +531,32 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { optional = [(mod_value, Some(vm.ctx.int_type()))] ); - let result = vm.call_or_reflection(x.clone(), y.clone(), "__pow__", "__rpow__", |vm, x, y| { - Err(vm.new_unsupported_operand_error(x, y, "pow")) - }); - - //Check if the 3rd argument is defined and perform modulus on the result - //this should be optimized in the future to perform a "power-mod" algorithm in - //order to improve performance match mod_value { - Some(mod_value) => { - let mod_method_name = "__mod__"; - match vm.get_method(result.expect("result not defined").clone(), mod_method_name) { - Ok(value) => vm.invoke(value, vec![mod_value.clone()]), - Err(..) => { - Err(vm.new_type_error("unsupported operand type(s) for mod".to_string())) - } + None => vm.call_or_reflection(x.clone(), y.clone(), "__pow__", "__rpow__", |vm, x, y| { + Err(vm.new_unsupported_operand_error(x, y, "pow")) + }), + Some(m) => { + // Check if the 3rd argument is defined and perform modulus on the result + if !(objtype::isinstance(x, &vm.ctx.int_type()) + && objtype::isinstance(y, &vm.ctx.int_type())) + { + return Err(vm.new_type_error( + "pow() 3rd argument not allowed unless all arguments are integers".to_string(), + )); } + let y = objint::get_value(y); + if y.sign() == Sign::Minus { + return Err(vm.new_value_error( + "pow() 2nd argument cannot be negative when 3rd argument specified".to_string(), + )); + } + let m = objint::get_value(m); + if m.is_zero() { + return Err(vm.new_value_error("pow() 3rd argument cannot be 0".to_string())); + } + let x = objint::get_value(x); + Ok(vm.new_int(x.modpow(&y, &m))) } - None => result, } } From 0bf7ff96b804eb454f6bab2800a2644c19a63124 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 2 May 2019 22:42:28 +0900 Subject: [PATCH 440/658] Special handling for base -1, 0, 1 for big exp --- tests/snippets/math_basics.py | 5 +++++ vm/src/obj/objint.rs | 22 ++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 2cef18b74..d950b13a9 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -39,9 +39,14 @@ assert_raises( lambda: round(-float('inf')), 'OverflowError: cannot convert float NaN to integer') +assert pow(0, 0) == 1 assert pow(2, 2) == 4 assert pow(1, 2.0) == 1.0 assert pow(2.0, 1) == 2.0 +assert pow(0, 10**1000) == 0 +assert pow(1, 10**1000) == 1 +assert pow(-1, 10**1000+1) == -1 +assert pow(-1, 10**1000) == 1 assert pow(2, 4, 5) == 1 assert_raises( diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index fe977ca0b..88c91f89f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -3,7 +3,7 @@ use std::hash::{Hash, Hasher}; use num_bigint::BigInt; use num_integer::Integer; -use num_traits::{Pow, Signed, ToPrimitive, Zero}; +use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::{OptionalArg, PyFuncArgs}; @@ -13,7 +13,6 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objfloat::PyFloat; use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -116,11 +115,22 @@ fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { let v1 = int1.float(vm)?; let v2 = int2.float(vm)?; vm.ctx.new_float(v1.pow(v2)) - } else if let Some(v2) = int2.value.to_u64() { - vm.ctx.new_int(int1.value.pow(v2)) } else { - // missing feature: BigInt exp - vm.ctx.not_implemented() + if let Some(v2) = int2.value.to_u64() { + vm.ctx.new_int(int1.value.pow(v2)) + } else if int1.value.is_one() || int1.value.is_zero() { + vm.ctx.new_int(int1.value.clone()) + } else if int1.value == BigInt::from(-1) { + if int2.value.is_odd() { + vm.ctx.new_int(-1) + } else { + vm.ctx.new_int(1) + } + } else { + // missing feature: BigInt exp + // practically, exp over u64 is not possible to calculate anyway + vm.ctx.not_implemented() + } }) } From 5bc1aa08dc0f676df1490860a7b352953085d9f5 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 2 May 2019 16:55:33 +0200 Subject: [PATCH 441/658] Add async def and async for parsing. --- parser/src/ast.rs | 13 ++++++++++ parser/src/lexer.rs | 2 ++ parser/src/python.lalrpop | 39 ++++++++++++++++++++---------- parser/src/token.rs | 2 ++ tests/snippets/test_async_stuff.py | 13 ++++++++++ vm/src/compile.rs | 6 +++++ vm/src/stdlib/ast.rs | 37 ++++++++++++++++++++++++++++ vm/src/symboltable.rs | 13 ++++++++++ 8 files changed, 112 insertions(+), 13 deletions(-) create mode 100644 tests/snippets/test_async_stuff.py diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 9fbe8153e..5d3fcb921 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -98,6 +98,12 @@ pub enum Statement { body: Vec, orelse: Option>, }, + AsyncFor { + target: Expression, + iter: Expression, + body: Vec, + orelse: Option>, + }, Raise { exception: Option, cause: Option, @@ -122,6 +128,13 @@ pub enum Statement { decorator_list: Vec, returns: Option, }, + AsyncFunctionDef { + name: String, + args: Parameters, + body: Vec, + decorator_list: Vec, + returns: Option, + }, } #[derive(Debug, PartialEq)] diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 11d30cbee..ac0fc89fc 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -95,6 +95,8 @@ pub fn get_keywords() -> HashMap { keywords.insert(String::from("and"), Tok::And); keywords.insert(String::from("as"), Tok::As); keywords.insert(String::from("assert"), Tok::Assert); + keywords.insert(String::from("async"), Tok::Async); + keywords.insert(String::from("await"), Tok::Await); keywords.insert(String::from("break"), Tok::Break); keywords.insert(String::from("class"), Tok::Class); keywords.insert(String::from("continue"), Tok::Continue); diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index a9663943f..89f3fd38a 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -348,13 +348,14 @@ WhileStatement: ast::LocatedStatement = { }; ForStatement: ast::LocatedStatement = { - "for" "in" ":" => { - let or_else = s2.map(|s| s.2); + "for" "in" ":" => { + let orelse = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::For { - target: e, - iter: t, body: s, orelse: or_else + node: if is_async.is_some() { + ast::Statement::AsyncFor { target, iter, body, orelse } + } else { + ast::Statement::For { target, iter, body, orelse } }, } }, @@ -410,16 +411,26 @@ WithItem: ast::WithItem = { }; FuncDef: ast::LocatedStatement = { - "def" " Test)?> ":" => { + "def" " Test)?> ":" => { ast::LocatedStatement { location: loc, - node: ast::Statement::FunctionDef { - name: i, - args: a, - body: s, - decorator_list: d, - returns: r.map(|x| x.1), - } + node: if is_async.is_some() { + ast::Statement::AsyncFunctionDef { + name: i, + args: a, + body: s, + decorator_list: d, + returns: r.map(|x| x.1), + } + } else { + ast::Statement::FunctionDef { + name: i, + args: a, + body: s, + decorator_list: d, + returns: r.map(|x| x.1), + } + } } }, }; @@ -1045,6 +1056,8 @@ extern { "and" => lexer::Tok::And, "as" => lexer::Tok::As, "assert" => lexer::Tok::Assert, + "async" => lexer::Tok::Async, + "await" => lexer::Tok::Await, "break" => lexer::Tok::Break, "class" => lexer::Tok::Class, "continue" => lexer::Tok::Continue, diff --git a/parser/src/token.rs b/parser/src/token.rs index d2494a1d9..f9bd2a568 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -72,6 +72,8 @@ pub enum Tok { And, As, Assert, + Async, + Await, Break, Class, Continue, diff --git a/tests/snippets/test_async_stuff.py b/tests/snippets/test_async_stuff.py new file mode 100644 index 000000000..d19f5c382 --- /dev/null +++ b/tests/snippets/test_async_stuff.py @@ -0,0 +1,13 @@ + +import sys +import ast + +src = """ +async def x(): + async for x in [1,2,3]: + pass +""" + +mod = ast.parse(src) +# print(mod) +# print(ast.dump(mod)) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 76744f112..5a25be479 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -322,6 +322,9 @@ impl Compiler { body, orelse, } => self.compile_for(target, iter, body, orelse)?, + ast::Statement::AsyncFor { .. } => { + unimplemented!("async for"); + } ast::Statement::Raise { exception, cause } => match exception { Some(value) => { self.compile_expression(value)?; @@ -352,6 +355,9 @@ impl Compiler { decorator_list, returns, } => self.compile_function_def(name, args, body, decorator_list, returns)?, + ast::Statement::AsyncFunctionDef { .. } => { + unimplemented!("async def"); + } ast::Statement::ClassDef { name, body, diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 816882f31..16485e85d 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -96,6 +96,26 @@ fn statement_to_ast( returns => py_returns }) } + ast::Statement::AsyncFunctionDef { + name, + args, + body, + decorator_list, + returns, + } => { + let py_returns = if let Some(hint) = returns { + expression_to_ast(vm, hint)?.into_object() + } else { + vm.ctx.none() + }; + node!(vm, AsyncFunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + returns => py_returns + }) + } ast::Statement::Continue => node!(vm, Continue), ast::Statement::Break => node!(vm, Break), ast::Statement::Pass => node!(vm, Pass), @@ -152,6 +172,21 @@ fn statement_to_ast( vm.ctx.none() } }), + ast::Statement::AsyncFor { + target, + iter, + body, + orelse, + } => node!(vm, AsyncFor, { + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() + } else { + vm.ctx.none() + } + }), ast::Statement::While { test, body, orelse } => node!(vm, While, { test => expression_to_ast(vm, test)?, body => statements_to_ast(vm, body)?, @@ -462,6 +497,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { // TODO: There's got to be a better way! "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), + "AsyncFor" => py_class!(ctx, "_ast.AsyncFor", ast_base.clone(), {}), + "AsyncFunctionDef" => py_class!(ctx, "_ast.AsyncFunctionDef", ast_base.clone(), {}), "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 6bdec1e96..2b975cdf1 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -223,6 +223,13 @@ impl SymbolTableBuilder { args, decorator_list, returns, + } + | ast::Statement::AsyncFunctionDef { + name, + body, + args, + decorator_list, + returns, } => { self.scan_expressions(decorator_list)?; self.register_name(name, SymbolRole::Assigned)?; @@ -265,6 +272,12 @@ impl SymbolTableBuilder { iter, body, orelse, + } + | ast::Statement::AsyncFor { + target, + iter, + body, + orelse, } => { self.scan_expression(target)?; self.scan_expression(iter)?; From b3b523ffa9b58ea82943cb3afc0008e161c570a8 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 2 May 2019 17:11:51 +0200 Subject: [PATCH 442/658] Add await syntax. --- parser/src/ast.rs | 4 ++++ parser/src/python.lalrpop | 16 +++++++++++++--- tests/snippets/test_async_stuff.py | 2 +- vm/src/compile.rs | 3 +++ vm/src/stdlib/ast.rs | 7 +++++++ vm/src/symboltable.rs | 3 +++ 6 files changed, 31 insertions(+), 4 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 5d3fcb921..e0a050138 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -163,6 +163,9 @@ pub enum Expression { op: UnaryOperator, a: Box, }, + Await { + value: Box, + }, Yield { value: Option>, }, @@ -240,6 +243,7 @@ impl Expression { match self { BoolOp { .. } | Binop { .. } | Unop { .. } => "operator", Subscript { .. } => "subscript", + Await { .. } => "await expression", Yield { .. } | YieldFrom { .. } => "yield expression", Compare { .. } => "comparison", Attribute { .. } => "attribute", diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 89f3fd38a..02fd9e0b9 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -747,10 +747,20 @@ Power: ast::Expression = { }; AtomExpr: ast::Expression = { + => { + if is_await.is_some() { + ast::Expression::Await { value: Box::new(atom) } + } else { + atom + } + } +} + +AtomExpr2: ast::Expression = { Atom, - "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, - "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, - "." => ast::Expression::Attribute { value: Box::new(e), name: n }, + "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, + "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, + "." => ast::Expression::Attribute { value: Box::new(e), name: n }, }; SubscriptList: ast::Expression = { diff --git a/tests/snippets/test_async_stuff.py b/tests/snippets/test_async_stuff.py index d19f5c382..a58569296 100644 --- a/tests/snippets/test_async_stuff.py +++ b/tests/snippets/test_async_stuff.py @@ -5,7 +5,7 @@ import ast src = """ async def x(): async for x in [1,2,3]: - pass + await y() """ mod = ast.parse(src) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 5a25be479..ce8f66996 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -1271,6 +1271,9 @@ impl Compiler { }; self.emit(Instruction::YieldValue); } + ast::Expression::Await { .. } => { + unimplemented!("await"); + } ast::Expression::YieldFrom { value } => { self.mark_generator(); self.compile_expression(value)?; diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 16485e85d..f27b5eb40 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -385,6 +385,12 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes } } } + ast::Expression::Await { value } => { + let py_value = expression_to_ast(vm, value)?; + node!(vm, Await, { + value => py_value + }) + } ast::Expression::Yield { value } => { let py_value = match value { Some(value) => expression_to_ast(vm, value)?.into_object(), @@ -501,6 +507,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "AsyncFunctionDef" => py_class!(ctx, "_ast.AsyncFunctionDef", ast_base.clone(), {}), "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), + "Await" => py_class!(ctx, "_ast.Await", ast_base.clone(), {}), "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 2b975cdf1..1d4a72ed9 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -408,6 +408,9 @@ impl SymbolTableBuilder { self.scan_expression(value)?; } } + ast::Expression::Await { value } => { + self.scan_expression(value)?; + } ast::Expression::Yield { value } => { if let Some(expression) = value { self.scan_expression(expression)?; From dd0c70f289cb338d107dd39c68296790ee2d8742 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 2 May 2019 16:36:35 +0100 Subject: [PATCH 443/658] Fix typo in objobject. --- vm/src/obj/objobject.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 47cfa008d..31e52fc99 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -213,7 +213,7 @@ fn object_dict_setter( vm: &VirtualMachine, ) -> PyResult { Err(vm.new_not_implemented_error( - "Setting __dict__ attribute on am object isn't yet implemented".to_string(), + "Setting __dict__ attribute on an object isn't yet implemented".to_string(), )) } From 65d602571d50f692162520985163188bf14b7e06 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 2 May 2019 16:42:31 +0100 Subject: [PATCH 444/658] Create mappingproxy type with __getitem__ and __contains__. --- tests/snippets/mappingproxy.py | 18 +++++++++++++++ vm/src/obj/mod.rs | 1 + vm/src/obj/objmappingproxy.rs | 42 ++++++++++++++++++++++++++++++++++ vm/src/obj/objtype.rs | 16 +++++++++++++ vm/src/pyobject.rs | 5 ++++ 5 files changed, 82 insertions(+) create mode 100644 tests/snippets/mappingproxy.py create mode 100644 vm/src/obj/objmappingproxy.rs diff --git a/tests/snippets/mappingproxy.py b/tests/snippets/mappingproxy.py new file mode 100644 index 000000000..624bf6d5f --- /dev/null +++ b/tests/snippets/mappingproxy.py @@ -0,0 +1,18 @@ +from testutils import assertRaises + +class A(dict): + def a(): + pass + + def b(): + pass + + +assert A.__dict__['a'] == A.a +with assertRaises(KeyError): + A.__dict__['not_here'] + +assert 'b' in A.__dict__ +assert 'c' not in A.__dict__ + +assert '__dict__' in A.__dict__ diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index bfe89fd15..c9401d8c4 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -20,6 +20,7 @@ pub mod objint; pub mod objiter; pub mod objlist; pub mod objmap; +pub mod objmappingproxy; pub mod objmemory; pub mod objmodule; pub mod objnone; diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs new file mode 100644 index 000000000..c46591105 --- /dev/null +++ b/vm/src/obj/objmappingproxy.rs @@ -0,0 +1,42 @@ +use super::objstr::PyStringRef; +use super::objtype::{self, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[pyclass] +#[derive(Debug)] +pub struct PyMappingProxy { + class: PyClassRef, +} + +pub type PyMappingProxyRef = PyRef; + +impl PyValue for PyMappingProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.mappingproxy_type.clone() + } +} + +#[pyimpl] +impl PyMappingProxy { + pub fn new(class: PyClassRef) -> PyMappingProxy { + PyMappingProxy { class } + } + + #[pymethod(name = "__getitem__")] + pub fn getitem(&self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { + if let Some(value) = objtype::class_get_attr(&self.class, key.as_str()) { + return Ok(value); + } + Err(vm.new_key_error(format!("Key not found: {}", key))) + } + + #[pymethod(name = "__contains__")] + pub fn contains(&self, attr: PyStringRef, _vm: &VirtualMachine) -> bool { + objtype::class_has_attr(&self.class, attr.as_str()) + } +} + +pub fn init(context: &PyContext) { + PyMappingProxy::extend_class(context, &context.mappingproxy_type) +} diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index c14e51230..c66e57354 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -11,6 +11,7 @@ use crate::vm::VirtualMachine; use super::objdict::PyDictRef; use super::objlist::PyList; +use super::objmappingproxy::PyMappingProxy; use super::objproperty::PropertyBuilder; use super::objstr::PyStringRef; use super::objtuple::PyTuple; @@ -196,6 +197,11 @@ pub fn init(ctx: &PyContext) { extend_class!(&ctx, &ctx.type_type, { "__call__" => ctx.new_rustfunc(type_call), + "__dict__" => + PropertyBuilder::new(ctx) + .add_getter(type_dict) + .add_setter(type_dict_setter) + .create(), "__new__" => ctx.new_rustfunc(type_new), "__mro__" => PropertyBuilder::new(ctx) @@ -273,6 +279,16 @@ pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMach Ok(obj) } +fn type_dict(class: PyClassRef, _vm: &VirtualMachine) -> PyMappingProxy { + PyMappingProxy::new(class) +} + +fn type_dict_setter(_instance: PyClassRef, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on a type isn't yet implemented".to_string(), + )) +} + // This is the internal get_attr implementation for fast lookup on a class. pub fn class_get_attr(class: &PyClassRef, attr_name: &str) -> Option { if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 89e5c8c39..454a1605f 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -37,6 +37,7 @@ use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objiter; use crate::obj::objlist::{self, PyList}; use crate::obj::objmap; +use crate::obj::objmappingproxy; use crate::obj::objmemory; use crate::obj::objmodule::{self, PyModule}; use crate::obj::objnone::{self, PyNone, PyNoneRef}; @@ -160,6 +161,7 @@ pub struct PyContext { pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, pub weakproxy_type: PyClassRef, + pub mappingproxy_type: PyClassRef, pub object: PyClassRef, pub exceptions: exceptions::ExceptionZoo, } @@ -292,6 +294,7 @@ impl PyContext { let range_type = create_type("range", &type_type, &object_type); let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); + let mappingproxy_type = create_type("mappingproxy", &type_type, &object_type); let exceptions = exceptions::ExceptionZoo::new(&type_type, &object_type); fn create_object(payload: T, cls: &PyClassRef) -> PyRef { @@ -358,6 +361,7 @@ impl PyContext { function_type, builtin_function_or_method_type, super_type, + mappingproxy_type, property_type, readonly_property_type, generator_type, @@ -403,6 +407,7 @@ impl PyContext { objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); + objmappingproxy::init(&context); exceptions::init(&context); context } From 620bea7830dc18fffc7d7da57e94eb3dec2f276d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 3 May 2019 01:00:49 +0300 Subject: [PATCH 445/658] Preallocate output string of str.title() with required size --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 8ea438e35..22eda1f34 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -425,7 +425,7 @@ impl PyString { /// uppercase character and the remaining characters are lowercase. #[pymethod] fn title(&self, _vm: &VirtualMachine) -> String { - let mut title = String::new(); + let mut title = String::with_capacity(self.value.len()); let mut previous_is_cased = false; for c in self.value.chars() { if c.is_lowercase() { From 26306b528f513efc849c2f031c90f7efd99e8cda Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 01:38:32 +0300 Subject: [PATCH 446/658] Simplify PyInt.rxor to reuse PyInt.xor --- vm/src/obj/objint.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 3687ed2b0..fdb0459af 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -296,11 +296,7 @@ impl PyInt { #[pymethod(name = "__rxor__")] fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx.new_int(get_value(&other) ^ (&self.value)) - } else { - vm.ctx.not_implemented() - } + self.xor(other, vm) } #[pymethod(name = "__or__")] From ea347dfdeca1c7548513ad8deb89c6b66810cfe1 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 01:04:52 +0300 Subject: [PATCH 447/658] Implement bool.__or__, __and__ and __xor__ --- tests/snippets/bools.py | 12 ++++++++++++ vm/src/obj/objbool.rs | 39 +++++++++++++++++++++++++++++++++++++++ vm/src/obj/objint.rs | 6 +++--- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bools.py b/tests/snippets/bools.py index 0c277143b..b781668b6 100644 --- a/tests/snippets/bools.py +++ b/tests/snippets/bools.py @@ -51,3 +51,15 @@ assert True > 0 assert int(True) == 1 assert True.conjugate() == 1 assert isinstance(True.conjugate(), int) + +# Boolean operations on pairs of Bools should return Bools, not ints +assert (False | True) is True +assert (False & True) is False +assert (False ^ True) is True +# But only if both are Bools +assert (False | 1) is not True +assert (0 | True) is not True +assert (False & 1) is not False +assert (0 & True) is not False +assert (False ^ 1) is not True +assert (0 ^ True) is not True diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index 724c8bc5a..e96324f67 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -42,6 +42,9 @@ The class bool is a subclass of the class int, and cannot be subclassed."; extend_class!(context, bool_type, { "__new__" => context.new_rustfunc(bool_new), "__repr__" => context.new_rustfunc(bool_repr), + "__or__" => context.new_rustfunc(bool_or), + "__and__" => context.new_rustfunc(bool_and), + "__xor__" => context.new_rustfunc(bool_xor), "__doc__" => context.new_str(bool_doc.to_string()) }); } @@ -71,6 +74,42 @@ fn bool_repr(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + + if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs || rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().or(rhs.clone(), vm)) + } +} + +fn bool_and(vm: &VirtualMachine, args: PyFuncArgs) -> Result { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + + if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs && rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().and(rhs.clone(), vm)) + } +} + +fn bool_xor(vm: &VirtualMachine, args: PyFuncArgs) -> Result { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + + if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs ^ rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().xor(rhs.clone(), vm)) + } +} + fn bool_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index fdb0459af..1b728a53f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -286,7 +286,7 @@ impl PyInt { } #[pymethod(name = "__xor__")] - fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) } else { @@ -300,7 +300,7 @@ impl PyInt { } #[pymethod(name = "__or__")] - fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) } else { @@ -309,7 +309,7 @@ impl PyInt { } #[pymethod(name = "__and__")] - fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + pub fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); vm.ctx.new_int((&self.value) & v2) From 6f66e8440dca8f96c1fb45723e8c4342b209276a Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 01:41:52 +0300 Subject: [PATCH 448/658] Add tests for return type of bool.__or__ etc. --- tests/snippets/bools.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/snippets/bools.py b/tests/snippets/bools.py index b781668b6..23f22dce7 100644 --- a/tests/snippets/bools.py +++ b/tests/snippets/bools.py @@ -63,3 +63,17 @@ assert (False & 1) is not False assert (0 & True) is not False assert (False ^ 1) is not True assert (0 ^ True) is not True + +# Check that the same works with __XXX__ methods +assert False.__or__(0) is not False +assert False.__or__(False) is False +assert False.__ror__(0) is not False +assert False.__ror__(False) is False +assert False.__and__(0) is not False +assert False.__and__(False) is False +assert False.__rand__(0) is not False +assert False.__rand__(False) is False +assert False.__xor__(0) is not False +assert False.__xor__(False) is False +assert False.__rxor__(0) is not False +assert False.__rxor__(False) is False From a118497a4bede58ab01ced7b3428d9b007ede6f1 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 3 May 2019 02:55:51 +0300 Subject: [PATCH 449/658] Preallocate cloned strings with the minimum expected length Similar to 620bea78. --- vm/src/obj/objstr.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 22eda1f34..f1449f6dd 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -183,7 +183,12 @@ impl PyString { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); - let mut result = String::new(); + let capacity = if multiplier > 0 { + multiplier.to_usize().unwrap() * value.len() + } else { + 0 + }; + let mut result = String::with_capacity(capacity); for _x in 0..multiplier { result.push_str(value.as_str()); } @@ -206,7 +211,7 @@ impl PyString { } else { '\'' }; - let mut formatted = String::new(); + let mut formatted = String::with_capacity(value.len()); formatted.push(quote_char); for c in value.chars() { if c == quote_char || c == '\\' { @@ -799,7 +804,7 @@ impl PyString { #[pymethod] fn expandtabs(&self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); - let mut expanded_str = String::new(); + let mut expanded_str = String::with_capacity(self.value.len()); let mut tab_size = tab_stop; let mut col_count = 0 as usize; for ch in self.value.chars() { From f12268f79a7c94b2ee5864c43a07ce723187d690 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 3 May 2019 02:59:39 +0300 Subject: [PATCH 450/658] Implement str.__rmul__ --- vm/src/obj/objstr.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index f1449f6dd..e5e2afe37 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -198,6 +198,11 @@ impl PyString { } } + #[pymethod(name = "__rmul__")] + fn rmul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(val, vm) + } + #[pymethod(name = "__str__")] fn str(zelf: PyRef, _vm: &VirtualMachine) -> PyStringRef { zelf From d1730a53adb4ad44eb507944fe02bea3cda7ef23 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 3 May 2019 03:04:19 +0300 Subject: [PATCH 451/658] Add basic string multiplication tests --- tests/snippets/strings.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 0baa35d23..7cb1e8f30 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -29,6 +29,14 @@ assert repr('\n\t') == "'\\n\\t'" assert str(["a", "b", "can't"]) == "['a', 'b', \"can't\"]" +assert "xy" * 3 == "xyxyxy" +assert "x" * 0 == "" +assert "x" * -1 == "" + +assert 3 * "xy" == "xyxyxy" +assert 0 * "x" == "" +assert -1 * "x" == "" + a = 'Hallo' assert a.lower() == 'hallo' assert a.upper() == 'HALLO' From 29ec26917fb1078eb7314b41295be31cae963e35 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 03:08:42 +0300 Subject: [PATCH 452/658] Return last expression in compile Mode::Single --- vm/src/compile.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 76744f112..36b4c3ce7 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -130,18 +130,35 @@ impl Compiler { symbol_scope: SymbolScope, ) -> Result<(), CompileError> { self.scope_stack.push(symbol_scope); - for statement in &program.statements { + + let mut emitted_return = false; + + for (i, statement) in program.statements.iter().enumerate() { + let is_last = i == program.statements.len() - 1; + if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; - self.emit(Instruction::PrintExpr); + + if is_last { + self.emit(Instruction::Duplicate); + self.emit(Instruction::PrintExpr); + self.emit(Instruction::ReturnValue); + emitted_return = true; + } else { + self.emit(Instruction::PrintExpr); + } } else { self.compile_statement(&statement)?; } } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); - self.emit(Instruction::ReturnValue); + + if !emitted_return { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::None, + }); + self.emit(Instruction::ReturnValue); + } + Ok(()) } From 821959b889d81956e4a6f5ad15bef15727a41431 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 03:09:29 +0300 Subject: [PATCH 453/658] Save REPL return value as _ --- src/main.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 5620d851e..3e77ddbc2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,9 +147,23 @@ fn test_run_script() { fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> { match compile::compile(vm, source, &compile::Mode::Single, "".to_string()) { Ok(code) => { - if let Err(err) = vm.run_code_obj(code, scope) { - print_exception(vm, &err); + match vm.run_code_obj(code, scope.clone()) { + Ok(value) => { + // Save non-None values as "_" + + use rustpython_vm::pyobject::{IdProtocol, IntoPyObject, ItemProtocol}; + + if !value.is(&vm.get_none()) { + let key = objstr::PyString::from("_").into_pyobject(vm); + scope.globals.set_item(key, value, vm).unwrap(); + } + } + + Err(err) => { + print_exception(vm, &err); + } } + Ok(()) } // Don't inject syntax errors for line continuation From 857a7014c2fc341f91b6f86dabecc76056722c25 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 3 May 2019 03:05:27 +0300 Subject: [PATCH 454/658] Enable and extend strings/membership tests --- tests/snippets/membership.py | 15 ++++++++------- tests/snippets/strings.py | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/snippets/membership.py b/tests/snippets/membership.py index a944c4539..2987c3c0f 100644 --- a/tests/snippets/membership.py +++ b/tests/snippets/membership.py @@ -12,9 +12,8 @@ assert "foo" in "foobar" assert "whatever" not in "foobar" # test bytes -# TODO: uncomment this when bytes are implemented -# assert b"foo" in b"foobar" -# assert b"whatever" not in b"foobar" +assert b"foo" in b"foobar" +assert b"whatever" not in b"foobar" assert b"1" < b"2" assert b"1" <= b"2" assert b"5" <= b"5" @@ -32,18 +31,20 @@ assert 1 in set([1, 2]) assert 3 not in set([1, 2]) # test dicts -# TODO: test dicts when keys other than strings will be allowed assert "a" in {"a": 0, "b": 0} assert "c" not in {"a": 0, "b": 0} +assert 1 in {1: 5, 7: 12} +assert 5 not in {9: 10, 50: 100} +assert True in {True: 5} +assert False not in {True: 5} # test iter assert 3 in iter([1, 2, 3]) assert 3 not in iter([1, 2]) # test sequence -# TODO: uncomment this when ranges are usable -# assert 1 in range(0, 2) -# assert 3 not in range(0, 2) +assert 1 in range(0, 2) +assert 3 not in range(0, 2) # test __contains__ in user objects class MyNotContainingClass(): diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 7cb1e8f30..28df3f47f 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -102,6 +102,7 @@ assert ["kok".center(i, "|") for i in range(2, 10)] == [ ] +# requires CPython 3.7, and the CI currently runs with 3.6 # assert c.isascii() assert c.index('a') == 1 assert c.rindex('l') == 3 From d121b422b6c80fbb8f9cf50da4bbe71edf98f64a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 3 May 2019 12:14:47 +0300 Subject: [PATCH 455/658] Add browser.load_module --- vm/src/import.rs | 44 ++++++++++++++++++------------ wasm/lib/src/browser_module.rs | 49 ++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/vm/src/import.rs b/vm/src/import.rs index 45c7cd86a..f4c52a11e 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -31,29 +31,39 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; - let code_obj = compile::compile( + + import_file( vm, - &source, - &compile::Mode::Exec, + module_name, file_path.to_str().unwrap().to_string(), + source, ) - .map_err(|err| vm.new_syntax_error(&err))?; - // trace!("Code object: {:?}", code_obj); - - let attrs = vm.ctx.new_dict(); - attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; - let module = vm.ctx.new_module(module_name, attrs.clone()); - - // Store module in cache to prevent infinite loop with mutual importing libs: - sys_modules.set_item(module_name, module.clone(), vm)?; - - // Execute main code in module: - vm.run_code_obj(code_obj, Scope::new(None, attrs))?; - - Ok(module) } } +pub fn import_file( + vm: &VirtualMachine, + module_name: &str, + file_path: String, + content: String, +) -> PyResult { + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + let code_obj = compile::compile(vm, &content, &compile::Mode::Exec, file_path) + .map_err(|err| vm.new_syntax_error(&err))?; + // trace!("Code object: {:?}", code_obj); + + let attrs = vm.ctx.new_dict(); + attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; + let module = vm.ctx.new_module(module_name, attrs.clone()); + + // Store module in cache to prevent infinite loop with mutual importing libs: + sys_modules.set_item(module_name, module.clone(), vm)?; + + // Execute main code in module: + vm.run_code_obj(code_obj, Scope::new(None, attrs))?; + Ok(module) +} + fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result { let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap(); let mut paths: Vec = objsequence::get_elements(&sys_path) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 6fb7c6baa..b4c3fd193 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -6,6 +6,7 @@ use wasm_bindgen::JsCast; use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; +use rustpython_vm::import::import_file; use rustpython_vm::obj::{ objdict::PyDictRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; @@ -206,8 +207,12 @@ impl PyPromise { let vm = &stored_vm.vm; let ret = match res { Ok(val) => { - let val = convert::js_to_py(vm, val); - vm.invoke(on_fulfill.into_object(), PyFuncArgs::new(vec![val], vec![])) + let args = if val.is_null() { + vec![] + } else { + vec![convert::js_to_py(vm, val)] + }; + vm.invoke(on_fulfill.into_object(), PyFuncArgs::new(args, vec![])) } Err(err) => { if let OptionalArg::Present(on_reject) = on_reject { @@ -342,6 +347,45 @@ fn browser_prompt( Ok(result) } +fn browser_load_module(module: PyStringRef, path: PyStringRef, vm: &VirtualMachine) -> PyResult { + let weak_vm = weak_vm(vm); + + let mut opts = web_sys::RequestInit::new(); + opts.method("GET"); + + let request = web_sys::Request::new_with_str_and_init(path.as_str(), &opts) + .map_err(|err| convert::js_py_typeerror(vm, err))?; + + let window = window(); + let request_prom = window.fetch_with_request(&request); + + let future = JsFuture::from(request_prom) + .and_then(move |val| { + let response = val + .dyn_into::() + .expect("val to be of type Response"); + response.text() + }) + .and_then(move |text_value: Promise| { + // Convert this other `Promise` into a rust `Future`. + JsFuture::from(text_value) + }) + .and_then(move |text| { + let stored_vm = &weak_vm + .upgrade() + .expect("that the vm is valid when the promise resolves"); + let vm = &stored_vm.vm; + let resp_text = text.as_string().unwrap(); + let res = import_file(vm, module.as_str(), "WEB".to_string(), resp_text); + match res { + Ok(_) => Ok(JsValue::null()), + Err(err) => Err(convert::py_err_to_js_err(vm, &err)), + } + }); + + Ok(PyPromise::from_future(future).into_ref(vm).into_object()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -370,6 +414,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "alert" => ctx.new_rustfunc(browser_alert), "confirm" => ctx.new_rustfunc(browser_confirm), "prompt" => ctx.new_rustfunc(browser_prompt), + "load_module" => ctx.new_rustfunc(browser_load_module), }) } From 9448254914d7202c846ac6fa00a300fafdc5aea0 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:11:14 +0900 Subject: [PATCH 456/658] PyComplex uses extend_class for __new__ and __doc__ --- vm/src/obj/objcomplex.rs | 54 ++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index f9ff9d9b0..ec7afd110 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -9,6 +9,9 @@ use super::objfloat::{self, PyFloat}; use super::objint; use super::objtype::{self, PyClassRef}; +/// Create a complex number from a real part and an optional imaginary part. +/// +/// This is equivalent to (real + imag*1j) where imag defaults to 0. #[pyclass(name = "complex")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyComplex { @@ -30,42 +33,12 @@ impl From for PyComplex { pub fn init(context: &PyContext) { PyComplex::extend_class(context, &context.complex_type); - let complex_doc = - "Create a complex number from a real part and an optional imaginary part.\n\n\ - This is equivalent to (real + imag*1j) where imag defaults to 0."; - - extend_class!(context, &context.complex_type, { - "__doc__" => context.new_str(complex_doc.to_string()), - "__new__" => context.new_rustfunc(PyComplexRef::new), - }); } pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -impl PyComplexRef { - fn new( - cls: PyClassRef, - real: OptionalArg, - imag: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let real = match real { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; - - let imag = match imag { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; - - let value = Complex64::new(real, imag); - PyComplex { value }.into_ref_with_type(vm, cls) - } -} - fn to_complex(value: PyObjectRef, vm: &VirtualMachine) -> PyResult> { if objtype::isinstance(&value, &vm.ctx.complex_type()) { Ok(Some(get_value(&value))) @@ -204,4 +177,25 @@ impl PyComplex { fn bool(&self, _vm: &VirtualMachine) -> bool { self.value != Complex64::zero() } + + #[pymethod(name = "__new__")] + fn complex_new( + cls: PyClassRef, + real: OptionalArg, + imag: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let real = match real { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; + + let imag = match imag { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; + + let value = Complex64::new(real, imag); + PyComplex { value }.into_ref_with_type(vm, cls) + } } From 7b438d9be8e5b4a821e80e0d1494e678233c892c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:51:01 +0900 Subject: [PATCH 457/658] impl IntoPyObject for Complex64 --- vm/src/obj/objcomplex.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index ec7afd110..ab258064a 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -2,7 +2,9 @@ use num_complex::Complex64; use num_traits::{ToPrimitive, Zero}; use crate::function::OptionalArg; -use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; use crate::vm::VirtualMachine; use super::objfloat::{self, PyFloat}; @@ -25,6 +27,12 @@ impl PyValue for PyComplex { } } +impl IntoPyObject for Complex64 { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_complex(self)) + } +} + impl From for PyComplex { fn from(value: Complex64) -> Self { PyComplex { value } @@ -114,8 +122,8 @@ impl PyComplex { } #[pymethod(name = "conjugate")] - fn conjugate(&self, _vm: &VirtualMachine) -> PyComplex { - self.value.conj().into() + fn conjugate(&self, _vm: &VirtualMachine) -> Complex64 { + self.value.conj() } #[pymethod(name = "__eq__")] @@ -148,19 +156,15 @@ impl PyComplex { #[pymethod(name = "__mul__")] fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match to_complex(other, vm) { - Ok(Some(other)) => Ok(vm.ctx.new_complex(Complex64::new( - self.value.re * other.re - self.value.im * other.im, - self.value.re * other.im + self.value.im * other.re, - ))), - Ok(None) => Ok(vm.ctx.not_implemented()), - Err(err) => Err(err), - } + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value * other).into_pyobject(vm), + ) } #[pymethod(name = "__neg__")] - fn neg(&self, _vm: &VirtualMachine) -> PyComplex { - PyComplex::from(-self.value) + fn neg(&self, _vm: &VirtualMachine) -> Complex64 { + -self.value } #[pymethod(name = "__repr__")] From 9a7fadcb6c457b1704f1e357991f485b0b7eb6d8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 01:52:42 +0900 Subject: [PATCH 458/658] Refactor PyComplex using try_complex --- vm/src/obj/objcomplex.rs | 57 ++++++++++++++-------------------------- vm/src/obj/objfloat.rs | 2 +- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index ab258064a..eaee63259 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -47,20 +47,14 @@ pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -fn to_complex(value: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - if objtype::isinstance(&value, &vm.ctx.complex_type()) { - Ok(Some(get_value(&value))) - } else if objtype::isinstance(&value, &vm.ctx.int_type()) { - match objint::get_value(&value).to_f64() { - Some(v) => Ok(Some(Complex64::new(v, 0.0))), - None => Err(vm.new_overflow_error("int too large to convert to float".to_string())), - } - } else if objtype::isinstance(&value, &vm.ctx.float_type()) { - let v = objfloat::get_value(&value); - Ok(Some(Complex64::new(v, 0.0))) +fn try_complex(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { + Ok(if objtype::isinstance(&value, &vm.ctx.complex_type()) { + Some(get_value(&value)) + } else if let Some(float) = objfloat::try_float(value, vm)? { + Some(Complex64::new(float, 0.0)) } else { - Ok(None) - } + None + }) } #[pyimpl] @@ -83,42 +77,31 @@ impl PyComplex { #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&other, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(self.value + get_value(&other))) - } else { - self.radd(other, vm) - } + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value + other).into_pyobject(vm), + ) } #[pymethod(name = "__radd__")] fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match to_complex(other, vm) { - Ok(Some(other)) => Ok(vm.ctx.new_complex(self.value + other)), - Ok(None) => Ok(vm.ctx.not_implemented()), - Err(err) => Err(err), - } + self.add(other, vm) } #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&other, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(self.value - get_value(&other))) - } else { - match to_complex(other, vm) { - Ok(Some(other)) => Ok(vm.ctx.new_complex(self.value - other)), - Ok(None) => Ok(vm.ctx.not_implemented()), - Err(err) => Err(err), - } - } + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value - other).into_pyobject(vm), + ) } #[pymethod(name = "__rsub__")] fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match to_complex(other, vm) { - Ok(Some(other)) => Ok(vm.ctx.new_complex(other - self.value)), - Ok(None) => Ok(vm.ctx.not_implemented()), - Err(err) => Err(err), - } + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other - self.value).into_pyobject(vm), + ) } #[pymethod(name = "conjugate")] diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index de4ee9ff5..b52a469e1 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -44,7 +44,7 @@ impl From for PyFloat { } } -fn try_float(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { +pub fn try_float(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { Ok(if objtype::isinstance(&value, &vm.ctx.float_type()) { Some(get_value(&value)) } else if objtype::isinstance(&value, &vm.ctx.int_type()) { From 61de5f2efc7f30d2f968638ab6848943f061d831 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:03:28 +0900 Subject: [PATCH 459/658] complex.__eq__ using try_float --- tests/snippets/builtin_complex.py | 8 +++++--- vm/src/obj/objcomplex.rs | 16 ++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 4d0fa1202..1c0e78b4e 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -23,6 +23,7 @@ assert bool(complex(1, 0)) assert complex(1, 2) != complex(1, 1) assert complex(1, 2) != 'foo' assert complex(1, 2).__eq__('foo') == NotImplemented +assert 1j != 10 ** 1000 # __mul__ @@ -40,18 +41,19 @@ assert bool(complex(0, 0)) is False assert bool(complex(0, 1)) is True assert bool(complex(1, 0)) is True -# real +# numbers.Complex a = complex(3, 4) b = 4j assert a.real == 3 assert b.real == 0 -# imag - assert a.imag == 4 assert b.imag == 4 +assert a.conjugate() == 3 - 4j +assert b.conjugate() == -4j + # int and complex addition assert 1 + 1j == complex(1, 1) assert 1j + 1 == complex(1, 1) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index eaee63259..6f13e76fa 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,5 +1,5 @@ use num_complex::Complex64; -use num_traits::{ToPrimitive, Zero}; +use num_traits::Zero; use crate::function::OptionalArg; use crate::pyobject::{ @@ -8,7 +8,6 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use super::objfloat::{self, PyFloat}; -use super::objint; use super::objtype::{self, PyClassRef}; /// Create a complex number from a real part and an optional imaginary part. @@ -113,15 +112,12 @@ impl PyComplex { fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { self.value == get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - match objint::get_value(&other).to_f64() { - Some(f) => self.value.im == 0.0f64 && self.value.re == f, - None => false, - } - } else if objtype::isinstance(&other, &vm.ctx.float_type()) { - self.value.im == 0.0 && self.value.re == objfloat::get_value(&other) } else { - return vm.ctx.not_implemented(); + match objfloat::try_float(&other, vm) { + Ok(Some(other)) => self.value.im == 0.0f64 && self.value.re == other, + Err(_) => false, + Ok(None) => return vm.ctx.not_implemented(), + } }; vm.ctx.new_bool(result) From 982bbd69d803b288dc052c1b0e872d11d2628294 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:07:50 +0900 Subject: [PATCH 460/658] complex.__bool__ uses Zero::is_zero instead of zero() --- vm/src/obj/objcomplex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 6f13e76fa..9c38650b0 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -158,7 +158,7 @@ impl PyComplex { #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { - self.value != Complex64::zero() + !Complex64::is_zero(&self.value) } #[pymethod(name = "__new__")] From 2a2d0e47648fef006303a4a629a41861bd6c465d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:11:13 +0900 Subject: [PATCH 461/658] Add complex.__rmul__ --- tests/snippets/builtin_complex.py | 1 + vm/src/obj/objcomplex.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 1c0e78b4e..20300c4e7 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -29,6 +29,7 @@ assert 1j != 10 ** 1000 assert complex(2, -3) * complex(-5, 7) == complex(11, 29) assert complex(2, -3) * 5 == complex(10, -15) +assert 5 * complex(2, -3) == complex(2, -3) * 5 # __neg__ diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 9c38650b0..83dd4ebf1 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -141,6 +141,11 @@ impl PyComplex { ) } + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(other, vm) + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> Complex64 { -self.value From 7c8880fb4ad5d30e42bca3272f1eb04414353ec1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:27:50 +0900 Subject: [PATCH 462/658] complex overflow message test --- tests/snippets/builtin_complex.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 20300c4e7..a3a911fa7 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -1,4 +1,4 @@ -from testutils import assertRaises +from testutils import assert_raises # __abs__ @@ -73,19 +73,13 @@ assert 1j - 1 == complex(-1, 1) assert 2j - 1j == complex(0, 1) # type error addition -with assertRaises(TypeError): - assert 1j + 'str' -with assertRaises(TypeError): - assert 1j - 'str' -with assertRaises(TypeError): - assert 'str' + 1j -with assertRaises(TypeError): - assert 'str' - 1j +assert_raises(TypeError, lambda: 1j + 'str') +assert_raises(TypeError, lambda: 1j - 'str') +assert_raises(TypeError, lambda: 'str' + 1j) +assert_raises(TypeError, lambda: 'str' - 1j) # overflow -with assertRaises(OverflowError): - complex(10 ** 1000, 0) -with assertRaises(OverflowError): - complex(0, 10 ** 1000) -with assertRaises(OverflowError): - complex(0, 0) + 10 ** 1000 +msg = 'int too large to convert to float' +assert_raises(OverflowError, lambda: complex(10 ** 1000, 0), msg) +assert_raises(OverflowError, lambda: complex(0, 10 ** 1000), msg) +assert_raises(OverflowError, lambda: 0j + 10 ** 1000, msg) From 9523baf5ac4c6f8e242aba9b624440082989de18 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:35:16 +0900 Subject: [PATCH 463/658] complex [r]truediv, [r]floordiv --- tests/snippets/builtin_complex.py | 18 +++++++++++++++++- vm/src/obj/objcomplex.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index a3a911fa7..5c28a6b11 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -25,12 +25,28 @@ assert complex(1, 2) != 'foo' assert complex(1, 2).__eq__('foo') == NotImplemented assert 1j != 10 ** 1000 -# __mul__ +# __mul__, __rmul__ assert complex(2, -3) * complex(-5, 7) == complex(11, 29) assert complex(2, -3) * 5 == complex(10, -15) assert 5 * complex(2, -3) == complex(2, -3) * 5 +# __truediv__, __rtruediv__ + +assert complex(2, -3) / 2 == complex(1, -1.5) +assert 5 / complex(3, -4) == complex(0.6, 0.8) + +# __floordiv__, __rfloordiv__ + +assert_raises( + TypeError, + lambda: complex(2, -3) // 2, + "can't take floor of complex number.") +assert_raises( + TypeError, + lambda: 2 // complex(2, -3), + "can't take floor of complex number.") + # __neg__ assert -complex(1, -1) == complex(-1, 1) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 83dd4ebf1..c0392a3ab 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -146,6 +146,32 @@ impl PyComplex { self.mul(other, vm) } + #[pymethod(name = "__truediv__")] + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value / other).into_pyobject(vm), + ) + } + + #[pymethod(name = "__rtruediv__")] + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other / self.value).into_pyobject(vm), + ) + } + + #[pymethod(name = "__floordiv__")] + fn floordiv(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't take floor of complex number.".to_string())) + } + + #[pymethod(name = "__rfloordiv__")] + fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.floordiv(other, vm) + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> Complex64 { -self.value From 930c8eef503a4b7254759c19a09a2982e944a96d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 02:57:23 +0900 Subject: [PATCH 464/658] Add complex.{__mod__, __rmod__, __divmod__, __rdivmod__} --- tests/snippets/builtin_complex.py | 22 ++++++++++++++++++++++ vm/src/obj/objcomplex.rs | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 5c28a6b11..81cc9186e 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -36,6 +36,17 @@ assert 5 * complex(2, -3) == complex(2, -3) * 5 assert complex(2, -3) / 2 == complex(1, -1.5) assert 5 / complex(3, -4) == complex(0.6, 0.8) +# __mod__, __rmod__ + +assert_raises( + TypeError, + lambda: complex(2, -3) % 2, + "can't mod complex numbers.") +assert_raises( + TypeError, + lambda: 2 % complex(2, -3), + "can't mod complex numbers.") + # __floordiv__, __rfloordiv__ assert_raises( @@ -47,6 +58,17 @@ assert_raises( lambda: 2 // complex(2, -3), "can't take floor of complex number.") +# __divmod__, __rdivmod__ + +assert_raises( + TypeError, + lambda: divmod(complex(2, -3), 2), + "can't take floor or mod of complex number.") +assert_raises( + TypeError, + lambda: divmod(2, complex(2, -3)), + "can't take floor or mod of complex number.") + # __neg__ assert -complex(1, -1) == complex(-1, 1) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index c0392a3ab..3343b5dc7 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -162,6 +162,16 @@ impl PyComplex { ) } + #[pymethod(name = "__mod__")] + fn mod_(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't mod complex numbers.".to_string())) + } + + #[pymethod(name = "__rmod__")] + fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mod_(other, vm) + } + #[pymethod(name = "__floordiv__")] fn floordiv(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("can't take floor of complex number.".to_string())) @@ -172,6 +182,16 @@ impl PyComplex { self.floordiv(other, vm) } + #[pymethod(name = "__divmod__")] + fn divmod(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't take floor or mod of complex number.".to_string())) + } + + #[pymethod(name = "__rdivmod__")] + fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.divmod(other, vm) + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> Complex64 { -self.value From 88e64adc561b4bac6b34f07b01f2c26252f25f9e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 05:43:31 +0900 Subject: [PATCH 465/658] Add complex __pow__ and __rpow__ --- tests/snippets/builtin_complex.py | 6 ++++++ vm/src/obj/objcomplex.rs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 81cc9186e..8d883a9dd 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -69,6 +69,12 @@ assert_raises( lambda: divmod(2, complex(2, -3)), "can't take floor or mod of complex number.") +# __pow__, __rpow__ + +# assert 1j ** 2 == -1 +assert complex(1) ** 2 == 1 +assert 2 ** complex(2) == 4 + # __neg__ assert -complex(1, -1) == complex(-1, 1) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 3343b5dc7..677763213 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -207,6 +207,22 @@ impl PyComplex { } } + #[pymethod(name = "__pow__")] + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value.powc(other)).into_pyobject(vm), + ) + } + + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other.powc(self.value)).into_pyobject(vm), + ) + } + #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { !Complex64::is_zero(&self.value) From c57fd1b96e218c7dae2cfa6134ecdb53c509d549 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 3 May 2019 23:52:02 +0300 Subject: [PATCH 466/658] Remove closure --- wasm/lib/src/browser_module.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index b4c3fd193..b3d689d3f 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -366,10 +366,7 @@ fn browser_load_module(module: PyStringRef, path: PyStringRef, vm: &VirtualMachi .expect("val to be of type Response"); response.text() }) - .and_then(move |text_value: Promise| { - // Convert this other `Promise` into a rust `Future`. - JsFuture::from(text_value) - }) + .and_then(JsFuture::from) .and_then(move |text| { let stored_vm = &weak_vm .upgrade() From 5864c6e063ab2d2f8f5378029def87a5c242e496 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 4 May 2019 01:14:08 +0200 Subject: [PATCH 467/658] rewrite pybytearray with pybyteinner --- tests/snippets/bytearray.py | 384 +++++++++++++++++++++++--- tests/snippets/bytes.py | 3 +- vm/src/obj/objbytearray.rs | 536 +++++++++++++++++++++--------------- vm/src/obj/objbyteinner.rs | 55 ++-- vm/src/obj/objbytes.rs | 34 +-- vm/src/stdlib/io.rs | 8 +- 6 files changed, 713 insertions(+), 307 deletions(-) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 4286f8171..4f29fa187 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -1,53 +1,368 @@ -#__getitem__ not implemented yet -#a = bytearray(b'abc') -#assert a[0] == b'a' -#assert a[1] == b'b' +from testutils import assertRaises -assert len(bytearray([1,2,3])) == 3 +# new +assert bytearray([1, 2, 3]) +assert bytearray((1, 2, 3)) +assert bytearray(range(4)) +assert bytearray(3) +assert b"bla" +assert ( + bytearray("bla", "utf8") == bytearray("bla", encoding="utf-8") == bytearray(b"bla") +) +with assertRaises(TypeError): + bytearray("bla") +with assertRaises(TypeError): + bytearray("bla", encoding=b"jilj") -assert bytearray(b'1a23').isalnum() -assert not bytearray(b'1%a23').isalnum() +assert bytearray( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" +) == bytearray(range(0, 256)) +assert bytearray( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" +) == bytearray(range(0, 256)) +assert bytearray(b"omkmok\Xaa") == bytearray( + [111, 109, 107, 109, 111, 107, 92, 88, 97, 97] +) -assert bytearray(b'abc').isalpha() -assert not bytearray(b'abc1').isalpha() + +a = bytearray(b"abcd") +b = bytearray(b"ab") +c = bytearray(b"abcd") + + +# repr +assert repr(bytearray([0, 1, 2])) == repr(bytearray(b"\x00\x01\x02")) +assert ( + repr(bytearray([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255])) + == "bytearray(b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff')" +) +assert repr(bytearray(b"abcd")) == "bytearray(b'abcd')" + +# len +assert len(bytearray("abcdé", "utf8")) == 6 + +# comp +assert a == b"abcd" +assert a > b +assert a >= b +assert b < a +assert b <= a + +assert bytearray(b"foobar").__eq__(2) == NotImplemented +assert bytearray(b"foobar").__ne__(2) == NotImplemented +assert bytearray(b"foobar").__gt__(2) == NotImplemented +assert bytearray(b"foobar").__ge__(2) == NotImplemented +assert bytearray(b"foobar").__lt__(2) == NotImplemented +assert bytearray(b"foobar").__le__(2) == NotImplemented + +# # hash +with assertRaises(TypeError): + hash(bytearray(b"abcd")) # unashable + +# # iter +[i for i in bytearray(b"abcd")] == ["a", "b", "c", "d"] +assert list(bytearray(3)) == [0, 0, 0] + +# add +assert a + b == bytearray(b"abcdab") + +# contains +assert bytearray(b"ab") in bytearray(b"abcd") +assert bytearray(b"cd") in bytearray(b"abcd") +assert bytearray(b"abcd") in bytearray(b"abcd") +assert bytearray(b"a") in bytearray(b"abcd") +assert bytearray(b"d") in bytearray(b"abcd") +assert bytearray(b"dc") not in bytearray(b"abcd") +assert 97 in bytearray(b"abcd") +assert 150 not in bytearray(b"abcd") +with assertRaises(ValueError): + 350 in bytearray(b"abcd") + + +# getitem +d = bytearray(b"abcdefghij") + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == bytearray(b"cdef") +assert d[-6:] == bytearray(b"efghij") +assert d[1:8:2] == bytearray(b"bdfh") +assert d[8:1:-2] == bytearray(b"igec") + + +# # is_xx methods + +assert bytearray(b"1a23").isalnum() +assert not bytearray(b"1%a23").isalnum() + +assert bytearray(b"abc").isalpha() +assert not bytearray(b"abc1").isalpha() # travis doesn't like this -#assert bytearray(b'xyz').isascii() -#assert not bytearray([128, 157, 32]).isascii() +# assert bytearray(b'xyz').isascii() +# assert not bytearray([128, 157, 32]).isascii() -assert bytearray(b'1234567890').isdigit() -assert not bytearray(b'12ab').isdigit() +assert bytearray(b"1234567890").isdigit() +assert not bytearray(b"12ab").isdigit() + +l = bytearray(b"lower") +b = bytearray(b"UPPER") -l = bytearray(b'lower') assert l.islower() assert not l.isupper() -assert l.upper().isupper() -assert not bytearray(b'Super Friends').islower() +assert b.isupper() +assert not bytearray(b"Super Friends").islower() -assert bytearray(b' \n\t').isspace() -assert not bytearray(b'\td\n').isspace() +assert bytearray(b" \n\t").isspace() +assert not bytearray(b"\td\n").isspace() -b = bytearray(b'UPPER') assert b.isupper() assert not b.islower() -assert b.lower().islower() -assert not bytearray(b'tuPpEr').isupper() +assert l.islower() +assert not bytearray(b"tuPpEr").isupper() -assert bytearray(b'Is Title Case').istitle() -assert not bytearray(b'is Not title casE').istitle() +assert bytearray(b"Is Title Case").istitle() +assert not bytearray(b"is Not title casE").istitle() -a = bytearray(b'abcd') +# upper lower, capitalize, swapcase +l = bytearray(b"lower") +b = bytearray(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytearray().capitalize() == bytearray() +assert b"AaBbCc123'@/".swapcase().swapcase() == b"AaBbCc123'@/" +assert b"AaBbCc123'@/".swapcase() == b"aAbBcC123'@/" + +# # hex from hex +assert bytearray([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" +try: + bytearray.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytearray.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" +with assertRaises(TypeError): + bytearray.fromhex(b"hhjjk") +# center +assert [bytearray(b"koki").center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [bytearray(b"kok").center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +bytearray(b"kok").center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").center(2, "a") +with assertRaises(TypeError): + bytearray(b"b").center(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").center(b"ba") +assert bytearray(b"kok").center(5, bytearray(b"x")) == b"xkokx" +bytearray(b"kok").center(-5) == b"kok" + + +# ljust +assert [bytearray(b"koki").ljust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"koki|", + b"koki||", + b"koki|||", + b"koki||||", + b"koki|||||", +] +assert [bytearray(b"kok").ljust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"kok||", + b"kok|||", + b"kok||||", + b"kok|||||", + b"kok||||||", +] + +bytearray(b"kok").ljust(4) == b"kok " # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").ljust(2, "a") +with assertRaises(TypeError): + bytearray(b"b").ljust(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").ljust(b"ba") +assert bytearray(b"kok").ljust(5, bytearray(b"x")) == b"kokxx" +assert bytearray(b"kok").ljust(-5) == b"kok" + +# rjust +assert [bytearray(b"koki").rjust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"||koki", + b"|||koki", + b"||||koki", + b"|||||koki", +] +assert [bytearray(b"kok").rjust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"|kok", + b"||kok", + b"|||kok", + b"||||kok", + b"|||||kok", + b"||||||kok", +] + + +bytearray(b"kok").rjust(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").rjust(2, "a") +with assertRaises(TypeError): + bytearray(b"b").rjust(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").rjust(b"ba") +assert bytearray(b"kok").rjust(5, bytearray(b"x")) == b"xxkok" +assert bytearray(b"kok").rjust(-5) == b"kok" + + +# count +assert bytearray(b"azeazerazeazopia").count(b"aze") == 3 +assert bytearray(b"azeazerazeazopia").count(b"az") == 4 +assert bytearray(b"azeazerazeazopia").count(b"a") == 5 +assert bytearray(b"123456789").count(b"") == 10 +assert bytearray(b"azeazerazeazopia").count(bytearray(b"aze")) == 3 +assert bytearray(b"azeazerazeazopia").count(memoryview(b"aze")) == 3 +assert bytearray(b"azeazerazeazopia").count(memoryview(b"aze"), 1, 9) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, None) == 3 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2, None) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, 7) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, 7) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2, 7) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", -13, -10) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", 1, 10000) == 2 +with assertRaises(ValueError): + bytearray(b"ilj").count(3550) +assert bytearray(b"azeazerazeazopia").count(97) == 5 + +# join +assert bytearray(b"").join( + (b"jiljl", bytearray(b"kmoomk"), memoryview(b"aaaa")) +) == bytearray(b"jiljlkmoomkaaaa") +with assertRaises(TypeError): + bytearray(b"").join((b"km", "kl")) + + +# endswith startswith +assert bytearray(b"abcde").endswith(b"de") +assert bytearray(b"abcde").endswith(b"") +assert not bytearray(b"abcde").endswith(b"zx") +assert bytearray(b"abcde").endswith(b"bc", 0, 3) +assert not bytearray(b"abcde").endswith(b"bc", 2, 3) +assert bytearray(b"abcde").endswith((b"c", bytearray(b"de"))) + +assert bytearray(b"abcde").startswith(b"ab") +assert bytearray(b"abcde").startswith(b"") +assert not bytearray(b"abcde").startswith(b"zx") +assert bytearray(b"abcde").startswith(b"cd", 2) +assert not bytearray(b"abcde").startswith(b"cd", 1, 4) +assert bytearray(b"abcde").startswith((b"a", bytearray(b"bc"))) + + +# index find +assert bytearray(b"abcd").index(b"cd") == 2 +assert bytearray(b"abcd").index(b"cd", 0) == 2 +assert bytearray(b"abcd").index(b"cd", 1) == 2 +assert bytearray(b"abcd").index(99) == 2 +with assertRaises(ValueError): + bytearray(b"abcde").index(b"c", 3, 1) +with assertRaises(ValueError): + bytearray(b"abcd").index(b"cdaaaaa") +with assertRaises(ValueError): + bytearray(b"abcd").index(b"b", 3, 4) +with assertRaises(ValueError): + bytearray(b"abcd").index(1) + + +assert bytearray(b"abcd").find(b"cd") == 2 +assert bytearray(b"abcd").find(b"cd", 0) == 2 +assert bytearray(b"abcd").find(b"cd", 1) == 2 +assert bytearray(b"abcde").find(b"c", 3, 1) == -1 +assert bytearray(b"abcd").find(b"cdaaaaa") == -1 +assert bytearray(b"abcd").find(b"b", 3, 4) == -1 +assert bytearray(b"abcd").find(1) == -1 +assert bytearray(b"abcd").find(99) == 2 + +assert bytearray(b"abcdabcda").find(b"a") == 0 +assert bytearray(b"abcdabcda").rfind(b"a") == 8 +assert bytearray(b"abcdabcda").rfind(b"a", 2, 6) == 4 +assert bytearray(b"abcdabcda").rfind(b"a", None, 6) == 4 +assert bytearray(b"abcdabcda").rfind(b"a", 2, None) == 8 +assert bytearray(b"abcdabcda").index(b"a") == 0 +assert bytearray(b"abcdabcda").rindex(b"a") == 8 + + +# make trans +# fmt: off +assert ( + bytearray.maketrans(memoryview(b"abc"), bytearray(b"zzz")) + == bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 122, 122, 122, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]) +) +# fmt: on + +# translate +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")), bytearray(b"h") +) == bytearray(b"btuybyubuyb") +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")), bytearray(b"a") +) == bytearray(b"abatuybyubuyb") +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")) +) == bytearray(b"abatuybyubuyb") +assert bytearray(b"hjhtuyfjtyhuhjuyj").translate(None, bytearray(b"ht")) == bytearray( + b"juyfjyujuyj" +) +assert bytearray(b"hjhtuyfjtyhuhjuyj").translate(None, delete=b"ht") == bytearray( + b"juyfjyujuyj" +) + + +# strip lstrip rstrip +assert bytearray(b" spacious ").strip() == bytearray(b"spacious") +assert bytearray(b"www.example.com").strip(b"cmowz.") == bytearray(b"example") +assert bytearray(b" spacious ").lstrip() == bytearray(b"spacious ") +assert bytearray(b"www.example.com").lstrip(b"cmowz.") == bytearray(b"example.com") +assert bytearray(b" spacious ").rstrip() == bytearray(b" spacious") +assert bytearray(b"mississippi").rstrip(b"ipz") == bytearray(b"mississ") + + +# clear +a = bytearray(b"abcd") a.clear() assert len(a) == 0 -try: - bytearray([400]) -except ValueError: - pass -else: - assert False - -b = bytearray(b'test') +b = bytearray(b"test") assert len(b) == 4 b.pop() assert len(b) == 3 @@ -66,8 +381,11 @@ except IndexError: else: assert False -a = bytearray(b'appen') +a = bytearray(b"appen") assert len(a) == 5 a.append(100) +assert a == bytearray(b"append") assert len(a) == 6 assert a.pop() == 100 + +import bytes as bbytes diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2733533d8..7086a1423 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -175,7 +175,8 @@ with assertRaises(TypeError): with assertRaises(TypeError): b"b".center(b"ba") assert b"kok".center(5, bytearray(b"x")) == b"xkokx" -b"kok".center(-5) +b"kok".center(-5) == b"kok" + # ljust diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 6f78ecf48..433624339 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,32 +1,62 @@ //! Implementation of the python bytearray object. -use std::cell::{Cell, RefCell}; -use std::fmt::Write; -use std::ops::{Deref, DerefMut}; - -use num_traits::ToPrimitive; - use crate::function::OptionalArg; -use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objbyteinner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, + ByteInnerTranslateOptions, ByteOr, PyByteInner, +}; +use crate::obj::objint::PyIntRef; +use crate::obj::objslice::PySliceRef; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyobject::{ + Either, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, +}; use crate::vm::VirtualMachine; +use std::cell::{Cell, RefCell}; -use super::objint; use super::objiter; use super::objtype::PyClassRef; -#[derive(Debug)] +/// "bytearray(iterable_of_ints) -> bytearray\n\ +/// bytearray(string, encoding[, errors]) -> bytearray\n\ +/// bytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\n\ +/// bytearray(int) -> bytes array of size given by the parameter initialized with null bytes\n\ +/// bytearray() -> empty bytes array\n\n\ +/// Construct a mutable bytearray object from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - a bytes or a buffer object\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytearray")] +#[derive(Clone, Debug)] pub struct PyByteArray { - // TODO: shouldn't be public - pub value: RefCell>, + pub inner: RefCell, } type PyByteArrayRef = PyRef; impl PyByteArray { pub fn new(data: Vec) -> Self { PyByteArray { - value: RefCell::new(data), + inner: RefCell::new(PyByteInner { elements: data }), } } + + pub fn from_inner(inner: PyByteInner) -> Self { + PyByteArray { + inner: RefCell::new(inner), + } + } + + // pub fn get_value(&self) -> Vec { + // self.inner.borrow().clone().elements + // } + + // pub fn get_value_mut(&self) -> Vec { + // self.inner.borrow_mut().clone().elements + // } } impl PyValue for PyByteArray { @@ -35,254 +65,318 @@ impl PyValue for PyByteArray { } } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - obj.payload::().unwrap().value.borrow() -} +// pub fn get_value(obj: &PyObjectRef) -> Vec { +// obj.payload::().unwrap().get_value() +// } -pub fn get_mut_value<'a>(obj: &'a PyObjectRef) -> impl DerefMut> + 'a { - obj.payload::().unwrap().value.borrow_mut() -} - -// Binary data support +// pub fn get_value_mut(obj: &PyObjectRef) -> Vec { +// obj.payload::().unwrap().get_value_mut() +// } /// Fill bytearray class methods dictionary. pub fn init(context: &PyContext) { + PyByteArrayRef::extend_class(context, &context.bytearray_type); let bytearray_type = &context.bytearray_type; - - let bytearray_doc = - "bytearray(iterable_of_ints) -> bytearray\n\ - bytearray(string, encoding[, errors]) -> bytearray\n\ - bytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\n\ - bytearray(int) -> bytes array of size given by the parameter initialized with null bytes\n\ - bytearray() -> empty bytes array\n\n\ - Construct a mutable bytearray object from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - a bytes or a buffer object\n \ - - any object implementing the buffer API.\n \ - - an integer"; - extend_class!(context, bytearray_type, { - "__doc__" => context.new_str(bytearray_doc.to_string()), - "__new__" => context.new_rustfunc(bytearray_new), - "__eq__" => context.new_rustfunc(PyByteArrayRef::eq), - "__len__" => context.new_rustfunc(PyByteArrayRef::len), - "__repr__" => context.new_rustfunc(PyByteArrayRef::repr), - "__iter__" => context.new_rustfunc(PyByteArrayRef::iter), - "clear" => context.new_rustfunc(PyByteArrayRef::clear), - "isalnum" => context.new_rustfunc(PyByteArrayRef::isalnum), - "isalpha" => context.new_rustfunc(PyByteArrayRef::isalpha), - "isascii" => context.new_rustfunc(PyByteArrayRef::isascii), - "isdigit" => context.new_rustfunc(PyByteArrayRef::isdigit), - "islower" => context.new_rustfunc(PyByteArrayRef::islower), - "isspace" => context.new_rustfunc(PyByteArrayRef::isspace), - "istitle" =>context.new_rustfunc(PyByteArrayRef::istitle), - "isupper" => context.new_rustfunc(PyByteArrayRef::isupper), - "lower" => context.new_rustfunc(PyByteArrayRef::lower), - "append" => context.new_rustfunc(PyByteArrayRef::append), - "pop" => context.new_rustfunc(PyByteArrayRef::pop), - "upper" => context.new_rustfunc(PyByteArrayRef::upper) + "fromhex" => context.new_rustfunc(PyByteArrayRef::fromhex), + "maketrans" => context.new_rustfunc(PyByteInner::maketrans), }); PyByteArrayIterator::extend_class(context, &context.bytearrayiterator_type); } -fn bytearray_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); - } - } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; - PyByteArray::new(value).into_ref_with_type(vm, cls.clone()) -} - +#[pyimpl] impl PyByteArrayRef { + #[pymethod(name = "__new__")] + fn bytearray_new( + cls: PyClassRef, + options: ByteInnerNewOptions, + vm: &VirtualMachine, + ) -> PyResult { + PyByteArray::from_inner(options.get_value(vm)?).into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__repr__")] + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("bytearray(b'{}')", self.inner.borrow().repr()?))) + } + + #[pymethod(name = "__len__")] fn len(self, _vm: &VirtualMachine) -> usize { - self.value.borrow().len() + self.inner.borrow().len() } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx - .new_bool(self.value.borrow().as_slice() == other.value.borrow().as_slice()) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().eq(other, vm) } - fn isalnum(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphanumeric()) + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().ge(other, vm) } - fn isalpha(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphabetic()) + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().le(other, vm) } - fn isascii(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_ascii()) + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().gt(other, vm) } - fn isdigit(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_digit(10)) + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().lt(other, vm) } - fn islower(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() - && bytes - .iter() - .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_lowercase()) - } - - fn isspace(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_whitespace()) - } - - fn isupper(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - !bytes.is_empty() - && bytes - .iter() - .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_uppercase()) - } - - fn istitle(self, _vm: &VirtualMachine) -> bool { - let bytes = self.value.borrow(); - if bytes.is_empty() { - return false; - } - - let mut iter = bytes.iter().peekable(); - let mut prev_cased = false; - - while let Some(c) = iter.next() { - let current = char::from(*c); - let next = if let Some(k) = iter.peek() { - char::from(**k) - } else if current.is_uppercase() { - return !prev_cased; - } else { - return prev_cased; - }; - - if (is_cased(current) && next.is_uppercase() && !prev_cased) - || (!is_cased(current) && next.is_lowercase()) - { - return false; - } - - prev_cased = is_cased(current); - } - - true - } - - fn repr(self, _vm: &VirtualMachine) -> String { - let bytes = self.value.borrow(); - let data = String::from_utf8(bytes.to_vec()).unwrap_or_else(|_| to_hex(&bytes.to_vec())); - format!("bytearray(b'{}')", data) - } - - fn clear(self, _vm: &VirtualMachine) { - self.value.borrow_mut().clear(); - } - - fn append(self, x: u8, _vm: &VirtualMachine) { - self.value.borrow_mut().push(x); - } - - fn pop(self, vm: &VirtualMachine) -> PyResult { - let mut bytes = self.value.borrow_mut(); - bytes - .pop() - .ok_or_else(|| vm.new_index_error("pop from empty bytearray".to_string())) - } - - fn lower(self, _vm: &VirtualMachine) -> PyByteArray { - let bytes = self.value.borrow().clone().to_ascii_lowercase(); - PyByteArray { - value: RefCell::new(bytes), - } - } - - fn upper(self, _vm: &VirtualMachine) -> PyByteArray { - let bytes = self.value.borrow().clone().to_ascii_uppercase(); - PyByteArray { - value: RefCell::new(bytes), - } + #[pymethod(name = "__hash__")] + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type: bytearray".to_string())) } + #[pymethod(name = "__iter__")] fn iter(self, _vm: &VirtualMachine) -> PyByteArrayIterator { PyByteArrayIterator { position: Cell::new(0), bytearray: self, } } -} -// helper function for istitle -fn is_cased(c: char) -> bool { - c.to_uppercase().next().unwrap() != c || c.to_lowercase().next().unwrap() != c -} + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.ctx.new_bytearray(self.inner.borrow().add(other))) + } else { + Ok(vm.ctx.not_implemented()) + } + } -/* -fn getitem(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, Some(vm.ctx.bytearray_type())), (needle, None)] - ); - let elements = get_elements(obj); - get_item(vm, list, &, needle.clone()) -} -*/ -/* -fn set_value(obj: &PyObjectRef, value: Vec) { - obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; -} -*/ + #[pymethod(name = "__contains__")] + fn contains(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(needle, vm) + } -/// Return a lowercase hex representation of a bytearray -fn to_hex(bytearray: &[u8]) -> String { - bytearray.iter().fold(String::new(), |mut s, b| { - let _ = write!(s, "\\x{:02x}", b); - s - }) -} + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().getitem(needle, vm) + } -#[cfg(test)] -mod tests { - use super::*; + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isalnum(vm) + } - #[test] - fn bytearray_to_hex_formatting() { - assert_eq!(&to_hex(&[11u8, 222u8]), "\\x0b\\xde"); + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isalpha(vm) + } + + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isascii(vm) + } + + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isdigit(vm) + } + + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().islower(vm) + } + + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isspace(vm) + } + + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isupper(vm) + } + + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().istitle(vm) + } + + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().upper(vm))) + } + + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().capitalize(vm))) + } + + #[pymethod(name = "swapcase")] + fn swapcase(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().swapcase(vm))) + } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().hex(vm) + } + + fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(PyByteInner::fromhex(string.as_str(), vm)?)) + } + + #[pymethod(name = "center")] + fn center(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().center(options, vm)?)) + } + + #[pymethod(name = "ljust")] + fn ljust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().ljust(options, vm)?)) + } + + #[pymethod(name = "rjust")] + fn rjust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().rjust(options, vm)?)) + } + + #[pymethod(name = "count")] + fn count(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().count(options, vm) + } + + #[pymethod(name = "join")] + fn join(self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().join(iter, vm) + } + + #[pymethod(name = "endswith")] + fn endswith( + self, + suffix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner + .borrow() + .startsendswith(suffix, start, end, true, vm) + } + + #[pymethod(name = "startswith")] + fn startswith( + self, + prefix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner + .borrow() + .startsendswith(prefix, start, end, false, vm) + } + + #[pymethod(name = "find")] + fn find(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().find(options, false, vm) + } + + #[pymethod(name = "index")] + fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.borrow().find(options, false, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(res) + } + + #[pymethod(name = "rfind")] + fn rfind(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().find(options, true, vm) + } + + #[pymethod(name = "rindex")] + fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.borrow().find(options, true, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(res) + } + + #[pymethod(name = "translate")] + fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().translate(options, vm) + } + + #[pymethod(name = "strip")] + fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::All, vm)?, + )) + } + + #[pymethod(name = "lstrip")] + fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::Left, vm)?, + )) + } + + #[pymethod(name = "rstrip")] + fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::Right, vm)?, + )) + } + + #[pymethod(name = "clear")] + fn clear(self, _vm: &VirtualMachine) { + self.inner.borrow_mut().elements.clear(); + } + + #[pymethod(name = "append")] + fn append(self, x: PyIntRef, vm: &VirtualMachine) -> Result<(), PyObjectRef> { + self.inner + .borrow_mut() + .elements + .push(x.as_bigint().byte_or(vm)?); + Ok(()) + } + #[pymethod(name = "pop")] + fn pop(self, vm: &VirtualMachine) -> PyResult { + let bytes = &mut self.inner.borrow_mut().elements; + bytes + .pop() + .ok_or_else(|| vm.new_index_error("pop from empty bytearray".to_string())) } } +// fn set_value(obj: &PyObjectRef, value: Vec) { +// obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; +// } + #[pyclass] #[derive(Debug)] pub struct PyByteArrayIterator { @@ -300,8 +394,8 @@ impl PyValue for PyByteArrayIterator { impl PyByteArrayIterator { #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { - if self.position.get() < self.bytearray.value.borrow().len() { - let ret = self.bytearray.value.borrow()[self.position.get()]; + if self.position.get() < self.bytearray.inner.borrow().len() { + let ret = self.bytearray.inner.borrow().elements[self.position.get()]; self.position.set(self.position.get() + 1); Ok(ret) } else { diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index b75f8be69..9e3473b93 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -27,7 +27,7 @@ use crate::obj::objint::PyInt; use num_integer::Integer; use num_traits::ToPrimitive; -use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; +use super::objbytearray::PyByteArray; use super::objbytes::PyBytes; use super::objmemory::PyMemoryView; @@ -43,7 +43,7 @@ impl TryFromObject for PyByteInner { match_class!(obj, i @ PyBytes => Ok(PyByteInner{elements: i.get_value().to_vec()}), - j @ PyByteArray => Ok(PyByteInner{elements: get_value_bytearray(&j.as_object()).to_vec()}), + j @ PyByteArray => Ok(PyByteInner{elements: j.inner.borrow().elements.to_vec()}), k @ PyMemoryView => Ok(PyByteInner{elements: k.get_obj_value().unwrap()}), obj => Err(vm.new_type_error(format!( "a bytes-like object is required, not {}", @@ -261,6 +261,7 @@ impl PyByteInner { 0..=8 => res.push_str(&format!("\\x0{}", i)), 9 => res.push_str("\\t"), 10 => res.push_str("\\n"), + 11 => res.push_str(&format!("\\x0{:x}", i)), 13 => res.push_str("\\r"), 32..=126 => res.push(*(i) as char), _ => res.push_str(&format!("\\x{:x}", i)), @@ -277,43 +278,43 @@ impl PyByteInner { self.elements.len() == 0 } - pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { - if self.elements == other.elements { - Ok(vm.new_bool(true)) + pub fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements == other.elements)) } else { - Ok(vm.new_bool(false)) + Ok(vm.ctx.not_implemented()) } } - pub fn ge(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { - if self.elements >= other.elements { - Ok(vm.new_bool(true)) + pub fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements >= other.elements)) } else { - Ok(vm.new_bool(false)) + Ok(vm.ctx.not_implemented()) } } - pub fn le(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { - if self.elements <= other.elements { - Ok(vm.new_bool(true)) + pub fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements <= other.elements)) } else { - Ok(vm.new_bool(false)) + Ok(vm.ctx.not_implemented()) } } - pub fn gt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { - if self.elements > other.elements { - Ok(vm.new_bool(true)) + pub fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements > other.elements)) } else { - Ok(vm.new_bool(false)) + Ok(vm.ctx.not_implemented()) } } - pub fn lt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { - if self.elements < other.elements { - Ok(vm.new_bool(true)) + pub fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements < other.elements)) } else { - Ok(vm.new_bool(false)) + Ok(vm.ctx.not_implemented()) } } @@ -323,14 +324,12 @@ impl PyByteInner { hasher.finish() as usize } - pub fn add(&self, other: &PyByteInner, _vm: &VirtualMachine) -> Vec { - let elements: Vec = self - .elements + pub fn add(&self, other: PyByteInner) -> Vec { + self.elements .iter() .chain(other.elements.iter()) .cloned() - .collect(); - elements + .collect::>() } pub fn contains(&self, needle: Either, vm: &VirtualMachine) -> PyResult { @@ -763,7 +762,7 @@ pub fn try_as_byte(obj: &PyObjectRef) -> Option> { match_class!(obj.clone(), i @ PyBytes => Some(i.get_value().to_vec()), - j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), + j @ PyByteArray => Some(j.inner.borrow().elements.to_vec()), _ => None) } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 59b0731f4..30ad6623e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -9,7 +9,9 @@ use core::cell::Cell; use std::ops::Deref; use crate::function::OptionalArg; -use crate::pyobject::{PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, +}; use super::objbyteinner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, @@ -100,35 +102,25 @@ impl PyBytesRef { #[pymethod(name = "__eq__")] fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => self.inner.eq(&bytes.inner, vm), - _ => Ok(vm.ctx.not_implemented())) + self.inner.eq(other, vm) } - #[pymethod(name = "__ge__")] fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => self.inner.ge(&bytes.inner, vm), - _ => Ok(vm.ctx.not_implemented())) + self.inner.ge(other, vm) } #[pymethod(name = "__le__")] fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => self.inner.le(&bytes.inner, vm), - _ => Ok(vm.ctx.not_implemented())) + self.inner.le(other, vm) } #[pymethod(name = "__gt__")] fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => self.inner.gt(&bytes.inner, vm), - _ => Ok(vm.ctx.not_implemented())) + self.inner.gt(other, vm) } #[pymethod(name = "__lt__")] fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), - _ => Ok(vm.ctx.not_implemented())) + self.inner.lt(other, vm) } + #[pymethod(name = "__hash__")] fn hash(self, _vm: &VirtualMachine) -> usize { self.inner.hash() @@ -144,9 +136,11 @@ impl PyBytesRef { #[pymethod(name = "__add__")] fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - bytes @ PyBytes => Ok(vm.ctx.new_bytes(self.inner.add(&bytes.inner, vm))), - _ => Ok(vm.ctx.not_implemented())) + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.ctx.new_bytearray(self.inner.add(other))) + } else { + Ok(vm.ctx.not_implemented()) + } } #[pymethod(name = "__contains__")] diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index b8e72cc15..e9b49ec44 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -124,7 +124,7 @@ fn buffered_reader_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { //Copy bytes from the buffer vector into the results vector if let Some(bytes) = buffer.payload::() { - result.extend_from_slice(&bytes.value.borrow()); + result.extend_from_slice(&bytes.inner.borrow().elements); }; let py_len = vm.call_method(&buffer, "__len__", PyFuncArgs::default())?; @@ -207,9 +207,9 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(bytes) = obj.payload::() { //TODO: Implement for MemoryView - let mut value_mut = bytes.value.borrow_mut(); + let value_mut = &mut bytes.inner.borrow_mut().elements; value_mut.clear(); - match f.read_to_end(&mut value_mut) { + match f.read_to_end(value_mut) { Ok(_) => {} Err(_) => return Err(vm.new_value_error("Error reading from Take".to_string())), } @@ -237,7 +237,7 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { match obj.payload::() { Some(bytes) => { - let value_mut = bytes.value.borrow(); + let value_mut = &mut bytes.inner.borrow_mut().elements; match handle.write(&value_mut[..]) { Ok(len) => { //reset raw fd on the FileIO object From 01b245f5df7cee71eb469545cdeb5406de07e5ab Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 10:59:32 +0300 Subject: [PATCH 468/658] Add stat to DirEntry --- vm/src/stdlib/os.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 2b6c859eb..fd5893e74 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -15,7 +15,9 @@ use crate::obj::objiter; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, +}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -230,6 +232,10 @@ impl DirEntryRef { .map_err(|s| vm.new_os_error(s.to_string()))? .is_file()) } + + fn stat(self, vm: &VirtualMachine) -> PyResult { + os_stat(self.path(vm).try_into_ref(vm)?, vm) + } } #[pyclass] @@ -428,6 +434,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "path" => ctx.new_property(DirEntryRef::path), "is_dir" => ctx.new_rustfunc(DirEntryRef::is_dir), "is_file" => ctx.new_rustfunc(DirEntryRef::is_file), + "stat" => ctx.new_rustfunc(DirEntryRef::stat), }); let stat_result = py_class!(ctx, "stat_result", ctx.object(), { From b96f4a575809ecaf2bc8c8699b2286ceec7b5498 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 11:00:36 +0300 Subject: [PATCH 469/658] Add CPython stat.py file --- Lib/stat.py | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 Lib/stat.py diff --git a/Lib/stat.py b/Lib/stat.py new file mode 100644 index 000000000..a9c678ec0 --- /dev/null +++ b/Lib/stat.py @@ -0,0 +1,179 @@ +"""Constants/functions for interpreting results of os.stat() and os.lstat(). + +Suggested usage: from stat import * +""" + +# Indices for stat struct members in the tuple returned by os.stat() + +ST_MODE = 0 +ST_INO = 1 +ST_DEV = 2 +ST_NLINK = 3 +ST_UID = 4 +ST_GID = 5 +ST_SIZE = 6 +ST_ATIME = 7 +ST_MTIME = 8 +ST_CTIME = 9 + +# Extract bits from the mode + +def S_IMODE(mode): + """Return the portion of the file's mode that can be set by + os.chmod(). + """ + return mode & 0o7777 + +def S_IFMT(mode): + """Return the portion of the file's mode that describes the + file type. + """ + return mode & 0o170000 + +# Constants used as S_IFMT() for various file types +# (not all are implemented on all systems) + +S_IFDIR = 0o040000 # directory +S_IFCHR = 0o020000 # character device +S_IFBLK = 0o060000 # block device +S_IFREG = 0o100000 # regular file +S_IFIFO = 0o010000 # fifo (named pipe) +S_IFLNK = 0o120000 # symbolic link +S_IFSOCK = 0o140000 # socket file + +# Functions to test for each file type + +def S_ISDIR(mode): + """Return True if mode is from a directory.""" + return S_IFMT(mode) == S_IFDIR + +def S_ISCHR(mode): + """Return True if mode is from a character special device file.""" + return S_IFMT(mode) == S_IFCHR + +def S_ISBLK(mode): + """Return True if mode is from a block special device file.""" + return S_IFMT(mode) == S_IFBLK + +def S_ISREG(mode): + """Return True if mode is from a regular file.""" + return S_IFMT(mode) == S_IFREG + +def S_ISFIFO(mode): + """Return True if mode is from a FIFO (named pipe).""" + return S_IFMT(mode) == S_IFIFO + +def S_ISLNK(mode): + """Return True if mode is from a symbolic link.""" + return S_IFMT(mode) == S_IFLNK + +def S_ISSOCK(mode): + """Return True if mode is from a socket.""" + return S_IFMT(mode) == S_IFSOCK + +# Names for permission bits + +S_ISUID = 0o4000 # set UID bit +S_ISGID = 0o2000 # set GID bit +S_ENFMT = S_ISGID # file locking enforcement +S_ISVTX = 0o1000 # sticky bit +S_IREAD = 0o0400 # Unix V7 synonym for S_IRUSR +S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR +S_IEXEC = 0o0100 # Unix V7 synonym for S_IXUSR +S_IRWXU = 0o0700 # mask for owner permissions +S_IRUSR = 0o0400 # read by owner +S_IWUSR = 0o0200 # write by owner +S_IXUSR = 0o0100 # execute by owner +S_IRWXG = 0o0070 # mask for group permissions +S_IRGRP = 0o0040 # read by group +S_IWGRP = 0o0020 # write by group +S_IXGRP = 0o0010 # execute by group +S_IRWXO = 0o0007 # mask for others (not in group) permissions +S_IROTH = 0o0004 # read by others +S_IWOTH = 0o0002 # write by others +S_IXOTH = 0o0001 # execute by others + +# Names for file flags + +UF_NODUMP = 0x00000001 # do not dump file +UF_IMMUTABLE = 0x00000002 # file may not be changed +UF_APPEND = 0x00000004 # file may only be appended to +UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack +UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted +UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed +UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +SF_ARCHIVED = 0x00010000 # file may be archived +SF_IMMUTABLE = 0x00020000 # file may not be changed +SF_APPEND = 0x00040000 # file may only be appended to +SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted +SF_SNAPSHOT = 0x00200000 # file is a snapshot file + + +_filemode_table = ( + ((S_IFLNK, "l"), + (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR + (S_IFREG, "-"), + (S_IFBLK, "b"), + (S_IFDIR, "d"), + (S_IFCHR, "c"), + (S_IFIFO, "p")), + + ((S_IRUSR, "r"),), + ((S_IWUSR, "w"),), + ((S_IXUSR|S_ISUID, "s"), + (S_ISUID, "S"), + (S_IXUSR, "x")), + + ((S_IRGRP, "r"),), + ((S_IWGRP, "w"),), + ((S_IXGRP|S_ISGID, "s"), + (S_ISGID, "S"), + (S_IXGRP, "x")), + + ((S_IROTH, "r"),), + ((S_IWOTH, "w"),), + ((S_IXOTH|S_ISVTX, "t"), + (S_ISVTX, "T"), + (S_IXOTH, "x")) +) + +def filemode(mode): + """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" + perm = [] + for table in _filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) + + +# Windows FILE_ATTRIBUTE constants for interpreting os.stat()'s +# "st_file_attributes" member + +FILE_ATTRIBUTE_ARCHIVE = 32 +FILE_ATTRIBUTE_COMPRESSED = 2048 +FILE_ATTRIBUTE_DEVICE = 64 +FILE_ATTRIBUTE_DIRECTORY = 16 +FILE_ATTRIBUTE_ENCRYPTED = 16384 +FILE_ATTRIBUTE_HIDDEN = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 +FILE_ATTRIBUTE_NORMAL = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 +FILE_ATTRIBUTE_OFFLINE = 4096 +FILE_ATTRIBUTE_READONLY = 1 +FILE_ATTRIBUTE_REPARSE_POINT = 1024 +FILE_ATTRIBUTE_SPARSE_FILE = 512 +FILE_ATTRIBUTE_SYSTEM = 4 +FILE_ATTRIBUTE_TEMPORARY = 256 +FILE_ATTRIBUTE_VIRTUAL = 65536 + + +# If available, use C implementation +try: + from _stat import * +except ImportError: + pass From 775be3a36afa421acac122ba11ff1b7ef815b47b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 11:07:56 +0300 Subject: [PATCH 470/658] Add stat mode tests --- tests/snippets/stdlib_os.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 4b0344f12..2e6aa0a35 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -1,5 +1,6 @@ import os import time +import stat from testutils import assert_raises @@ -99,9 +100,11 @@ with TestWithTempDir() as tmpdir: names.add(dir_entry.name) paths.add(dir_entry.path) if dir_entry.is_dir(): + assert stat.S_ISDIR(dir_entry.stat().st_mode) == True dirs.add(dir_entry.name) if dir_entry.is_file(): files.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) == True assert names == set([FILE_NAME, FILE_NAME2, FOLDER]) assert paths == set([fname, fname2, folder]) @@ -111,6 +114,7 @@ with TestWithTempDir() as tmpdir: # Stat stat_res = os.stat(fname) print(stat_res.st_mode) + assert stat.S_ISREG(stat_res.st_mode) == True print(stat_res.st_ino) print(stat_res.st_dev) print(stat_res.st_nlink) From 5270da39344cddcf5ace17b3e8c6c5646e4bc435 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 12:25:17 +0300 Subject: [PATCH 471/658] Add ExceptHandler block --- tests/snippets/try_exceptions.py | 13 +++++++++++++ vm/src/frame.rs | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index c6a148e78..5c90a502e 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -193,3 +193,16 @@ for _ in [1, 2]: raise ArithmeticError() except ArithmeticError as e: continue + + +def g(): + try: + 1/0 + except ArithmeticError: + return 5 + +try: + g() + raise NameError +except NameError as ex: + assert ex.__context__ == None diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a9e155969..23c9b8e2b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -203,6 +203,7 @@ enum BlockType { end: bytecode::Label, context_manager: PyObjectRef, }, + ExceptHandler {}, } pub type FrameRef = PyRef; @@ -862,6 +863,12 @@ impl Frame { } bytecode::Instruction::PopException {} => { assert!(vm.pop_exception().is_some()); + let block = self.pop_block(); + assert!(block.is_some()); + match block.unwrap().typ { + BlockType::ExceptHandler {} => (), + _ => assert!(false), + }; Ok(None) } } @@ -937,6 +944,9 @@ impl Frame { } } } + BlockType::ExceptHandler { .. } => { + vm.pop_exception(); + } } } @@ -959,6 +969,9 @@ impl Frame { panic!("Exception in with __exit__ {:?}", exc); } }, + BlockType::ExceptHandler { .. } => { + vm.pop_exception(); + } } self.pop_block(); @@ -970,6 +983,7 @@ impl Frame { while let Some(block) = self.pop_block() { match block.typ { BlockType::TryExcept { handler } => { + self.push_block(BlockType::ExceptHandler {}); self.push_value(exc.clone()); vm.push_exception(exc); self.jump(handler); @@ -1004,6 +1018,8 @@ impl Frame { } } BlockType::Loop { .. } => {} + // Exception was already poped on Raised. + BlockType::ExceptHandler { .. } => {} } } Some(exc) From d803acfdc5e3493471c18e95bb503e52d8541421 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 12:28:19 +0300 Subject: [PATCH 472/658] Add yield in except test --- tests/snippets/try_exceptions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 5c90a502e..f37101b61 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -206,3 +206,17 @@ try: raise NameError except NameError as ex: assert ex.__context__ == None + + +def y(): + try: + 1/0 + except ArithmeticError: + yield 5 + + +try: + y() + raise NameError +except NameError as ex: + assert ex.__context__ == None From 159063bb4711e1c5864bd4c23fc92960dd9e544a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 12:50:28 +0300 Subject: [PATCH 473/658] Convert windows file attributes to unix mode --- vm/src/stdlib/os.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index fd5893e74..da35d6c69 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -386,12 +386,33 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +// Copied from CPython fileutils.c +#[cfg(windows)] +fn attributes_to_mode(attr: u32) -> u32 { + let file_attribute_directory: u32 = 16; + let file_attribute_readonly: u32 = 1; + let s_ifdir: u32 = 0o040000; + let s_ifreg: u32 = 0o100000; + let mut m: u32 = 0; + if attr & file_attribute_directory == file_attribute_directory { + m |= s_ifdir | 0111; /* IFEXEC for user,group,other */ + } else { + m |= s_ifreg; + } + if attr & file_attribute_readonly == file_attribute_readonly { + m |= 0444; + } else { + m |= 0666; + } + m +} + #[cfg(windows)] fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { use std::os::windows::fs::MetadataExt; match fs::metadata(&path.value) { Ok(meta) => Ok(StatResult { - st_mode: meta.file_attributes(), + st_mode: attributes_to_mode(meta.file_attributes()), st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. From 32714f889e968a49afdcf29b556a1bf7db5c0493 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 17:18:04 +0300 Subject: [PATCH 474/658] Remove empty params from ExceptHandler --- vm/src/frame.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 23c9b8e2b..c22ad8897 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -203,7 +203,7 @@ enum BlockType { end: bytecode::Label, context_manager: PyObjectRef, }, - ExceptHandler {}, + ExceptHandler, } pub type FrameRef = PyRef; @@ -866,7 +866,7 @@ impl Frame { let block = self.pop_block(); assert!(block.is_some()); match block.unwrap().typ { - BlockType::ExceptHandler {} => (), + BlockType::ExceptHandler => (), _ => assert!(false), }; Ok(None) @@ -944,7 +944,7 @@ impl Frame { } } } - BlockType::ExceptHandler { .. } => { + BlockType::ExceptHandler => { vm.pop_exception(); } } @@ -969,7 +969,7 @@ impl Frame { panic!("Exception in with __exit__ {:?}", exc); } }, - BlockType::ExceptHandler { .. } => { + BlockType::ExceptHandler => { vm.pop_exception(); } } @@ -1019,7 +1019,7 @@ impl Frame { } BlockType::Loop { .. } => {} // Exception was already poped on Raised. - BlockType::ExceptHandler { .. } => {} + BlockType::ExceptHandler => {} } } Some(exc) From 810b8ee4d0d8a1fa4be05f1fc92b04c717c16235 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 17:19:59 +0300 Subject: [PATCH 475/658] Simplify PopException --- vm/src/frame.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index c22ad8897..ddddad91e 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -862,14 +862,13 @@ impl Frame { Ok(None) } bytecode::Instruction::PopException {} => { - assert!(vm.pop_exception().is_some()); - let block = self.pop_block(); - assert!(block.is_some()); - match block.unwrap().typ { - BlockType::ExceptHandler => (), - _ => assert!(false), - }; - Ok(None) + let block = self.pop_block().unwrap(); // this asserts that the block is_some. + if let BlockType::ExceptHandler = block.typ { + assert!(vm.pop_exception().is_some()); + Ok(None) + } else { + panic!("Block type must be ExceptHandler here.") + } } } } From 46ad1f5449925c0dc762d8610018b6961e9063b5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 18:01:23 +0300 Subject: [PATCH 476/658] Add os.symlink --- tests/snippets/stdlib_os.py | 8 ++++++-- vm/src/stdlib/os.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 2e6aa0a35..8a3c4a251 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -67,6 +67,7 @@ class TestWithTempDir(): FILE_NAME = "test1" FILE_NAME2 = "test2" +SYMLINK = "symlink" FOLDER = "dir1" CONTENT = b"testing" CONTENT2 = b"rustpython" @@ -92,6 +93,9 @@ with TestWithTempDir() as tmpdir: folder = tmpdir + os.sep + FOLDER os.mkdir(folder) + symlink = tmpdir + os.sep + SYMLINK + os.symlink(fname, symlink) + names = set() paths = set() dirs = set() @@ -106,8 +110,8 @@ with TestWithTempDir() as tmpdir: files.add(dir_entry.name) assert stat.S_ISREG(dir_entry.stat().st_mode) == True - assert names == set([FILE_NAME, FILE_NAME2, FOLDER]) - assert paths == set([fname, fname2, folder]) + assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK]) + assert paths == set([fname, fname2, folder, symlink]) assert dirs == set([FOLDER]) assert files == set([FILE_NAME, FILE_NAME2]) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index da35d6c69..c48ec2aa1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -436,6 +436,35 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { unimplemented!(); } +#[cfg(unix)] +fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + use std::os::unix::fs; + fs::symlink(&src.value, &dst.value).map_err(|s| vm.new_os_error(s.to_string())) +} + +#[cfg(windows)] +fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + use std::os::windows::fs; + let ret = match fs::metadata(&dst.value) { + Ok(meta) => { + if meta.is_file() { + fs::symlink_file(&src.value, &dst.value) + } else if meta.is_dir() { + fs::symlink_dir(&src.value, &dst.value) + } else { + panic!("Uknown file type"); + } + } + Err(_) => fs::symlink_file(&src.value, &dst.value), + }; + ret.map_err(|s| vm.new_os_error(s.to_string())) +} + +#[cfg(not(any(unix)))] +fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + unimplemented!(); +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -489,6 +518,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "DirEntry" => dir_entry, "stat_result" => stat_result, "stat" => ctx.new_rustfunc(os_stat), + "symlink" => ctx.new_rustfunc(os_symlink), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From 6c1dca7cc23456023e0a4f51186a3896b0f56923 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 18:14:21 +0300 Subject: [PATCH 477/658] Add DirEntry.is_symlink --- tests/snippets/stdlib_os.py | 4 ++++ vm/src/stdlib/os.rs | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 8a3c4a251..5cad2f7ec 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -100,6 +100,7 @@ with TestWithTempDir() as tmpdir: paths = set() dirs = set() files = set() + symlinks = set() for dir_entry in os.scandir(tmpdir): names.add(dir_entry.name) paths.add(dir_entry.path) @@ -109,11 +110,14 @@ with TestWithTempDir() as tmpdir: if dir_entry.is_file(): files.add(dir_entry.name) assert stat.S_ISREG(dir_entry.stat().st_mode) == True + if dir_entry.is_symlink(): + symlinks.add(dir_entry.name) assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK]) assert paths == set([fname, fname2, folder, symlink]) assert dirs == set([FOLDER]) assert files == set([FILE_NAME, FILE_NAME2]) + assert symlinks == set([SYMLINK]) # Stat stat_res = os.stat(fname) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c48ec2aa1..f7252ef8c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -233,6 +233,14 @@ impl DirEntryRef { .is_file()) } + fn is_symlink(self, vm: &VirtualMachine) -> PyResult { + Ok(self + .entry + .file_type() + .map_err(|s| vm.new_os_error(s.to_string()))? + .is_symlink()) + } + fn stat(self, vm: &VirtualMachine) -> PyResult { os_stat(self.path(vm).try_into_ref(vm)?, vm) } @@ -484,6 +492,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "path" => ctx.new_property(DirEntryRef::path), "is_dir" => ctx.new_rustfunc(DirEntryRef::is_dir), "is_file" => ctx.new_rustfunc(DirEntryRef::is_file), + "is_symlink" => ctx.new_rustfunc(DirEntryRef::is_symlink), "stat" => ctx.new_rustfunc(DirEntryRef::stat), }); From 45d7c383cdbffd29d7b42fbed6680b5bee13fd61 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 01:54:10 +0300 Subject: [PATCH 478/658] Implement bool.__ror__, __rand__ and __rxor__ --- vm/src/obj/objbool.rs | 49 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index e96324f67..c1a999e45 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -43,8 +43,11 @@ The class bool is a subclass of the class int, and cannot be subclassed."; "__new__" => context.new_rustfunc(bool_new), "__repr__" => context.new_rustfunc(bool_repr), "__or__" => context.new_rustfunc(bool_or), + "__ror__" => context.new_rustfunc(bool_ror), "__and__" => context.new_rustfunc(bool_and), + "__rand__" => context.new_rustfunc(bool_rand), "__xor__" => context.new_rustfunc(bool_xor), + "__rxor__" => context.new_rustfunc(bool_rxor), "__doc__" => context.new_str(bool_doc.to_string()) }); } @@ -74,10 +77,10 @@ fn bool_repr(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result { - arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); - - if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { +fn do_bool_or(vm: &VirtualMachine, lhs: &PyObjectRef, rhs: &PyObjectRef) -> PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { let lhs = get_value(lhs); let rhs = get_value(rhs); (lhs || rhs).into_pyobject(vm) @@ -86,10 +89,20 @@ fn bool_or(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result { +fn bool_or(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_or(vm, lhs, rhs) +} - if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { +fn bool_ror(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_or(vm, lhs, rhs) +} + +fn do_bool_and(vm: &VirtualMachine, lhs: &PyObjectRef, rhs: &PyObjectRef) -> PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { let lhs = get_value(lhs); let rhs = get_value(rhs); (lhs && rhs).into_pyobject(vm) @@ -98,10 +111,20 @@ fn bool_and(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result { +fn bool_and(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_and(vm, lhs, rhs) +} - if objtype::isinstance(lhs, &vm.ctx.bool_type()) && objtype::isinstance(rhs, &vm.ctx.bool_type()) { +fn bool_rand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_and(vm, lhs, rhs) +} + +fn do_bool_xor(vm: &VirtualMachine, lhs: &PyObjectRef, rhs: &PyObjectRef) -> PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { let lhs = get_value(lhs); let rhs = get_value(rhs); (lhs ^ rhs).into_pyobject(vm) @@ -110,6 +133,16 @@ fn bool_xor(vm: &VirtualMachine, args: PyFuncArgs) -> Result PyResult { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_xor(vm, lhs, rhs) +} + +fn bool_rxor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_xor(vm, lhs, rhs) +} + fn bool_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, From 4cce44ab31d10e2dd5fc2d96d65ecc5e092f7bf7 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 02:24:55 +0300 Subject: [PATCH 479/658] Add test for print file parameter --- tests/snippets/printing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/snippets/printing.py b/tests/snippets/printing.py index e420c6c7f..b2c9ca55d 100644 --- a/tests/snippets/printing.py +++ b/tests/snippets/printing.py @@ -1,4 +1,5 @@ from testutils import assert_raises +import io print(2 + 3) @@ -9,3 +10,7 @@ try: print('test', end=None, sep=None, flush=None) except: assert False, 'Expected None passed to end, sep, and flush to not raise errors' + +buf = io.StringIO() +print('hello, world', file=buf) +assert buf.getvalue() == 'hello, world\n', buf.getvalue() From ad2499f432276a79ad1ef50dcd0b86ff26bd8f7d Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 02:25:32 +0300 Subject: [PATCH 480/658] Implement print file parameter --- vm/src/builtins.rs | 70 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 300d11b31..bf6ce663e 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -22,8 +22,8 @@ use crate::obj::objtype::{self, PyClass, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, - TypeProtocol, + IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -559,32 +559,60 @@ pub struct PrintOptions { end: Option, #[pyarg(keyword_only, default = "false")] flush: bool, + #[pyarg(keyword_only, default = "None")] + file: Option, } pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { - let stdout = io::stdout(); - let mut stdout_lock = stdout.lock(); - let mut first = true; - for object in objects { - if first { - first = false; - } else if let Some(ref sep) = options.sep { - write!(stdout_lock, "{}", sep.value).unwrap(); - } else { - write!(stdout_lock, " ").unwrap(); + if let Some(file) = options.file { + let sep = match options.sep { + Some(sep) => sep.into_pyobject(vm).unwrap(), + _ => PyString::from(" ").into_pyobject(vm).unwrap(), + }; + + let mut first = true; + for object in objects { + if first { + first = false; + } else { + vm.call_method(&file, "write", vec![sep.clone()])?; + } + + vm.call_method(&file, "write", vec![object])?; } - let s = &vm.to_str(&object)?.value; - write!(stdout_lock, "{}", s).unwrap(); - } - if let Some(end) = options.end { - write!(stdout_lock, "{}", end.value).unwrap(); + let end = match options.end { + Some(end) => end.into_pyobject(vm).unwrap(), + _ => PyString::from("\n").into_pyobject(vm).unwrap(), + }; + vm.call_method(&file, "write", vec![end])?; + + if options.flush { + vm.call_method(&file, "flush", vec![])?; + } } else { - writeln!(stdout_lock).unwrap(); - } + let sep = options.sep.as_ref().map_or(" ", |sep| &sep.value); - if options.flush { - stdout_lock.flush().unwrap(); + let stdout = io::stdout(); + let mut stdout_lock = stdout.lock(); + let mut first = true; + for object in objects { + if first { + first = false; + } else { + write!(stdout_lock, "{}", sep).unwrap(); + } + + let s = &vm.to_str(&object)?.value; + write!(stdout_lock, "{}", s).unwrap(); + } + + let end = options.end.as_ref().map_or("\n", |end| &end.value); + write!(stdout_lock, "{}", end).unwrap(); + + if options.flush { + stdout_lock.flush().unwrap(); + } } Ok(()) From bbb8df797fee90574053be1efd018f1c54235308 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Sat, 4 May 2019 22:55:14 +0300 Subject: [PATCH 481/658] Refactor builtin_print to avoid duplicate loop code --- vm/src/builtins.rs | 101 ++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index bf6ce663e..e80c35010 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -563,56 +563,73 @@ pub struct PrintOptions { file: Option, } +trait Printer { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()>; + fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()>; +} + +impl Printer for &'_ PyObjectRef { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> { + vm.call_method(self, "write", vec![obj])?; + Ok(()) + } + + fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()> { + vm.call_method(self, "flush", vec![])?; + Ok(()) + } +} + +impl Printer for std::io::StdoutLock<'_> { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> { + let s = &vm.to_str(&obj)?.value; + write!(self, "{}", s).unwrap(); + Ok(()) + } + + fn flush(&mut self, _vm: &VirtualMachine) -> PyResult<()> { + ::flush(self).unwrap(); + Ok(()) + } +} + pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { - if let Some(file) = options.file { - let sep = match options.sep { - Some(sep) => sep.into_pyobject(vm).unwrap(), - _ => PyString::from(" ").into_pyobject(vm).unwrap(), - }; + let stdout = io::stdout(); - let mut first = true; - for object in objects { - if first { - first = false; - } else { - vm.call_method(&file, "write", vec![sep.clone()])?; - } - - vm.call_method(&file, "write", vec![object])?; - } - - let end = match options.end { - Some(end) => end.into_pyobject(vm).unwrap(), - _ => PyString::from("\n").into_pyobject(vm).unwrap(), - }; - vm.call_method(&file, "write", vec![end])?; - - if options.flush { - vm.call_method(&file, "flush", vec![])?; - } + let mut printer: Box = if let Some(file) = &options.file { + Box::new(file) } else { - let sep = options.sep.as_ref().map_or(" ", |sep| &sep.value); + Box::new(stdout.lock()) + }; - let stdout = io::stdout(); - let mut stdout_lock = stdout.lock(); - let mut first = true; - for object in objects { - if first { - first = false; - } else { - write!(stdout_lock, "{}", sep).unwrap(); - } + let sep = options + .sep + .as_ref() + .map_or(" ", |sep| &sep.value) + .into_pyobject(vm) + .unwrap(); - let s = &vm.to_str(&object)?.value; - write!(stdout_lock, "{}", s).unwrap(); + let mut first = true; + for object in objects { + if first { + first = false; + } else { + printer.write(vm, sep.clone())?; } - let end = options.end.as_ref().map_or("\n", |end| &end.value); - write!(stdout_lock, "{}", end).unwrap(); + printer.write(vm, object)?; + } - if options.flush { - stdout_lock.flush().unwrap(); - } + let end = options + .end + .as_ref() + .map_or("\n", |end| &end.value) + .into_pyobject(vm) + .unwrap(); + printer.write(vm, end)?; + + if options.flush { + printer.flush(vm)?; } Ok(()) From 6225de30986f8f6f11b5f88c584d5d9b36e3dab9 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 4 May 2019 23:15:22 +0300 Subject: [PATCH 482/658] Change file attributes to consts --- vm/src/stdlib/os.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f7252ef8c..cebdc665a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -397,17 +397,17 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { // Copied from CPython fileutils.c #[cfg(windows)] fn attributes_to_mode(attr: u32) -> u32 { - let file_attribute_directory: u32 = 16; - let file_attribute_readonly: u32 = 1; - let s_ifdir: u32 = 0o040000; - let s_ifreg: u32 = 0o100000; + const FILE_ATTRIBUTE_DIRECTORY: u32 = 16; + const FILE_ATTRIBUTE_READONLY: u32 = 1; + const S_IFDIR: u32 = 0o040000; + const S_IFREG: u32 = 0o100000; let mut m: u32 = 0; - if attr & file_attribute_directory == file_attribute_directory { - m |= s_ifdir | 0111; /* IFEXEC for user,group,other */ + if attr & FILE_ATTRIBUTE_DIRECTORY == FILE_ATTRIBUTE_DIRECTORY { + m |= S_IFDIR | 0111; /* IFEXEC for user,group,other */ } else { - m |= s_ifreg; + m |= S_IFREG; } - if attr & file_attribute_readonly == file_attribute_readonly { + if attr & FILE_ATTRIBUTE_READONLY == FILE_ATTRIBUTE_READONLY { m |= 0444; } else { m |= 0666; From 1a5bddc41b42aff37152cc1cfed8940d174ea9de Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 3 May 2019 02:34:13 +0300 Subject: [PATCH 483/658] Fix clippy error about float comparison --- vm/src/obj/objfloat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 6c097b706..2da23bf4f 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -397,7 +397,7 @@ impl PyFloat { }; if ndigits.is_none() { let fract = self.value.fract(); - let value = if fract.abs() == 0.5 { + let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON { if self.value.trunc() % 2.0 == 0.0 { self.value - fract } else { From 88c3852b734c33fcd85f134da11432826411c762 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Sun, 5 May 2019 00:02:10 +0300 Subject: [PATCH 484/658] Don't check for methods on iter - it's a builtin function --- tests/not_impl_gen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py index 2e20f5609..e70cfc51b 100644 --- a/tests/not_impl_gen.py +++ b/tests/not_impl_gen.py @@ -7,14 +7,13 @@ objects = [ float, frozenset, int, - iter, list, memoryview, range, set, str, tuple, - object + object, ] header = open("generator/not_impl_header.txt") From c38a3711d0151560b0ac73c01a5e9f4664c6ec01 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Sun, 5 May 2019 00:05:13 +0300 Subject: [PATCH 485/658] Include reimplemented methods in whats_left.sh, and skip inherited methods --- tests/not_impl_gen.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py index e70cfc51b..2f1df44a2 100644 --- a/tests/not_impl_gen.py +++ b/tests/not_impl_gen.py @@ -16,6 +16,18 @@ objects = [ object, ] + +def attr_is_not_inherited(type_, attr): + """ + returns True if type_'s attr is not inherited from any of its base classes + """ + + bases = obj.__mro__[1:] + + return getattr(obj, attr) not in ( + getattr(base, attr, None) for base in bases) + + header = open("generator/not_impl_header.txt") footer = open("generator/not_impl_footer.txt") output = open("snippets/whats_left_to_implement.py", "w") @@ -25,7 +37,11 @@ output.write("expected_methods = {\n") for obj in objects: output.write(f" '{obj.__name__}': ({obj.__name__}, [\n") - output.write("\n".join(f" '{attr}'," for attr in dir(obj))) + output.write("\n".join( + f" {attr!r}," + for attr in dir(obj) + if attr_is_not_inherited(obj, attr) + )) output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n")) output.write("}\n\n") From 5bd23b4fb63cd5d200f9c62f804c68a6f3a11de1 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 19:14:34 +0300 Subject: [PATCH 486/658] is_dir and is_file follow_symlink by default --- vm/src/stdlib/os.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cebdc665a..a5634de64 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -218,19 +218,17 @@ impl DirEntryRef { } fn is_dir(self, vm: &VirtualMachine) -> PyResult { - Ok(self - .entry - .file_type() - .map_err(|s| vm.new_os_error(s.to_string()))? - .is_dir()) + match fs::metadata(self.entry.path()) { + Ok(meta) => Ok(meta.is_dir()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } } fn is_file(self, vm: &VirtualMachine) -> PyResult { - Ok(self - .entry - .file_type() - .map_err(|s| vm.new_os_error(s.to_string()))? - .is_file()) + match fs::metadata(self.entry.path()) { + Ok(meta) => Ok(meta.is_file()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } } fn is_symlink(self, vm: &VirtualMachine) -> PyResult { From 914f32afc357c868f99ea840269013ad853e6bb8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 19:17:30 +0300 Subject: [PATCH 487/658] Add symlink tests --- tests/snippets/stdlib_os.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 5cad2f7ec..d4e1edb4e 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -67,7 +67,8 @@ class TestWithTempDir(): FILE_NAME = "test1" FILE_NAME2 = "test2" -SYMLINK = "symlink" +SYMLINK_FILE = "symlink" +SYMLINK_FOLDER = "symlink1" FOLDER = "dir1" CONTENT = b"testing" CONTENT2 = b"rustpython" @@ -93,8 +94,10 @@ with TestWithTempDir() as tmpdir: folder = tmpdir + os.sep + FOLDER os.mkdir(folder) - symlink = tmpdir + os.sep + SYMLINK - os.symlink(fname, symlink) + symlink_file = tmpdir + os.sep + SYMLINK_FILE + os.symlink(fname, symlink_file) + symlink_folder = tmpdir + os.sep + SYMLINK_FOLDER + os.symlink(folder, symlink_folder) names = set() paths = set() @@ -113,11 +116,11 @@ with TestWithTempDir() as tmpdir: if dir_entry.is_symlink(): symlinks.add(dir_entry.name) - assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK]) - assert paths == set([fname, fname2, folder, symlink]) - assert dirs == set([FOLDER]) - assert files == set([FILE_NAME, FILE_NAME2]) - assert symlinks == set([SYMLINK]) + assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) + assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) + assert dirs == set([FOLDER, SYMLINK_FOLDER]) + assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE]) + assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER]) # Stat stat_res = os.stat(fname) @@ -130,3 +133,7 @@ with TestWithTempDir() as tmpdir: print(stat_res.st_gid) print(stat_res.st_size) assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) + + # stat default is follow_symlink=True + os.stat(fname).st_ino == os.stat(symlink_file).st_ino + os.stat(fname).st_mode == os.stat(symlink_file).st_mode From 260af4d877749e280152f834db08b23ef31b13e5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 19:40:01 +0300 Subject: [PATCH 488/658] Add follow_symlinks to DirEntry.{is_file, is_dir} --- tests/snippets/stdlib_os.py | 10 +++++++++ vm/src/stdlib/os.rs | 44 +++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index d4e1edb4e..f50f5c9b2 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -102,7 +102,9 @@ with TestWithTempDir() as tmpdir: names = set() paths = set() dirs = set() + dirs_no_symlink = set() files = set() + files_no_symlink = set() symlinks = set() for dir_entry in os.scandir(tmpdir): names.add(dir_entry.name) @@ -110,16 +112,24 @@ with TestWithTempDir() as tmpdir: if dir_entry.is_dir(): assert stat.S_ISDIR(dir_entry.stat().st_mode) == True dirs.add(dir_entry.name) + if dir_entry.is_dir(follow_symlinks=False): + assert stat.S_ISDIR(dir_entry.stat().st_mode) == True + dirs_no_symlink.add(dir_entry.name) if dir_entry.is_file(): files.add(dir_entry.name) assert stat.S_ISREG(dir_entry.stat().st_mode) == True + if dir_entry.is_file(follow_symlinks=False): + files_no_symlink.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) == True if dir_entry.is_symlink(): symlinks.add(dir_entry.name) assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) assert dirs == set([FOLDER, SYMLINK_FOLDER]) + assert dirs_no_symlink == set([FOLDER]) assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE]) + assert files_no_symlink == set([FILE_NAME, FILE_NAME2]) assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER]) # Stat diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a5634de64..a8941badc 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -208,6 +208,12 @@ impl PyValue for DirEntry { } } +#[derive(FromArgs)] +struct FollowSymlinks { + #[pyarg(keyword_only, default = "true")] + follow_symlinks: bool, +} + impl DirEntryRef { fn name(self, _vm: &VirtualMachine) -> String { self.entry.file_name().into_string().unwrap() @@ -217,18 +223,38 @@ impl DirEntryRef { self.entry.path().to_str().unwrap().to_string() } - fn is_dir(self, vm: &VirtualMachine) -> PyResult { - match fs::metadata(self.entry.path()) { - Ok(meta) => Ok(meta.is_dir()), - Err(s) => Err(vm.new_os_error(s.to_string())), + fn perform_on_metadata( + self, + follow_symlinks: FollowSymlinks, + action: &Fn(fs::Metadata) -> bool, + vm: &VirtualMachine, + ) -> PyResult { + match follow_symlinks.follow_symlinks { + true => match fs::metadata(self.entry.path()) { + Ok(meta) => Ok(action(meta)), + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + false => match fs::symlink_metadata(self.entry.path()) { + Ok(meta) => Ok(action(meta)), + Err(s) => Err(vm.new_os_error(s.to_string())), + }, } } - fn is_file(self, vm: &VirtualMachine) -> PyResult { - match fs::metadata(self.entry.path()) { - Ok(meta) => Ok(meta.is_file()), - Err(s) => Err(vm.new_os_error(s.to_string())), - } + fn is_dir(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + &|meta: fs::Metadata| -> bool { meta.is_dir() }, + vm, + ) + } + + fn is_file(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + &|meta: fs::Metadata| -> bool { meta.is_file() }, + vm, + ) } fn is_symlink(self, vm: &VirtualMachine) -> PyResult { From 2c30e83896ea11234fc6c63350f837aa2f7a69b7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 20:04:39 +0300 Subject: [PATCH 489/658] Add follow_symlinks to stat --- tests/snippets/stdlib_os.py | 3 + vm/src/stdlib/os.rs | 111 +++++++++++++++--------------------- 2 files changed, 49 insertions(+), 65 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index f50f5c9b2..8d111b6f3 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -147,3 +147,6 @@ with TestWithTempDir() as tmpdir: # stat default is follow_symlink=True os.stat(fname).st_ino == os.stat(symlink_file).st_ino os.stat(fname).st_mode == os.stat(symlink_file).st_mode + + os.stat(fname, follow_symlinks=False).st_ino == os.stat(symlink_file, follow_symlinks=False).st_ino + os.stat(fname, follow_symlinks=False).st_mode == os.stat(symlink_file, follow_symlinks=False).st_mode diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a8941badc..a67114da6 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -229,16 +229,12 @@ impl DirEntryRef { action: &Fn(fs::Metadata) -> bool, vm: &VirtualMachine, ) -> PyResult { - match follow_symlinks.follow_symlinks { - true => match fs::metadata(self.entry.path()) { - Ok(meta) => Ok(action(meta)), - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - false => match fs::symlink_metadata(self.entry.path()) { - Ok(meta) => Ok(action(meta)), - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - } + let metadata = match follow_symlinks.follow_symlinks { + true => fs::metadata(self.entry.path()), + false => fs::symlink_metadata(self.entry.path()), + }; + let meta = metadata.map_err(|s| vm.new_os_error(s.to_string()))?; + Ok(action(meta)) } fn is_dir(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { @@ -265,8 +261,8 @@ impl DirEntryRef { .is_symlink()) } - fn stat(self, vm: &VirtualMachine) -> PyResult { - os_stat(self.path(vm).try_into_ref(vm)?, vm) + fn stat(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + os_stat(self.path(vm).try_into_ref(vm)?, follow_symlinks, vm) } } @@ -361,11 +357,15 @@ impl StatResultRef { } } -#[cfg(target_os = "linux")] -fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - use std::os::linux::fs::MetadataExt; - match fs::metadata(&path.value) { - Ok(meta) => Ok(StatResult { +macro_rules! os_unix_stat_inner { + ( $path:expr, $follow_symlinks:expr, $vm:expr) => {{ + let metadata = match $follow_symlinks.follow_symlinks { + true => fs::metadata($path), + false => fs::symlink_metadata($path), + }; + let meta = metadata.map_err(|s| $vm.new_os_error(s.to_string()))?; + + Ok(StatResult { st_mode: meta.st_mode(), st_ino: meta.st_ino(), st_dev: meta.st_dev(), @@ -374,48 +374,27 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { st_gid: meta.st_gid(), st_size: meta.st_size(), } - .into_ref(vm) - .into_object()), - Err(s) => Err(vm.new_os_error(s.to_string())), - } + .into_ref($vm) + .into_object()) + }}; +} + +#[cfg(target_os = "linux")] +fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + use std::os::linux::fs::MetadataExt; + os_unix_stat_inner!(&path.value, follow_symlinks, vm) } #[cfg(target_os = "macos")] -fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { +fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { use std::os::macos::fs::MetadataExt; - match fs::metadata(&path.value) { - Ok(meta) => Ok(StatResult { - st_mode: meta.st_mode(), - st_ino: meta.st_ino(), - st_dev: meta.st_dev(), - st_nlink: meta.st_nlink(), - st_uid: meta.st_uid(), - st_gid: meta.st_gid(), - st_size: meta.st_size(), - } - .into_ref(vm) - .into_object()), - Err(s) => Err(vm.new_os_error(s.to_string())), - } + os_unix_stat_inner!(&path.value, follow_symlinks, vm) } #[cfg(target_os = "android")] -fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { +fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { use std::os::android::fs::MetadataExt; - match fs::metadata(&path.value) { - Ok(meta) => Ok(StatResult { - st_mode: meta.st_mode(), - st_ino: meta.st_ino(), - st_dev: meta.st_dev(), - st_nlink: meta.st_nlink(), - st_uid: meta.st_uid(), - st_gid: meta.st_gid(), - st_size: meta.st_size(), - } - .into_ref(vm) - .into_object()), - Err(s) => Err(vm.new_os_error(s.to_string())), - } + os_unix_stat_inner!(&path.value, follow_symlinks, vm) } // Copied from CPython fileutils.c @@ -440,22 +419,24 @@ fn attributes_to_mode(attr: u32) -> u32 { } #[cfg(windows)] -fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { +fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { use std::os::windows::fs::MetadataExt; - match fs::metadata(&path.value) { - Ok(meta) => Ok(StatResult { - st_mode: attributes_to_mode(meta.file_attributes()), - st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_uid: 0, // 0 on windows - st_gid: 0, // 0 on windows - st_size: meta.file_size(), - } - .into_ref(vm) - .into_object()), - Err(s) => Err(vm.new_os_error(s.to_string())), + let metadata = match follow_symlinks.follow_symlinks { + true => fs::metadata(&path.value), + false => fs::symlink_metadata(&path.value), + }; + let meta = metadata.map_err(|s| vm.new_os_error(s.to_string()))?; + Ok(StatResult { + st_mode: attributes_to_mode(meta.file_attributes()), + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_uid: 0, // 0 on windows + st_gid: 0, // 0 on windows + st_size: meta.file_size(), } + .into_ref(vm) + .into_object()) } #[cfg(not(any( From 6b495eac1e9c9ed049a18f98d19eeb561b72c00f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 May 2019 02:13:47 +0900 Subject: [PATCH 490/658] str(1.0) == '1.0' --- tests/snippets/floats.py | 4 ++++ vm/src/obj/objfloat.rs | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 2ac10b94f..82f2b86e3 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -173,6 +173,10 @@ assert_raises(OverflowError, float('inf').as_integer_ratio) assert_raises(OverflowError, float('-inf').as_integer_ratio) assert_raises(ValueError, float('nan').as_integer_ratio) +assert str(1.0) == '1.0' +assert str(0.0) == '0.0' +assert str(1.123456789) == '1.123456789' + # Test special case for lexer, float starts with a dot: a = .5 assert a == 0.5 diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index b52a469e1..39618c797 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -335,8 +335,12 @@ impl PyFloat { } #[pymethod(name = "__repr__")] - fn repr(&self, _vm: &VirtualMachine) -> String { - self.value.to_string() + fn repr(&self, vm: &VirtualMachine) -> String { + if self.is_integer(vm) { + format!("{:.1}", self.value) + } else { + self.value.to_string() + } } #[pymethod(name = "__truediv__")] From 717c96837e960e5820610eea9a08680636b61d47 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 5 May 2019 20:24:20 +0300 Subject: [PATCH 491/658] Borrow instead of cloning in `ord()` --- vm/src/builtins.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a82914ad1..f2b7bee87 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -507,7 +507,7 @@ fn builtin_oct(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn builtin_ord(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(string, Some(vm.ctx.str_type()))]); - let string = objstr::get_value(string); + let string = objstr::borrow_value(string); let string_len = string.chars().count(); if string_len > 1 { return Err(vm.new_type_error(format!( From 6a73790b262b9c90f543d97b536d7079e1493878 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 5 May 2019 20:24:54 +0300 Subject: [PATCH 492/658] Raise proper error when string of length 0 is given to `ord()` --- vm/src/builtins.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index f2b7bee87..d3158fe6b 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -509,7 +509,7 @@ fn builtin_ord(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(string, Some(vm.ctx.str_type()))]); let string = objstr::borrow_value(string); let string_len = string.chars().count(); - if string_len > 1 { + if string_len != 1 { return Err(vm.new_type_error(format!( "ord() expected a character, but string of length {} found", string_len From c06b6ccc9667acc823c5618c6ef7fd5d7f1d65ff Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 21:19:20 +0300 Subject: [PATCH 493/658] Fix windows build --- vm/src/stdlib/os.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a67114da6..0794eb99b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -420,6 +420,7 @@ fn attributes_to_mode(attr: u32) -> u32 { #[cfg(windows)] fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + use std::os::windows::fs; use std::os::windows::fs::MetadataExt; let metadata = match follow_symlinks.follow_symlinks { true => fs::metadata(&path.value), @@ -473,7 +474,7 @@ fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResu ret.map_err(|s| vm.new_os_error(s.to_string())) } -#[cfg(not(any(unix)))] +#[cfg(all(not(unix), not(windows)))] fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { unimplemented!(); } From ecd6e85f3c3e3b1d579f45db69d02a98e57ed70d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 5 May 2019 21:32:49 +0300 Subject: [PATCH 494/658] Fix windows build try 2 --- vm/src/stdlib/os.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0794eb99b..b52e3762e 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -420,7 +420,6 @@ fn attributes_to_mode(attr: u32) -> u32 { #[cfg(windows)] fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - use std::os::windows::fs; use std::os::windows::fs::MetadataExt; let metadata = match follow_symlinks.follow_symlinks { true => fs::metadata(&path.value), @@ -452,24 +451,24 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { #[cfg(unix)] fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - use std::os::unix::fs; - fs::symlink(&src.value, &dst.value).map_err(|s| vm.new_os_error(s.to_string())) + use std::os::unix::fs as unix_fs; + unix_fs::symlink(&src.value, &dst.value).map_err(|s| vm.new_os_error(s.to_string())) } #[cfg(windows)] fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - use std::os::windows::fs; + use std::os::windows::fs as win_fs; let ret = match fs::metadata(&dst.value) { Ok(meta) => { if meta.is_file() { - fs::symlink_file(&src.value, &dst.value) + win_fs::symlink_file(&src.value, &dst.value) } else if meta.is_dir() { - fs::symlink_dir(&src.value, &dst.value) + win_fs::symlink_dir(&src.value, &dst.value) } else { panic!("Uknown file type"); } } - Err(_) => fs::symlink_file(&src.value, &dst.value), + Err(_) => win_fs::symlink_file(&src.value, &dst.value), }; ret.map_err(|s| vm.new_os_error(s.to_string())) } From 5a95c5af22d2c43e04431aba5e36db0396c3e0ab Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 5 May 2019 23:29:49 +0200 Subject: [PATCH 495/658] remove unittests --- vm/src/obj/objbyteinner.rs | 299 ------------------------------------- 1 file changed, 299 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index d6c47c2f0..eebeb4fb5 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1183,302 +1183,3 @@ fn split_slice_reverse<'a>(slice: &'a [u8], sep: &[u8], maxsplit: i32) -> Vec<&' splitted.reverse(); splitted } - -#[cfg(test)] - -// needed for dev. Same as python tests in bytes.py. should it be kept ? - -mod tests { - use super::*; - - #[test] - fn no_end() { - assert_eq!( - split_slice(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[4, 5], -1), - vec![[1, 2, 3], [1, 2, 3], [1, 2, 3]] - ); - assert_eq!( - split_slice_reverse(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[4, 5], -1), - vec![[1, 2, 3], [1, 2, 3], [1, 2, 3]] - ) - } - - #[test] - fn needle_end() { - let v: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3], &[1, 2, 3], &[]]; - assert_eq!( - split_slice(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], &[4, 5], -1), - v - ); - assert_eq!( - split_slice_reverse(&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], &[4, 5], -1), - v - ) - } - - #[test] - fn needle_end_minus_one() { - let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3], &[1, 2, 3], &[3]]; - - assert_eq!(split_slice(&v, &n, -1), res); - assert_eq!(split_slice_reverse(&v, &n, -1), res) - } - - #[test] - fn needle_start() { - let v = [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[], &[2, 3], &[1, 2, 3], &[1, 2, 3]]; - - assert_eq!(split_slice(&v, &n, -1), res); - assert_eq!(split_slice_reverse(&v, &n, -1), res) - } - - #[test] - fn needle_start_plus_one() { - let v = [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1], &[2, 3], &[1, 2, 3], &[1, 2, 3]]; - - assert_eq!(split_slice(&v, &n, -1), res); - assert_eq!(split_slice_reverse(&v, &n, -1), res) - } - #[test] - fn needles_next_to() { - let v = [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[], &[], &[1, 2, 3], &[1, 2, 3]]; - - assert_eq!(split_slice(&v, &n, -1), res); - assert_eq!(split_slice_reverse(&v, &n, -1), res) - } - #[test] - fn no_end_max() { - let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3]]; - let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn needle_end_max() { - let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]]; - let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - #[test] - fn needle_end_minus_one_max() { - let v = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]]; - let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], &[3]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn needle_start_max() { - let v = [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[], &[2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; - let res_rev: Vec<&[u8]> = vec![&[4, 5, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - #[test] - fn needle_start_minus_one_max() { - let v = [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1], &[2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; - let res_rev: Vec<&[u8]> = vec![&[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], &[1, 2, 3]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn needle_next_to() { - let v = [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]; - let n = [4, 5]; - let res: Vec<&[u8]> = vec![&[1, 2, 3], &[], &[4, 5, 1, 2, 3, 4, 5, 1, 2, 3]]; - let res_rev: Vec<&[u8]> = vec![&[1, 2, 3, 4, 5, 4, 5], &[1, 2, 3], &[1, 2, 3]]; - let max = 2; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle() { - let v = [107, 13, 117, 104, 10, 102, 122, 32, 101, 9, 102]; - let n = []; - let res: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; - let res_rev: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle_end_nostring() { - let v = [107, 13, 117, 104, 10, 102, 122, 32, 101, 9, 102, 9]; - let n = []; - let res: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; - let res_rev: Vec<&[u8]> = vec![&[107], &[117, 104], &[102, 122], &[101], &[102]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle_sides_no_max() { - let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; - let n = []; - let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; - let res_rev: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle_sides_max_zero() { - let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; - let n = []; - let res: Vec<&[u8]> = vec![&[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]]; - let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]]; - let max = 0; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle_sides_max_one() { - let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; - let n = []; - let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122, 32, 101, 102, 9, 9]]; - let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104, 10, 102, 122], &[101, 102]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn no_needle_sides_max_two() { - let v = [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9]; - let n = []; - let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102, 9, 9]]; - let res_rev: Vec<&[u8]> = vec![&[13, 13, 13, 117, 104], &[102, 122], &[101, 102]]; - let max = 2; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - #[test] - fn no_needle_no_max_big_spaces() { - let v = [ - 13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9, - ]; - let n = []; - let res: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; - let res_rev: Vec<&[u8]> = vec![&[117, 104], &[102, 122], &[101, 102]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_needle() { - let v = [49, 44, 50, 44, 51]; - let n = [44]; - let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_needle_max_one() { - let v = [49, 44, 50, 44, 51]; - let n = [44]; - let res: Vec<&[u8]> = vec![&[49], &[50, 44, 51]]; - let res_rev: Vec<&[u8]> = vec![&[49, 44, 50], &[51]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_nearneedle() { - let v = [49, 44, 50, 44, 44, 51, 44]; - let n = [44]; - let res: Vec<&[u8]> = vec![&[49], &[50], &[], &[51], &[]]; - let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[], &[51], &[]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_space_no_sep() { - let v = [49, 32, 50, 32, 51]; - let n = []; - let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_space_no_sep_max_one() { - let v = [49, 32, 50, 32, 51]; - let n = []; - let res: Vec<&[u8]> = vec![&[49], &[50, 32, 51]]; - let res_rev: Vec<&[u8]> = vec![&[49, 32, 50], &[51]]; - let max = 1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - - #[test] - fn cpython_bigspace() { - let v = [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32]; - let n = []; - let res: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let res_rev: Vec<&[u8]> = vec![&[49], &[50], &[51]]; - let max = -1; - - assert_eq!(split_slice(&v, &n, max), res); - assert_eq!(split_slice_reverse(&v, &n, max), res_rev) - } - -} From ea535856b4f7715dec102655aabc57fa8dc9b015 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 5 May 2019 20:40:32 +0300 Subject: [PATCH 496/658] Raise error on out-of-range input in `chr()` --- vm/src/builtins.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a82914ad1..b99a38608 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -73,10 +73,10 @@ fn builtin_callable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { vm.is_callable(&obj) } -fn builtin_chr(i: u32, _vm: &VirtualMachine) -> String { +fn builtin_chr(i: u32, vm: &VirtualMachine) -> PyResult { match char::from_u32(i) { - Some(value) => value.to_string(), - None => '_'.to_string(), + Some(value) => Ok(value.to_string()), + None => Err(vm.new_value_error("chr() arg not in range(0x110000)".to_string())), } } From c656c5cc72b732a6200779e04d9ecb950f63a310 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 6 May 2019 03:10:53 +0300 Subject: [PATCH 497/658] Add `chr()` test snippets --- tests/snippets/builtin_chr.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/snippets/builtin_chr.py diff --git a/tests/snippets/builtin_chr.py b/tests/snippets/builtin_chr.py new file mode 100644 index 000000000..1ad642a58 --- /dev/null +++ b/tests/snippets/builtin_chr.py @@ -0,0 +1,8 @@ +from testutils import assert_raises + +assert "a" == chr(97) +assert "é" == chr(233) +assert "🤡" == chr(129313) + +assert_raises(TypeError, lambda: chr(), "chr() takes exactly one argument (0 given)") +assert_raises(ValueError, lambda: chr(0x110005), "ValueError: chr() arg not in range(0x110000)") From e35dec64f9410bfc06115b2bb1bae42cb498a97d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 6 May 2019 21:47:18 +0300 Subject: [PATCH 498/658] Add `vm.new_name_error` helper --- vm/src/frame.rs | 5 +---- vm/src/vm.rs | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ddddad91e..ae9958969 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1090,10 +1090,7 @@ impl Frame { let value = match optional_value { Some(value) => value, None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - return Err(name_error); + return Err(vm.new_name_error(format!("name '{}' is not defined", name))); } }; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 48bec9302..6b2f0e0b1 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -186,6 +186,11 @@ impl VirtualMachine { self.new_exception(type_error, msg) } + pub fn new_name_error(&self, msg: String) -> PyObjectRef { + let name_error = self.ctx.exceptions.name_error.clone(); + self.new_exception(name_error, msg) + } + pub fn new_unsupported_operand_error( &self, a: PyObjectRef, From ed94ddba2dbf6e9c9e1562694cda27aa3a08dbba Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 6 May 2019 21:04:09 +0200 Subject: [PATCH 499/658] add split, rsplit, partition, rpartition, expandtabs, spitlines, zfill, replace to bytearray --- tests/snippets/bytearray.py | 239 +++++++++++++++++++++++++++++++++++- vm/src/obj/objbytearray.rs | 86 ++++++++++++- vm/src/obj/objbytes.rs | 4 - 3 files changed, 322 insertions(+), 7 deletions(-) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 4f29fa187..d00af5bbd 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -357,6 +357,243 @@ assert bytearray(b" spacious ").rstrip() == bytearray(b" spacious") assert bytearray(b"mississippi").rstrip(b"ipz") == bytearray(b"mississ") + +# split +assert bytearray(b"1,2,3").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [bytearray(b"1"), bytearray(b"2,3")] +assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b""), bytearray(b"3"), bytearray(b"")] +assert bytearray(b"1 2 3").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1 2 3").split(maxsplit=1) == [bytearray(b"1"), bytearray(b"2 3")] +assert bytearray(b" 1 2 3 ").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"k\ruh\nfz e f").split() == [bytearray(b"k"), bytearray(b"uh"), bytearray(b"fz"), bytearray(b"e"), bytearray(b"f")] +assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [bytearray(b"Two lines"), bytearray(b"")] +assert bytearray(b"").split() == [] +assert bytearray(b"").split(bytearray(b"\n")) == [bytearray(b"")] +assert bytearray(b"\n").split(bytearray(b"\n")) == [bytearray(b""), bytearray(b"")] + +SPLIT_FIXTURES = [ + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + -1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + -1, + ], + # maxsplit + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], []], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], [3]], + 1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 4, 5], [1, 2, 3], [1, 2, 3]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]], + 0, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122], [101, 102]], + 1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102, 9, 9]], + [[13, 13, 13, 117, 104], [102, 122], [101, 102]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [[49, 44, 50, 44, 51], [44], [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 44, 50, 44, 51], [44], [[49], [50, 44, 51]], [[49, 44, 50], [51]], 1], + [ + [49, 44, 50, 44, 44, 51, 44], + [44], + [[49], [50], [], [51], []], + [[49], [50], [], [51], []], + -1, + ], + [[49, 32, 50, 32, 51], None, [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 32, 50, 32, 51], None, [[49], [50, 32, 51]], [[49, 32, 50], [51]], 1], + [ + [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32], + None, + [[49], [50], [51]], + [[49], [50], [51]], + -1, + ], +] + + +# for i in SPLIT_FIXTURES: # for not yet implemented : TypeError: Unsupported method: __next__ +n_sp = 0 +while n_sp < len(SPLIT_FIXTURES): + i = SPLIT_FIXTURES[n_sp] + sep = None if i[1] == None else bytearray(i[1]) + try: + assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[2]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected : ", [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])] + ) + break + + try: + assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[3]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected Rev : ", + [list(x) for x in bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4])], + ) + break + + n_sp += 1 + + +# expandtabs +a = bytearray(b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14") +assert ( + a.expandtabs() == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +) +assert a.expandtabs(5) == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray(b"01 012 0123 01234") +assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray(b"01 012 0123 01234") +assert bytearray(b"123\t123").expandtabs(-5) == bytearray(b"123123") +assert bytearray(b"123\t123").expandtabs(0) == bytearray(b"123123") + + +# # partition +assert bytearray(b"123456789").partition(b"45") == ((b"123"), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"14523456789").partition(b"45") == ((b"1"), bytearray(b"45"), bytearray(b"23456789")) +a = bytearray(b"14523456789").partition(b"45") +assert isinstance(a[1], bytearray) +a = bytearray(b"14523456789").partition(memoryview(b"45")) +assert isinstance(a[1], bytearray) + +# partition +assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ((bytearray(b"123")), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ((bytearray(b"14523")), bytearray(b"45"), bytearray(b"6789")) +a = bytearray(b"14523456789").rpartition(b"45") +assert isinstance(a[1], bytearray) +a = bytearray(b"14523456789").rpartition(memoryview(b"45")) +assert isinstance(a[1], bytearray) + +# splitlines +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [bytearray(b"ab c"), bytearray(b""), bytearray(b"de fg"), bytearray(b"kl")] +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines(keepends=True) == [ + bytearray(b"ab c\n"), + bytearray(b"\n"), + bytearray(b"de fg\r"), + bytearray(b"kl\r\n"), +] +assert bytearray(b"").splitlines() == [] +assert bytearray(b"One line\n").splitlines() == [b"One line"] + +# zfill + +assert bytearray(b"42").zfill(5) == bytearray(b"00042") +assert bytearray(b"-42").zfill(5) == bytearray(b"-0042") +assert bytearray(b"42").zfill(1) == bytearray(b"42") +assert bytearray(b"42").zfill(-1) == bytearray(b"42") + +# replace +assert bytearray(b"123456789123").replace(b"23",b"XX") ==bytearray(b'1XX4567891XX') +assert bytearray(b"123456789123").replace(b"23",b"XX", 1) ==bytearray(b'1XX456789123') +assert bytearray(b"123456789123").replace(b"23",b"XX", 0) == bytearray(b"123456789123") +assert bytearray(b"123456789123").replace(b"23",b"XX", -1) ==bytearray(b'1XX4567891XX') +assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray(b"14567891") + + # clear a = bytearray(b"abcd") a.clear() @@ -387,5 +624,3 @@ a.append(100) assert a == bytearray(b"append") assert len(a) == 6 assert a.pop() == 100 - -import bytes as bbytes diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 433624339..f6441587d 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -2,7 +2,8 @@ use crate::function::OptionalArg; use crate::obj::objbyteinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerPosition, + ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerPosition, ByteInnerSplitOptions, ByteInnerSplitlinesOptions, ByteInnerTranslateOptions, ByteOr, PyByteInner, }; use crate::obj::objint::PyIntRef; @@ -351,6 +352,89 @@ impl PyByteArrayRef { )) } + #[pymethod(name = "split")] + fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .split(options, false)? + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "rsplit")] + fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .split(options, true)? + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "partition")] + fn partition(self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + // sep ALWAYS converted to bytearray even it's bytes or memoryview + // so its ok to accept PyByteInner + let (left, right) = self.inner.borrow().partition(&sep, false)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytearray(left), + vm.ctx.new_bytearray(sep.elements), + vm.ctx.new_bytearray(right), + ])) + } + + #[pymethod(name = "rpartition")] + fn rpartition(self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + let (left, right) = self.inner.borrow().partition(&sep, true)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytearray(left), + vm.ctx.new_bytearray(sep.elements), + vm.ctx.new_bytearray(right), + ])) + } + + #[pymethod(name = "expandtabs")] + fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().expandtabs(options))) + } + + #[pymethod(name = "splitlines")] + fn splitlines(self, options: ByteInnerSplitlinesOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .splitlines(options) + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "zfill")] + fn zfill(self, width: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().zfill(width))) + } + + #[pymethod(name = "replace")] + fn replace( + self, + old: PyByteInner, + new: PyByteInner, + count: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().replace(old, new, count)?)) + } + #[pymethod(name = "clear")] fn clear(self, _vm: &VirtualMachine) { self.inner.borrow_mut().elements.clear(); diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8aca62646..6ffe0c029 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -350,8 +350,6 @@ impl PyBytesRef { #[pymethod(name = "partition")] fn partition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray - // even it's bytes or memoryview so PyByteInner wiil be ok for args let sepa = PyByteInner::try_from_object(vm, sep.clone())?; let (left, right) = self.inner.partition(&sepa, false)?; @@ -361,8 +359,6 @@ impl PyBytesRef { } #[pymethod(name = "rpartition")] fn rpartition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // TODO: when implementing bytearray,remember sep ALWAYS converted to bytearray - // even it's bytes or memoryview so PyByteInner wiil be ok for args let sepa = PyByteInner::try_from_object(vm, sep.clone())?; let (left, right) = self.inner.partition(&sepa, true)?; From dc18356947aeeffe670711024436ecc78aa3532d Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 6 May 2019 21:37:15 +0200 Subject: [PATCH 500/658] add bytes/byterray title --- tests/snippets/bytearray.py | 8 +++++++- tests/snippets/bytes.py | 13 +++++++++---- vm/src/obj/objbytearray.rs | 6 ++++++ vm/src/obj/objbyteinner.rs | 24 ++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 5 +++++ 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 4f29fa187..4cda6229e 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -388,4 +388,10 @@ assert a == bytearray(b"append") assert len(a) == 6 assert a.pop() == 100 -import bytes as bbytes +# title +assert bytearray(b"Hello world").title() == bytearray(b"Hello World") +assert ( + bytearray(b"they're bill's friends from the UK").title() + == bytearray(b"They'Re Bill'S Friends From The Uk") +) + diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 99de4898a..a0cc348db 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -178,7 +178,6 @@ assert b"kok".center(5, bytearray(b"x")) == b"xkokx" b"kok".center(-5) == b"kok" - # ljust assert [b"koki".ljust(i, b"|") for i in range(3, 10)] == [ b"koki", @@ -577,9 +576,15 @@ assert b"42".zfill(1) == b"42" assert b"42".zfill(-1) == b"42" # replace -assert b"123456789123".replace(b"23", b"XX") == b'1XX4567891XX' -assert b"123456789123".replace(b"23", b"XX", 1) == b'1XX456789123' +assert b"123456789123".replace(b"23", b"XX") == b"1XX4567891XX" +assert b"123456789123".replace(b"23", b"XX", 1) == b"1XX456789123" assert b"123456789123".replace(b"23", b"XX", 0) == b"123456789123" -assert b"123456789123".replace(b"23", b"XX", -1) == b'1XX4567891XX' +assert b"123456789123".replace(b"23", b"XX", -1) == b"1XX4567891XX" assert b"123456789123".replace(b"23", b"") == b"14567891" +# title +assert b"Hello world".title() == b"Hello World" +assert ( + b"they're bill's friends from the UK".title() + == b"They'Re Bill'S Friends From The Uk" +) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 433624339..75c817ac8 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -364,6 +364,7 @@ impl PyByteArrayRef { .push(x.as_bigint().byte_or(vm)?); Ok(()) } + #[pymethod(name = "pop")] fn pop(self, vm: &VirtualMachine) -> PyResult { let bytes = &mut self.inner.borrow_mut().elements; @@ -371,6 +372,11 @@ impl PyByteArrayRef { .pop() .ok_or_else(|| vm.new_index_error("pop from empty bytearray".to_string())) } + + #[pymethod(name = "title")] + fn title(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().title())) + } } // fn set_value(obj: &PyObjectRef, value: Vec) { diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 89e483bbc..16b371309 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -975,6 +975,30 @@ impl PyByteInner { Ok(res) } + + pub fn title(&self) -> Vec { + let mut res = vec![]; + let mut spaced = true; + + for i in self.elements.iter() { + match i { + 65..=90 | 97..=122 => { + if spaced { + res.push(i.to_ascii_uppercase()); + spaced = false + } else { + res.push(i.to_ascii_lowercase()); + } + } + _ => { + res.push(*i); + spaced = true + } + } + } + + res + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8aca62646..7b8082b52 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -402,6 +402,11 @@ impl PyBytesRef { ) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.replace(old, new, count)?)) } + + #[pymethod(name = "title")] + fn title(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.title())) + } } #[pyclass] From 04bff49ee0092507eed7a5d4e0c437e679cfbe0e Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 6 May 2019 22:40:00 +0300 Subject: [PATCH 501/658] Raise proper `NameError` on `del` when name is not defined Closes #929. --- tests/snippets/delete.py | 5 ++++- vm/src/frame.rs | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/snippets/delete.py b/tests/snippets/delete.py index 57d432ff6..aa7ae0536 100644 --- a/tests/snippets/delete.py +++ b/tests/snippets/delete.py @@ -1,4 +1,4 @@ -from testutils import assert_raises +from testutils import assert_raises, assertRaises a = 1 del a @@ -16,3 +16,6 @@ y = 2 del (x, y) assert_raises(NameError, lambda: x) assert_raises(NameError, lambda: y) + +with assertRaises(NameError): + del y diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ae9958969..50c317c37 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -124,7 +124,7 @@ impl Scope { pub trait NameProtocol { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option; fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); - fn delete_name(&self, vm: &VirtualMachine, name: &str); + fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult; fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; @@ -169,8 +169,8 @@ impl NameProtocol for Scope { self.get_locals().set_item(key, value, vm).unwrap(); } - fn delete_name(&self, vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key, vm).unwrap(); + fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult { + self.get_locals().del_item(key, vm) } fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { @@ -1071,8 +1071,10 @@ impl Frame { } fn delete_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - self.scope.delete_name(vm, name); - Ok(None) + match self.scope.delete_name(vm, name) { + Ok(_) => Ok(None), + Err(_) => Err(vm.new_name_error(format!("name '{}' is not defined", name))), + } } fn load_name( From e12c6813efcd8ab060ea464e96087c80051e3d41 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 7 May 2019 04:41:31 +0900 Subject: [PATCH 502/658] Add dict unpacking support for literal --- parser/src/ast.rs | 2 +- parser/src/python.lalrpop | 9 +++++++-- tests/snippets/dict.py | 7 +++++++ vm/src/compile.rs | 18 +++++++++++++++--- vm/src/stdlib/ast.rs | 6 +++++- vm/src/symboltable.rs | 6 +++++- 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index e0a050138..ec0677fe8 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -195,7 +195,7 @@ pub enum Expression { elements: Vec, }, Dict { - elements: Vec<(Expression, Expression)>, + elements: Vec<(Option, Expression)>, }, Set { elements: Vec, diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 02fd9e0b9..8d6dd7cac 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -833,8 +833,8 @@ TestListComp2: ast::Expression = { }, }; -TestDict: Vec<(ast::Expression, ast::Expression)> = { - > <_trailing_comma:","?> => elements, +TestDict: Vec<(Option, ast::Expression)> = { + > <_trailing_comma:","?> => elements, }; TestDictComp: ast::Expression = { @@ -850,6 +850,11 @@ DictEntry: (ast::Expression, ast::Expression) = { ":" => (e1, e2), }; +DictElement: (Option, ast::Expression) = { + => (Some(e.0), e.1), + "**" => (None, e), +}; + TestSet: Vec = { > ","? => e1 }; diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 7d67668c9..270d2c07e 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -191,3 +191,10 @@ assert x['c'] is None assert {1: None, "b": None} == dict.fromkeys([1, "b"]) assert {1: 0, "b": 0} == dict.fromkeys([1, "b"], 0) + +x = {'a': 1, 'b': 1, 'c': 1} +y = {'b': 2, 'c': 2, 'd': 2} +z = {'c': 3, 'd': 3, 'e': 3} + +w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} +assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 21f1402a6..b1964ca55 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -1256,13 +1256,25 @@ impl Compiler { } ast::Expression::Dict { elements } => { let size = elements.len(); + let has_double_star = elements.iter().any(|e| e.0.is_none()); for (key, value) in elements { - self.compile_expression(key)?; - self.compile_expression(value)?; + if let Some(key) = key { + self.compile_expression(key)?; + self.compile_expression(value)?; + if has_double_star { + self.emit(Instruction::BuildMap { + size: 1, + unpack: false, + }); + } + } else { + // dict unpacking + self.compile_expression(value)?; + } } self.emit(Instruction::BuildMap { size, - unpack: false, + unpack: has_double_star, }); } ast::Expression::Slice { elements } => { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index f27b5eb40..2ac36c81e 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -358,7 +358,11 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k)?.into_object()); + if let Some(k) = k { + keys.push(expression_to_ast(vm, k)?.into_object()); + } else { + keys.push(vm.ctx.none()); + } values.push(expression_to_ast(vm, v)?.into_object()); } diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 1d4a72ed9..ea9ae52a0 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -404,7 +404,11 @@ impl SymbolTableBuilder { } ast::Expression::Dict { elements } => { for (key, value) in elements { - self.scan_expression(key)?; + if let Some(key) = key { + self.scan_expression(key)?; + } else { + // dict unpacking marker + } self.scan_expression(value)?; } } From d4a3d7eb0189559012eab51b29c7acce87c44ede Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 6 May 2019 23:23:48 +0300 Subject: [PATCH 503/658] Raise an error on `range()` with step=0 --- tests/snippets/builtin_range.py | 1 + vm/src/obj/objrange.rs | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index 55d4f5194..03d1cb2cf 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -17,6 +17,7 @@ assert_raises(ValueError, lambda: range(10).index(-1), 'out of bounds') assert_raises(ValueError, lambda: range(10).index(10), 'out of bounds') assert_raises(ValueError, lambda: range(4, 10, 2).index(5), 'out of step') assert_raises(ValueError, lambda: range(10).index('foo'), 'not an int') +assert_raises(ValueError, lambda: range(1, 10, 0), 'step is zero') # count tests assert range(10).count(2) == 1 diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 43fcc668c..de937535f 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -130,14 +130,11 @@ impl PyRange { step: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PyRange { - start, - stop, - step: step - .into_option() - .unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)), + let step = step.unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)); + if step.as_bigint().is_zero() { + return Err(vm.new_value_error("range() arg 3 must not be zero".to_string())); } - .into_ref_with_type(vm, cls) + PyRange { start, stop, step }.into_ref_with_type(vm, cls) } #[pyproperty(name = "start")] From 6153e2d6ca2d8ff85df869c8cfba274b3b7b1592 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 16:25:33 +0300 Subject: [PATCH 504/658] Add CPython genericpath.py --- Lib/genericpath.py | 151 ++++++++++++++++++++++++++++++++++++ Lib/os.py | 33 ++++++++ tests/snippets/stdlib_os.py | 8 ++ 3 files changed, 192 insertions(+) create mode 100644 Lib/genericpath.py diff --git a/Lib/genericpath.py b/Lib/genericpath.py new file mode 100644 index 000000000..5dd703d73 --- /dev/null +++ b/Lib/genericpath.py @@ -0,0 +1,151 @@ +""" +Path operations common to more than one OS +Do not use directly. The OS specific modules import the appropriate +functions from this module themselves. +""" +import os +import stat + +__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', + 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', + 'samestat'] + + +# Does a path exist? +# This is false for dangling symbolic links on systems that support them. +def exists(path): + """Test whether a path exists. Returns False for broken symbolic links""" + try: + os.stat(path) + except (OSError, ValueError): + return False + return True + + +# This follows symbolic links, so both islink() and isdir() can be true +# for the same path on systems that support symlinks +def isfile(path): + """Test whether a path is a regular file""" + try: + st = os.stat(path) + except (OSError, ValueError): + return False + return stat.S_ISREG(st.st_mode) + + +# Is a path a directory? +# This follows symbolic links, so both islink() and isdir() +# can be true for the same path on systems that support symlinks +def isdir(s): + """Return true if the pathname refers to an existing directory.""" + try: + st = os.stat(s) + except (OSError, ValueError): + return False + return stat.S_ISDIR(st.st_mode) + + +def getsize(filename): + """Return the size of a file, reported by os.stat().""" + return os.stat(filename).st_size + + +def getmtime(filename): + """Return the last modification time of a file, reported by os.stat().""" + return os.stat(filename).st_mtime + + +def getatime(filename): + """Return the last access time of a file, reported by os.stat().""" + return os.stat(filename).st_atime + + +def getctime(filename): + """Return the metadata change time of a file, reported by os.stat().""" + return os.stat(filename).st_ctime + + +# Return the longest prefix of all list elements. +def commonprefix(m): + "Given a list of pathnames, returns the longest common leading component" + if not m: return '' + # Some people pass in a list of pathname parts to operate in an OS-agnostic + # fashion; don't try to translate in that case as that's an abuse of the + # API and they are already doing what they need to be OS-agnostic and so + # they most likely won't be using an os.PathLike object in the sublists. + if not isinstance(m[0], (list, tuple)): + m = tuple(map(os.fspath, m)) + s1 = min(m) + s2 = max(m) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 + +# Are two stat buffers (obtained from stat, fstat or lstat) +# describing the same file? +def samestat(s1, s2): + """Test whether two stat buffers reference the same file""" + return (s1.st_ino == s2.st_ino and + s1.st_dev == s2.st_dev) + + +# Are two filenames really pointing to the same file? +def samefile(f1, f2): + """Test whether two pathnames reference the same actual file""" + s1 = os.stat(f1) + s2 = os.stat(f2) + return samestat(s1, s2) + + +# Are two open files really referencing the same file? +# (Not necessarily the same file descriptor!) +def sameopenfile(fp1, fp2): + """Test whether two open file objects reference the same file""" + s1 = os.fstat(fp1) + s2 = os.fstat(fp2) + return samestat(s1, s2) + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +# Generic implementation of splitext, to be parametrized with +# the separators +def _splitext(p, sep, altsep, extsep): + """Split the extension from a pathname. + + Extension is everything from the last dot to the end, ignoring + leading dots. Returns "(root, ext)"; ext may be empty.""" + # NOTE: This code must work for text and bytes strings. + + sepIndex = p.rfind(sep) + if altsep: + altsepIndex = p.rfind(altsep) + sepIndex = max(sepIndex, altsepIndex) + + dotIndex = p.rfind(extsep) + if dotIndex > sepIndex: + # skip all leading dots + filenameIndex = sepIndex + 1 + while filenameIndex < dotIndex: + if p[filenameIndex:filenameIndex+1] != extsep: + return p[:dotIndex], p[dotIndex:] + filenameIndex += 1 + + return p, p[:0] + +def _check_arg_types(funcname, *args): + hasstr = hasbytes = False + for s in args: + if isinstance(s, str): + hasstr = True + elif isinstance(s, bytes): + hasbytes = True + else: + raise TypeError('%s() argument must be str or bytes, not %r' % + (funcname, s.__class__.__name__)) from None + if hasstr and hasbytes: + raise TypeError("Can't mix strings and bytes in path components") from None diff --git a/Lib/os.py b/Lib/os.py index a14702d5e..afe7bb4e0 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -135,3 +135,36 @@ def getenv(key, default=None): The optional second argument can specify an alternate default. key, default and the result are str.""" return environ.get(key, default) + + +def fspath(path): + """Return the path representation of a path-like object. + + If str or bytes is passed in, it is returned unchanged. Otherwise the + os.PathLike interface is used to get the path representation. If the + path representation is not str or bytes, TypeError is raised. If the + provided path is not str, bytes, or os.PathLike, TypeError is raised. + """ + if isinstance(path, (str, bytes)): + return path + + # Work from the object's type to match method resolution of other magic + # methods. + path_type = type(path) + try: + path_repr = path_type.__fspath__(path) + except AttributeError: + if hasattr(path_type, '__fspath__'): + raise + else: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) + if isinstance(path_repr, (str, bytes)): + return path_repr + else: + raise TypeError("expected {}.__fspath__() to return str or bytes, " + "not {}".format(path_type.__name__, + type(path_repr).__name__)) + + +import genericpath as path diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 8d111b6f3..079f733c6 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -150,3 +150,11 @@ with TestWithTempDir() as tmpdir: os.stat(fname, follow_symlinks=False).st_ino == os.stat(symlink_file, follow_symlinks=False).st_ino os.stat(fname, follow_symlinks=False).st_mode == os.stat(symlink_file, follow_symlinks=False).st_mode + + # os.path + assert os.path.exists(fname) == True + assert os.path.exists("NO_SUCH_FILE") == False + assert os.path.isfile(fname) == True + assert os.path.isdir(folder) == True + assert os.path.isfile(folder) == False + assert os.path.isdir(fname) == False From daae7aa998ab85b72c6c126816ca4761075ee22d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 16:31:21 +0300 Subject: [PATCH 505/658] Add CPython posixpath.py --- Lib/posixpath.py | 525 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 525 insertions(+) create mode 100644 Lib/posixpath.py diff --git a/Lib/posixpath.py b/Lib/posixpath.py new file mode 100644 index 000000000..ecb4e5a8f --- /dev/null +++ b/Lib/posixpath.py @@ -0,0 +1,525 @@ +"""Common operations on Posix pathnames. + +Instead of importing this module directly, import os and refer to +this module as os.path. The "os.path" name is an alias for this +module on Posix systems; on other systems (e.g. Windows), +os.path provides the same operations in a manner specific to that +platform, and is an alias to another module (e.g. ntpath). + +Some of this can actually be useful on non-Posix systems too, e.g. +for manipulation of the pathname component of URLs. +""" + +# Strings representing various path-related bits and pieces. +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '/' +pathsep = ':' +defpath = '/bin:/usr/bin' +altsep = None +devnull = '/dev/null' + +import os +import sys +import stat +import genericpath +from genericpath import * + +__all__ = ["normcase","isabs","join","splitdrive","split","splitext", + "basename","dirname","commonprefix","getsize","getmtime", + "getatime","getctime","islink","exists","lexists","isdir","isfile", + "ismount", "expanduser","expandvars","normpath","abspath", + "samefile","sameopenfile","samestat", + "curdir","pardir","sep","pathsep","defpath","altsep","extsep", + "devnull","realpath","supports_unicode_filenames","relpath", + "commonpath"] + + +def _get_sep(path): + if isinstance(path, bytes): + return b'/' + else: + return '/' + +# Normalize the case of a pathname. Trivial in Posix, string.lower on Mac. +# On MS-DOS this may also turn slashes into backslashes; however, other +# normalizations (such as optimizing '../' away) are not allowed +# (another function should be defined to do that). + +def normcase(s): + """Normalize case of pathname. Has no effect under Posix""" + return os.fspath(s) + + +# Return whether a path is absolute. +# Trivial in Posix, harder on the Mac or MS-DOS. + +def isabs(s): + """Test whether a path is absolute""" + s = os.fspath(s) + sep = _get_sep(s) + return s.startswith(sep) + + +# Join pathnames. +# Ignore the previous parts if a part is absolute. +# Insert a '/' unless the first part is empty or already ends in '/'. + +def join(a, *p): + """Join two or more pathname components, inserting '/' as needed. + If any component is an absolute path, all previous path components + will be discarded. An empty last part will result in a path that + ends with a separator.""" + a = os.fspath(a) + sep = _get_sep(a) + path = a + try: + if not p: + path[:0] + sep #23780: Ensure compatible data type even if p is null. + for b in map(os.fspath, p): + if b.startswith(sep): + path = b + elif not path or path.endswith(sep): + path += b + else: + path += sep + b + except (TypeError, AttributeError, BytesWarning): + genericpath._check_arg_types('join', a, *p) + raise + return path + + +# Split a path in head (everything up to the last '/') and tail (the +# rest). If the path ends in '/', tail will be empty. If there is no +# '/' in the path, head will be empty. +# Trailing '/'es are stripped from head unless it is the root. + +def split(p): + """Split a pathname. Returns tuple "(head, tail)" where "tail" is + everything after the final slash. Either part may be empty.""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + head, tail = p[:i], p[i:] + if head and head != sep*len(head): + head = head.rstrip(sep) + return head, tail + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +def splitext(p): + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + extsep = b'.' + else: + sep = '/' + extsep = '.' + return genericpath._splitext(p, sep, None, extsep) +splitext.__doc__ = genericpath._splitext.__doc__ + +# Split a pathname into a drive specification and the rest of the +# path. Useful on DOS/Windows/NT; on Unix, the drive is always empty. + +def splitdrive(p): + """Split a pathname into drive and path. On Posix, drive is always + empty.""" + p = os.fspath(p) + return p[:0], p + + +# Return the tail (basename) part of a path, same as split(path)[1]. + +def basename(p): + """Returns the final component of a pathname""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + return p[i:] + + +# Return the head (dirname) part of a path, same as split(path)[0]. + +def dirname(p): + """Returns the directory component of a pathname""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + head = p[:i] + if head and head != sep*len(head): + head = head.rstrip(sep) + return head + + +# Is a path a symbolic link? +# This will always return false on systems where os.lstat doesn't exist. + +def islink(path): + """Test whether a path is a symbolic link""" + try: + st = os.lstat(path) + except (OSError, ValueError, AttributeError): + return False + return stat.S_ISLNK(st.st_mode) + +# Being true for dangling symbolic links is also useful. + +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + + +# Is a path a mount point? +# (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) + +def ismount(path): + """Test whether a path is a mount point""" + try: + s1 = os.lstat(path) + except (OSError, ValueError): + # It doesn't exist -- so not a mount point. :-) + return False + else: + # A symlink can never be a mount point + if stat.S_ISLNK(s1.st_mode): + return False + + if isinstance(path, bytes): + parent = join(path, b'..') + else: + parent = join(path, '..') + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except (OSError, ValueError): + return False + + dev1 = s1.st_dev + dev2 = s2.st_dev + if dev1 != dev2: + return True # path/.. on a different device as path + ino1 = s1.st_ino + ino2 = s2.st_ino + if ino1 == ino2: + return True # path/.. is the same i-node as path + return False + + +# Expand paths beginning with '~' or '~user'. +# '~' means $HOME; '~user' means that user's home directory. +# If the path doesn't begin with '~', or if the user or $HOME is unknown, +# the path is returned unchanged (leaving error reporting to whatever +# function is called with the expanded path as argument). +# See also module 'glob' for expansion of *, ? and [...] in pathnames. +# (A function should also be defined to do full *sh-style environment +# variable expansion.) + +def expanduser(path): + """Expand ~ and ~user constructions. If user or $HOME is unknown, + do nothing.""" + path = os.fspath(path) + if isinstance(path, bytes): + tilde = b'~' + else: + tilde = '~' + if not path.startswith(tilde): + return path + sep = _get_sep(path) + i = path.find(sep, 1) + if i < 0: + i = len(path) + if i == 1: + if 'HOME' not in os.environ: + import pwd + try: + userhome = pwd.getpwuid(os.getuid()).pw_dir + except KeyError: + # bpo-10496: if the current user identifier doesn't exist in the + # password database, return the path unchanged + return path + else: + userhome = os.environ['HOME'] + else: + import pwd + name = path[1:i] + if isinstance(name, bytes): + name = str(name, 'ASCII') + try: + pwent = pwd.getpwnam(name) + except KeyError: + # bpo-10496: if the user name from the path doesn't exist in the + # password database, return the path unchanged + return path + userhome = pwent.pw_dir + if isinstance(path, bytes): + userhome = os.fsencode(userhome) + root = b'/' + else: + root = '/' + userhome = userhome.rstrip(root) + return (userhome + path[i:]) or root + + +# Expand paths containing shell variable substitutions. +# This expands the forms $variable and ${variable} only. +# Non-existent variables are left unchanged. + +_varprog = None +_varprogb = None + +def expandvars(path): + """Expand shell variables of form $var and ${var}. Unknown variables + are left unchanged.""" + path = os.fspath(path) + global _varprog, _varprogb + if isinstance(path, bytes): + if b'$' not in path: + return path + if not _varprogb: + import re + _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) + search = _varprogb.search + start = b'{' + end = b'}' + environ = getattr(os, 'environb', None) + else: + if '$' not in path: + return path + if not _varprog: + import re + _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) + search = _varprog.search + start = '{' + end = '}' + environ = os.environ + i = 0 + while True: + m = search(path, i) + if not m: + break + i, j = m.span(0) + name = m.group(1) + if name.startswith(start) and name.endswith(end): + name = name[1:-1] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(name)]) + else: + value = environ[name] + except KeyError: + i = j + else: + tail = path[j:] + path = path[:i] + value + i = len(path) + path += tail + return path + + +# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. +# It should be understood that this may change the meaning of the path +# if it contains symbolic links! + +def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'/' + empty = b'' + dot = b'.' + dotdot = b'..' + else: + sep = '/' + empty = '' + dot = '.' + dotdot = '..' + if path == empty: + return dot + initial_slashes = path.startswith(sep) + # POSIX allows one or two initial slashes, but treats three or more + # as single slash. + if (initial_slashes and + path.startswith(sep*2) and not path.startswith(sep*3)): + initial_slashes = 2 + comps = path.split(sep) + new_comps = [] + for comp in comps: + if comp in (empty, dot): + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = sep.join(comps) + if initial_slashes: + path = sep*initial_slashes + path + return path or dot + + +def abspath(path): + """Return an absolute path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) + + +# Return a canonical path (i.e. the absolute location of a file on the +# filesystem). + +def realpath(filename): + """Return the canonical path of the specified filename, eliminating any +symbolic links encountered in the path.""" + filename = os.fspath(filename) + path, ok = _joinrealpath(filename[:0], filename, {}) + return abspath(path) + +# Join two paths, normalizing and eliminating any symbolic links +# encountered in the second path. +def _joinrealpath(path, rest, seen): + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = split(path) + if name == pardir: + path = join(path, pardir, pardir) + else: + path = pardir + continue + newpath = join(path, name) + if not islink(newpath): + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + # Return already resolved part + rest of the path unchanged. + return join(newpath, rest), False + seen[newpath] = None # not resolved symlink + path, ok = _joinrealpath(path, os.readlink(newpath), seen) + if not ok: + return join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, True + + +supports_unicode_filenames = (sys.platform == 'darwin') + +def relpath(path, start=None): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + + path = os.fspath(path) + if isinstance(path, bytes): + curdir = b'.' + sep = b'/' + pardir = b'..' + else: + curdir = '.' + sep = '/' + pardir = '..' + + if start is None: + start = curdir + else: + start = os.fspath(start) + + try: + start_list = [x for x in abspath(start).split(sep) if x] + path_list = [x for x in abspath(path).split(sep) if x] + # Work out how much of the filepath is shared by start and path. + i = len(commonprefix([start_list, path_list])) + + rel_list = [pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return curdir + return join(*rel_list) + except (TypeError, AttributeError, BytesWarning, DeprecationWarning): + genericpath._check_arg_types('relpath', path, start) + raise + + +# Return the longest common sub-path of the sequence of paths given as input. +# The paths are not normalized before comparing them (this is the +# responsibility of the caller). Any trailing separator is stripped from the +# returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + paths = tuple(map(os.fspath, paths)) + if isinstance(paths[0], bytes): + sep = b'/' + curdir = b'.' + else: + sep = '/' + curdir = '.' + + try: + split_paths = [path.split(sep) for path in paths] + + try: + isabs, = set(p[:1] == sep for p in paths) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + common = s1 + for i, c in enumerate(s1): + if c != s2[i]: + common = s1[:i] + break + + prefix = sep if isabs else sep[:0] + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise From ac9ddf7756b4977a8e8e85b59b8b67909238a8d4 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 16:32:09 +0300 Subject: [PATCH 506/658] Add CPython ntpath.py --- Lib/ntpath.py | 661 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 Lib/ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py new file mode 100644 index 000000000..f3cfabf02 --- /dev/null +++ b/Lib/ntpath.py @@ -0,0 +1,661 @@ +# Module 'ntpath' -- common operations on WinNT/Win95 pathnames +"""Common pathname manipulations, WindowsNT/95 version. + +Instead of importing this module directly, import os and refer to this +module as os.path. +""" + +# strings representing various path-related bits and pieces +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '\\' +pathsep = ';' +altsep = '/' +defpath = '.;C:\\bin' +devnull = 'nul' + +import os +import sys +import stat +import genericpath +from genericpath import * + +__all__ = ["normcase","isabs","join","splitdrive","split","splitext", + "basename","dirname","commonprefix","getsize","getmtime", + "getatime","getctime", "islink","exists","lexists","isdir","isfile", + "ismount", "expanduser","expandvars","normpath","abspath", + "curdir","pardir","sep","pathsep","defpath","altsep", + "extsep","devnull","realpath","supports_unicode_filenames","relpath", + "samefile", "sameopenfile", "samestat", "commonpath"] + +def _get_bothseps(path): + if isinstance(path, bytes): + return b'\\/' + else: + return '\\/' + +# Normalize the case of a pathname and map slashes to backslashes. +# Other normalizations (such as optimizing '../' away) are not done +# (this is done by normpath). + +def normcase(s): + """Normalize case of pathname. + + Makes all characters lowercase and all slashes into backslashes.""" + s = os.fspath(s) + if isinstance(s, bytes): + return s.replace(b'/', b'\\').lower() + else: + return s.replace('/', '\\').lower() + + +# Return whether a path is absolute. +# Trivial in Posix, harder on Windows. +# For Windows it is absolute if it starts with a slash or backslash (current +# volume), or if a pathname after the volume-letter-and-colon or UNC-resource +# starts with a slash or backslash. + +def isabs(s): + """Test whether a path is absolute""" + s = os.fspath(s) + s = splitdrive(s)[1] + return len(s) > 0 and s[0] in _get_bothseps(s) + + +# Join two (or more) paths. +def join(path, *paths): + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + seps = b'\\/' + colon = b':' + else: + sep = '\\' + seps = '\\/' + colon = ':' + try: + if not paths: + path[:0] + sep #23780: Ensure compatible data type even if p is null. + result_drive, result_path = splitdrive(path) + for p in map(os.fspath, paths): + p_drive, p_path = splitdrive(p) + if p_path and p_path[0] in seps: + # Second path is absolute + if p_drive or not result_drive: + result_drive = p_drive + result_path = p_path + continue + elif p_drive and p_drive != result_drive: + if p_drive.lower() != result_drive.lower(): + # Different drives => ignore the first path entirely + result_drive = p_drive + result_path = p_path + continue + # Same drive in different case + result_drive = p_drive + # Second path is relative to the first + if result_path and result_path[-1] not in seps: + result_path = result_path + sep + result_path = result_path + p_path + ## add separator between UNC and non-absolute path + if (result_path and result_path[0] not in seps and + result_drive and result_drive[-1:] != colon): + return result_drive + sep + result_path + return result_drive + result_path + except (TypeError, AttributeError, BytesWarning): + genericpath._check_arg_types('join', path, *paths) + raise + + +# Split a path in a drive specification (a drive letter followed by a +# colon) and the path specification. +# It is always true that drivespec + pathspec == p +def splitdrive(p): + """Split a pathname into drive/UNC sharepoint and relative path specifiers. + Returns a 2-tuple (drive_or_unc, path); either part may be empty. + + If you assign + result = splitdrive(p) + It is always true that: + result[0] + result[1] == p + + If the path contained a drive letter, drive_or_unc will contain everything + up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + + If the path contained a UNC path, the drive_or_unc will contain the host name + and share up to but not including the fourth directory separator character. + e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + + Paths cannot contain both a drive letter and a UNC path. + + """ + p = os.fspath(p) + if len(p) >= 2: + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + else: + sep = '\\' + altsep = '/' + colon = ':' + normp = p.replace(altsep, sep) + if (normp[0:2] == sep*2) and (normp[2:3] != sep): + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + index = normp.find(sep, 2) + if index == -1: + return p[:0], p + index2 = normp.find(sep, index + 1) + # a UNC path can't have two slashes in a row + # (after the initial two) + if index2 == index + 1: + return p[:0], p + if index2 == -1: + index2 = len(p) + return p[:index2], p[index2:] + if normp[1:2] == colon: + return p[:2], p[2:] + return p[:0], p + + +# Split a path in head (everything up to the last '/') and tail (the +# rest). After the trailing '/' is stripped, the invariant +# join(head, tail) == p holds. +# The resulting head won't end in '/' unless it is the root. + +def split(p): + """Split a pathname. + + Return tuple (head, tail) where tail is everything after the final slash. + Either part may be empty.""" + p = os.fspath(p) + seps = _get_bothseps(p) + d, p = splitdrive(p) + # set i to index beyond p's last slash + i = len(p) + while i and p[i-1] not in seps: + i -= 1 + head, tail = p[:i], p[i:] # now tail has no slashes + # remove trailing slashes from head, unless it's all slashes + head = head.rstrip(seps) or head + return d + head, tail + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +def splitext(p): + p = os.fspath(p) + if isinstance(p, bytes): + return genericpath._splitext(p, b'\\', b'/', b'.') + else: + return genericpath._splitext(p, '\\', '/', '.') +splitext.__doc__ = genericpath._splitext.__doc__ + + +# Return the tail (basename) part of a path. + +def basename(p): + """Returns the final component of a pathname""" + return split(p)[1] + + +# Return the head (dirname) part of a path. + +def dirname(p): + """Returns the directory component of a pathname""" + return split(p)[0] + +# Is a path a symbolic link? +# This will always return false on systems where os.lstat doesn't exist. + +def islink(path): + """Test whether a path is a symbolic link. + This will always return false for Windows prior to 6.0. + """ + try: + st = os.lstat(path) + except (OSError, ValueError, AttributeError): + return False + return stat.S_ISLNK(st.st_mode) + +# Being true for dangling symbolic links is also useful. + +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + st = os.lstat(path) + except (OSError, ValueError): + return False + return True + +# Is a path a mount point? +# Any drive letter root (eg c:\) +# Any share UNC (eg \\server\share) +# Any volume mounted on a filesystem folder +# +# No one method detects all three situations. Historically we've lexically +# detected drive letter roots and share UNCs. The canonical approach to +# detecting mounted volumes (querying the reparse tag) fails for the most +# common case: drive letter roots. The alternative which uses GetVolumePathName +# fails if the drive letter is the result of a SUBST. +try: + from nt import _getvolumepathname +except ImportError: + _getvolumepathname = None +def ismount(path): + """Test whether a path is a mount point (a drive root, the root of a + share, or a mounted volume)""" + path = os.fspath(path) + seps = _get_bothseps(path) + path = abspath(path) + root, rest = splitdrive(path) + if root and root[0] in seps: + return (not rest) or (rest in seps) + if rest in seps: + return True + + if _getvolumepathname: + return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) + else: + return False + + +# Expand paths beginning with '~' or '~user'. +# '~' means $HOME; '~user' means that user's home directory. +# If the path doesn't begin with '~', or if the user or $HOME is unknown, +# the path is returned unchanged (leaving error reporting to whatever +# function is called with the expanded path as argument). +# See also module 'glob' for expansion of *, ? and [...] in pathnames. +# (A function should also be defined to do full *sh-style environment +# variable expansion.) + +def expanduser(path): + """Expand ~ and ~user constructs. + + If user or $HOME is unknown, do nothing.""" + path = os.fspath(path) + if isinstance(path, bytes): + tilde = b'~' + else: + tilde = '~' + if not path.startswith(tilde): + return path + i, n = 1, len(path) + while i < n and path[i] not in _get_bothseps(path): + i += 1 + + if 'USERPROFILE' in os.environ: + userhome = os.environ['USERPROFILE'] + elif not 'HOMEPATH' in os.environ: + return path + else: + try: + drive = os.environ['HOMEDRIVE'] + except KeyError: + drive = '' + userhome = join(drive, os.environ['HOMEPATH']) + + if isinstance(path, bytes): + userhome = os.fsencode(userhome) + + if i != 1: #~user + userhome = join(dirname(userhome), path[1:i]) + + return userhome + path[i:] + + +# Expand paths containing shell variable substitutions. +# The following rules apply: +# - no expansion within single quotes +# - '$$' is translated into '$' +# - '%%' is translated into '%' if '%%' are not seen in %var1%%var2% +# - ${varname} is accepted. +# - $varname is accepted. +# - %varname% is accepted. +# - varnames can be made out of letters, digits and the characters '_-' +# (though is not verified in the ${varname} and %varname% cases) +# XXX With COMMAND.COM you can use any characters in a variable name, +# XXX except '^|<>='. + +def expandvars(path): + """Expand shell variables of the forms $var, ${var} and %var%. + + Unknown variables are left unchanged.""" + path = os.fspath(path) + if isinstance(path, bytes): + if b'$' not in path and b'%' not in path: + return path + import string + varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') + quote = b'\'' + percent = b'%' + brace = b'{' + rbrace = b'}' + dollar = b'$' + environ = getattr(os, 'environb', None) + else: + if '$' not in path and '%' not in path: + return path + import string + varchars = string.ascii_letters + string.digits + '_-' + quote = '\'' + percent = '%' + brace = '{' + rbrace = '}' + dollar = '$' + environ = os.environ + res = path[:0] + index = 0 + pathlen = len(path) + while index < pathlen: + c = path[index:index+1] + if c == quote: # no expansion within single quotes + path = path[index + 1:] + pathlen = len(path) + try: + index = path.index(c) + res += c + path[:index + 1] + except ValueError: + res += c + path + index = pathlen - 1 + elif c == percent: # variable or '%' + if path[index + 1:index + 2] == percent: + res += c + index += 1 + else: + path = path[index+1:] + pathlen = len(path) + try: + index = path.index(percent) + except ValueError: + res += percent + path + index = pathlen - 1 + else: + var = path[:index] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = percent + var + percent + res += value + elif c == dollar: # variable or '$$' + if path[index + 1:index + 2] == dollar: + res += c + index += 1 + elif path[index + 1:index + 2] == brace: + path = path[index+2:] + pathlen = len(path) + try: + index = path.index(rbrace) + except ValueError: + res += dollar + brace + path + index = pathlen - 1 + else: + var = path[:index] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = dollar + brace + var + rbrace + res += value + else: + var = path[:0] + index += 1 + c = path[index:index + 1] + while c and c in varchars: + var += c + index += 1 + c = path[index:index + 1] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = dollar + var + res += value + if c: + index -= 1 + else: + res += c + index += 1 + return res + + +# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. +# Previously, this function also truncated pathnames to 8+3 format, +# but as this module is called "ntpath", that's obviously wrong! + +def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + pardir = b'..' + special_prefixes = (b'\\\\.\\', b'\\\\?\\') + else: + sep = '\\' + altsep = '/' + curdir = '.' + pardir = '..' + special_prefixes = ('\\\\.\\', '\\\\?\\') + if path.startswith(special_prefixes): + # in the case of paths with these prefixes: + # \\.\ -> device names + # \\?\ -> literal paths + # do not do any normalization, but return the path unchanged + return path + path = path.replace(altsep, sep) + prefix, path = splitdrive(path) + + # collapse initial backslashes + if path.startswith(sep): + prefix += sep + path = path.lstrip(sep) + + comps = path.split(sep) + i = 0 + while i < len(comps): + if not comps[i] or comps[i] == curdir: + del comps[i] + elif comps[i] == pardir: + if i > 0 and comps[i-1] != pardir: + del comps[i-1:i+1] + i -= 1 + elif i == 0 and prefix.endswith(sep): + del comps[i] + else: + i += 1 + else: + i += 1 + # If the path is now empty, substitute '.' + if not prefix and not comps: + comps.append(curdir) + return prefix + sep.join(comps) + +def _abspath_fallback(path): + """Return the absolute version of a path as a fallback function in case + `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for + more. + + """ + + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) + +# Return an absolute path. +try: + from nt import _getfullpathname + +except ImportError: # not running on Windows - mock up something sensible + abspath = _abspath_fallback + +else: # use native Windows method on Windows + def abspath(path): + """Return the absolute version of a path.""" + try: + return normpath(_getfullpathname(path)) + except (OSError, ValueError): + return _abspath_fallback(path) + +# realpath is a no-op on systems without islink support +realpath = abspath +# Win9x family and earlier have no Unicode filename support. +supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and + sys.getwindowsversion()[3] >= 2) + +def relpath(path, start=None): + """Return a relative version of a path""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + curdir = b'.' + pardir = b'..' + else: + sep = '\\' + curdir = '.' + pardir = '..' + + if start is None: + start = curdir + + if not path: + raise ValueError("no path specified") + + start = os.fspath(start) + try: + start_abs = abspath(normpath(start)) + path_abs = abspath(normpath(path)) + start_drive, start_rest = splitdrive(start_abs) + path_drive, path_rest = splitdrive(path_abs) + if normcase(start_drive) != normcase(path_drive): + raise ValueError("path is on mount %r, start on mount %r" % ( + path_drive, start_drive)) + + start_list = [x for x in start_rest.split(sep) if x] + path_list = [x for x in path_rest.split(sep) if x] + # Work out how much of the filepath is shared by start and path. + i = 0 + for e1, e2 in zip(start_list, path_list): + if normcase(e1) != normcase(e2): + break + i += 1 + + rel_list = [pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return curdir + return join(*rel_list) + except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): + genericpath._check_arg_types('relpath', path, start) + raise + + +# Return the longest common sub-path of the sequence of paths given as input. +# The function is case-insensitive and 'separator-insensitive', i.e. if the +# only difference between two paths is the use of '\' versus '/' as separator, +# they are deemed to be equal. +# +# However, the returned path will have the standard '\' separator (even if the +# given paths had the alternative '/' separator) and will have the case of the +# first path given in the sequence. Additionally, any trailing separator is +# stripped from the returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + paths = tuple(map(os.fspath, paths)) + if isinstance(paths[0], bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + else: + sep = '\\' + altsep = '/' + curdir = '.' + + try: + drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] + split_paths = [p.split(sep) for d, p in drivesplits] + + try: + isabs, = set(p[:1] == sep for d, p in drivesplits) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + # Check that all drive letters or UNC paths match. The check is made only + # now otherwise type errors for mixing strings and bytes would not be + # caught. + if len(set(d for d, p in drivesplits)) != 1: + raise ValueError("Paths don't have the same drive") + + drive, path = splitdrive(paths[0].replace(altsep, sep)) + common = path.split(sep) + common = [c for c in common if c and c != curdir] + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + for i, c in enumerate(s1): + if c != s2[i]: + common = common[:i] + break + else: + common = common[:len(s1)] + + prefix = drive + sep if isabs else drive + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise + + +# determine if two files are in fact the same file +try: + # GetFinalPathNameByHandle is available starting with Windows 6.0. + # Windows XP and non-Windows OS'es will mock _getfinalpathname. + if sys.getwindowsversion()[:2] >= (6, 0): + from nt import _getfinalpathname + else: + raise ImportError +except (AttributeError, ImportError): + # On Windows XP and earlier, two files are the same if their absolute + # pathnames are the same. + # Non-Windows operating systems fake this method with an XP + # approximation. + def _getfinalpathname(f): + return normcase(abspath(f)) + + +try: + # The genericpath.isdir implementation uses os.stat and checks the mode + # attribute to tell whether or not the path is a directory. + # This is overkill on Windows - just pass the path to GetFileAttributes + # and check the attribute from there. + from nt import _isdir as isdir +except ImportError: + # Use genericpath.isdir as imported above. + pass From 216f69e635e77e99ff08c4321c851114953c7148 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 16:46:45 +0300 Subject: [PATCH 507/658] Use os.path --- Lib/os.py | 20 +++++++++----------- Lib/posixpath.py | 2 +- tests/snippets/stdlib_os.py | 15 +++++++++------ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index afe7bb4e0..f1ff15b3c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,19 +1,19 @@ +import sys + from _os import * -curdir = '.' -pardir = '..' -extsep = '.' if name == 'nt': - sep = '\\' linesep = '\r\n' - altsep = '/' - pathsep = ';' + import ntpath as path else: - sep = '/' linesep = '\n' - altsep = None - pathsep = ':' + import posixpath as path + + +sys.modules['os.path'] = path +from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep, + devnull) # Change environ to automatically call putenv(), unsetenv if they exist. from _collections_abc import MutableMapping @@ -166,5 +166,3 @@ def fspath(path): "not {}".format(path_type.__name__, type(path_repr).__name__)) - -import genericpath as path diff --git a/Lib/posixpath.py b/Lib/posixpath.py index ecb4e5a8f..ae54f2387 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -444,7 +444,7 @@ def _joinrealpath(path, rest, seen): return path, True -supports_unicode_filenames = (sys.platform == 'darwin') +supports_unicode_filenames = (hasattr(sys, "platform") and sys.platform == 'darwin') def relpath(path, start=None): """Return a relative version of a path""" diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 079f733c6..26a101ecb 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -55,7 +55,7 @@ class TestWithTempDir(): base_folder = os.environ["TEMP"] else: base_folder = "/tmp" - name = base_folder + os.sep + "rustpython_test_os_" + str(int(time.time())) + name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time()))) os.mkdir(name) self.name = name return name @@ -75,7 +75,7 @@ CONTENT2 = b"rustpython" CONTENT3 = b"BOYA" with TestWithTempDir() as tmpdir: - fname = tmpdir + os.sep + FILE_NAME + fname = os.path.join(tmpdir, FILE_NAME) with open(fname, "wb"): pass fd = os.open(fname, 1) @@ -88,15 +88,15 @@ with TestWithTempDir() as tmpdir: assert os.read(fd, len(CONTENT3)) == CONTENT3 os.close(fd) - fname2 = tmpdir + os.sep + FILE_NAME2 + fname2 = os.path.join(tmpdir, FILE_NAME2) with open(fname2, "wb"): pass - folder = tmpdir + os.sep + FOLDER + folder = os.path.join(tmpdir, FOLDER) os.mkdir(folder) - symlink_file = tmpdir + os.sep + SYMLINK_FILE + symlink_file = os.path.join(tmpdir, SYMLINK_FILE) os.symlink(fname, symlink_file) - symlink_folder = tmpdir + os.sep + SYMLINK_FOLDER + symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER) os.symlink(folder, symlink_folder) names = set() @@ -158,3 +158,6 @@ with TestWithTempDir() as tmpdir: assert os.path.isdir(folder) == True assert os.path.isfile(folder) == False assert os.path.isdir(fname) == False + + assert os.path.basename(fname) == FILE_NAME + assert os.path.dirname(fname) == tmpdir From f5653cbc6819eb00b4200eccbc6dd2ab0ddd0d1f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 16:47:49 +0300 Subject: [PATCH 508/658] Use supported raise from --- Lib/os.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index f1ff15b3c..c768946c5 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -33,8 +33,7 @@ class _Environ(MutableMapping): value = self._data[self.encodekey(key)] except KeyError: # raise KeyError with the original key value - # raise KeyError(key) from None - raise KeyError(key) + raise KeyError(key) from None return self.decodevalue(value) @@ -51,8 +50,7 @@ class _Environ(MutableMapping): del self._data[encodedkey] except KeyError: # raise KeyError with the original key value - # raise KeyError(key) from None - raise KeyError(key) + raise KeyError(key) from None def __iter__(self): # list() from dict object is an atomic operation From 7d0ab6395bfa2e122df72bfc4f834686c8ad6ef0 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Thu, 9 May 2019 09:45:41 +0900 Subject: [PATCH 509/658] Revert mappingproxy hack --- Lib/_collections_abc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c9f7daa16..3158a4f16 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -52,7 +52,7 @@ dict_keys = type({}.keys()) dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## -# mappingproxy = type(type.__dict__) +mappingproxy = type(type.__dict__) # generator = type((lambda: (yield))()) ## coroutine ## # async def _coro(): pass @@ -688,7 +688,7 @@ class Mapping(Collection): __reversed__ = None -# Mapping.register(mappingproxy) +Mapping.register(mappingproxy) class MappingView(Sized): From 036b184dd38ad6022de355ef4f025aad6bb2503f Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 9 May 2019 10:52:10 +0800 Subject: [PATCH 510/658] add from_bytes for int --- tests/snippets/ints.py | 5 +++++ vm/src/obj/objint.rs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 40b4c2c6d..12f6f4b44 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -70,6 +70,11 @@ assert int("101", 2) == 5 assert int("101", base=2) == 5 assert int(1) == 1 +assert int.from_bytes(b'\x00\x10', 'big') == 16 +assert int.from_bytes(b'\x00\x10', 'little') == 4096 +assert int.from_bytes(b'\xfc\x00', 'big', True) == -1024 +assert int.from_bytes(b'\xfc\x00', 'big', False) == 64512 + with assertRaises(TypeError): int(base=2) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 7effb0897..33334f2e1 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -489,6 +489,29 @@ impl PyInt { zelf } + #[pymethod] + fn from_bytes(bytes: PyByteInner, byteorder: PyStringRef, signed: OptionalArg, vm: &VirtualMachine) -> PyResult{ + let x; + if byteorder.value == "big" { + x = match signed { + OptionalArg::Present(true) => BigInt::from_signed_bytes_be(&bytes.elements), + _ => BigInt::from_bytes_be(Sign::Plus, &bytes.elements), + } + } + else if byteorder.value == "little" { + x = match signed { + OptionalArg::Present(true) => BigInt::from_signed_bytes_le(&bytes.elements), + _ => BigInt::from_bytes_le(Sign::Plus, &bytes.elements), + } + } + else { + return Err(vm.new_value_error( + "byteorder must be either 'little' or 'big'".to_string(), + )); + } + Ok(x) + } + #[pyproperty] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf From e9fea281fb4faaf12a5bf93d724ca90758467a92 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 9 May 2019 11:02:34 +0800 Subject: [PATCH 511/658] add from_bytes for int --- vm/src/obj/objint.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 33334f2e1..d95afa54d 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,7 +1,7 @@ use std::fmt; use std::hash::{Hash, Hasher}; -use num_bigint::BigInt; +use num_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; @@ -15,6 +15,7 @@ use crate::vm::VirtualMachine; use super::objstr::{PyString, PyStringRef}; use super::objtype; +use super::objbyteinner::{PyByteInner}; use crate::obj::objtype::PyClassRef; /// int(x=0) -> integer From d3966af25d989fb2acad3de6aa7005783a7cc7b7 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Thu, 9 May 2019 15:54:07 +0900 Subject: [PATCH 512/658] Test all and any --- tests/snippets/builtin_all.py | 10 ++++++++++ tests/snippets/builtin_any.py | 19 ++++++++----------- tests/snippets/testutils.py | 9 +++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 tests/snippets/builtin_all.py diff --git a/tests/snippets/builtin_all.py b/tests/snippets/builtin_all.py new file mode 100644 index 000000000..4195d5f2e --- /dev/null +++ b/tests/snippets/builtin_all.py @@ -0,0 +1,10 @@ +from testutils import assert_raises +from testutils import TestFailingBool, TestFailingIter + +assert all([True]) +assert not all([False]) +assert all([]) +assert not all([False, TestFailingBool()]) + +assert_raises(RuntimeError, lambda: all(TestFailingIter())) +assert_raises(RuntimeError, lambda: all([TestFailingBool()])) diff --git a/tests/snippets/builtin_any.py b/tests/snippets/builtin_any.py index a8679d5fe..59b4514f8 100644 --- a/tests/snippets/builtin_any.py +++ b/tests/snippets/builtin_any.py @@ -1,13 +1,10 @@ -assert any([1]); -assert not any([]); -assert not any([0,0,0,0]); -assert any([0,0,1,0,0]); -def anything(a): - return a +from testutils import assert_raises +from testutils import TestFailingBool, TestFailingIter -class Test: - def __iter__(self): - while True: - yield True +assert any([True]) +assert not any([False]) +assert not any([]) +assert any([True, TestFailingBool()]) -assert any(map(anything, Test())) +assert_raises(RuntimeError, lambda: any(TestFailingIter())) +assert_raises(RuntimeError, lambda: any([TestFailingBool()])) diff --git a/tests/snippets/testutils.py b/tests/snippets/testutils.py index 49b7fedc1..ea4dd9138 100644 --- a/tests/snippets/testutils.py +++ b/tests/snippets/testutils.py @@ -34,3 +34,12 @@ class assertRaises: if not issubclass(exc_type, self.expected): return False return True + + +class TestFailingBool: + def __bool__(self): + raise RuntimeError + +class TestFailingIter: + def __iter__(self): + raise RuntimeError From 86fcc977305f1fa1a6d9adc3b693d3c3e348f1b9 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 9 May 2019 20:06:28 +0800 Subject: [PATCH 513/658] int_from_bytes --- tests/snippets/ints.py | 4 ++-- vm/src/obj/objint.rs | 41 +++++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 12f6f4b44..ef9328d69 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -72,8 +72,8 @@ assert int(1) == 1 assert int.from_bytes(b'\x00\x10', 'big') == 16 assert int.from_bytes(b'\x00\x10', 'little') == 4096 -assert int.from_bytes(b'\xfc\x00', 'big', True) == -1024 -assert int.from_bytes(b'\xfc\x00', 'big', False) == 64512 +assert int.from_bytes(b'\xfc\x00', 'big', signed=True) == -1024 +assert int.from_bytes(b'\xfc\x00', 'big', signed=False) == 64512 with assertRaises(TypeError): int(base=2) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index d95afa54d..21f50a182 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -6,16 +6,16 @@ use num_integer::Integer; use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use super::objbyteinner::PyByteInner; use super::objstr::{PyString, PyStringRef}; use super::objtype; -use super::objbyteinner::{PyByteInner}; use crate::obj::objtype::PyClassRef; /// int(x=0) -> integer @@ -491,24 +491,37 @@ impl PyInt { } #[pymethod] - fn from_bytes(bytes: PyByteInner, byteorder: PyStringRef, signed: OptionalArg, vm: &VirtualMachine) -> PyResult{ + fn from_bytes( + bytes: PyByteInner, + byteorder: PyStringRef, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let mut signed = false; + for (key, value) in kwargs.into_iter() { + if key == "signed" { + signed = match_class!(value, + + b @ PyInt => !b.as_bigint().is_zero(), + _ => false, + ); + } + } let x; if byteorder.value == "big" { x = match signed { - OptionalArg::Present(true) => BigInt::from_signed_bytes_be(&bytes.elements), - _ => BigInt::from_bytes_be(Sign::Plus, &bytes.elements), + true => BigInt::from_signed_bytes_be(&bytes.elements), + false => BigInt::from_bytes_be(Sign::Plus, &bytes.elements), } - } - else if byteorder.value == "little" { + } else if byteorder.value == "little" { x = match signed { - OptionalArg::Present(true) => BigInt::from_signed_bytes_le(&bytes.elements), - _ => BigInt::from_bytes_le(Sign::Plus, &bytes.elements), + true => BigInt::from_signed_bytes_le(&bytes.elements), + false => BigInt::from_bytes_le(Sign::Plus, &bytes.elements), } - } - else { - return Err(vm.new_value_error( - "byteorder must be either 'little' or 'big'".to_string(), - )); + } else { + return Err( + vm.new_value_error("byteorder must be either 'little' or 'big'".to_string()) + ); } Ok(x) } From 3196b6be73219eae356092c0dcf1683c920d1d06 Mon Sep 17 00:00:00 2001 From: zer0 Date: Thu, 9 May 2019 16:59:40 +0300 Subject: [PATCH 514/658] added str.translate --- vm/src/obj/objstr.rs | 50 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 56fc373c3..fe98d19c0 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -16,7 +16,8 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objint; +use super::objnone::PyNone; +use super::objint::{self, PyInt}; use super::objsequence::PySliceableSequence; use super::objslice::PySlice; use super::objtype::{self, PyClassRef}; @@ -829,6 +830,36 @@ impl PyString { false } } + + // https://docs.python.org/3/library/stdtypes.html#str.translate + // Make it work for only subscribtable types + #[pymethod] + fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut result = String::new(); + for c in self.value.chars() { + let args = PyFuncArgs::new(vec![vm.new_int(c as i32)], vec![]); + match vm.call_method(&table, "__getitem__", args) { + Ok(value) => { + if let Some(text) = value.payload::() { + result.extend(text.value.chars()); + } else if let Some(_) = value.payload::() { + // Do Nothing + } else if let Some(bigint) = value.payload::() { + match bigint.as_bigint() + .to_u32() + .and_then(std::char::from_u32) { + Some(ch) => result.push(ch as char), + None => return Err(vm.new_value_error(format!("character mapping must be in range(0x110000)"))), + } + } else { + return Err(vm.new_type_error("character mapping must return integer, None or str".to_owned())); + } + }, + _ => result.push(c), + } + } + Ok(result) + } } impl PyValue for PyString { @@ -1104,4 +1135,19 @@ mod tests { assert!(!PyString::from(s).istitle(&vm)); } } -} + + #[test] + fn str_translate() { + let vm = VirtualMachine::new(); + + let table = vm.context().new_dict(); + vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(97), vm.new_str("🎅".to_owned())]).unwrap(); + vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(98), vm.get_none()]).unwrap(); + vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(99), vm.new_str("xda".to_owned())]).unwrap(); + let text = PyString::from("abc"); + let translated = text.translate(table.into_object(), &vm).unwrap(); + assert_eq!(translated, "🎅xda".to_owned()); + let translated = text.translate(vm.new_int(3), &vm); + println!("{:?}", translated); + } +} \ No newline at end of file From 59ae590771c0389d77f535015917db0c3f67f9c3 Mon Sep 17 00:00:00 2001 From: zer0 Date: Thu, 9 May 2019 17:09:38 +0300 Subject: [PATCH 515/658] formatting --- parser/src/lexer.rs | 6 +----- vm/src/obj/objstr.rs | 43 ++++++++++++++++++++++++++++++------------ vm/src/obj/objsuper.rs | 6 ++---- whats_left.sh | 2 +- 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 11d30cbee..d23198ca8 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -826,11 +826,7 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok(( - tok_start, - Tok::DoubleSlash, - tok_end, - ))); + return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); } } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index fe98d19c0..6bbb9903d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -16,8 +16,8 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objnone::PyNone; use super::objint::{self, PyInt}; +use super::objnone::PyNone; use super::objsequence::PySliceableSequence; use super::objslice::PySlice; use super::objtype::{self, PyClassRef}; @@ -845,16 +845,20 @@ impl PyString { } else if let Some(_) = value.payload::() { // Do Nothing } else if let Some(bigint) = value.payload::() { - match bigint.as_bigint() - .to_u32() - .and_then(std::char::from_u32) { + match bigint.as_bigint().to_u32().and_then(std::char::from_u32) { Some(ch) => result.push(ch as char), - None => return Err(vm.new_value_error(format!("character mapping must be in range(0x110000)"))), + None => { + return Err(vm.new_value_error(format!( + "character mapping must be in range(0x110000)" + ))); + } } } else { - return Err(vm.new_type_error("character mapping must return integer, None or str".to_owned())); + return Err(vm.new_type_error( + "character mapping must return integer, None or str".to_owned(), + )); } - }, + } _ => result.push(c), } } @@ -1139,15 +1143,30 @@ mod tests { #[test] fn str_translate() { let vm = VirtualMachine::new(); - + let table = vm.context().new_dict(); - vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(97), vm.new_str("🎅".to_owned())]).unwrap(); - vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(98), vm.get_none()]).unwrap(); - vm.call_method(table.as_object(), "__setitem__", vec![vm.new_int(99), vm.new_str("xda".to_owned())]).unwrap(); + vm.call_method( + table.as_object(), + "__setitem__", + vec![vm.new_int(97), vm.new_str("🎅".to_owned())], + ) + .unwrap(); + vm.call_method( + table.as_object(), + "__setitem__", + vec![vm.new_int(98), vm.get_none()], + ) + .unwrap(); + vm.call_method( + table.as_object(), + "__setitem__", + vec![vm.new_int(99), vm.new_str("xda".to_owned())], + ) + .unwrap(); let text = PyString::from("abc"); let translated = text.translate(table.into_object(), &vm).unwrap(); assert_eq!(translated, "🎅xda".to_owned()); let translated = text.translate(vm.new_int(3), &vm); println!("{:?}", translated); } -} \ No newline at end of file +} diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 194212a49..94fe0fc7b 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -127,10 +127,8 @@ fn super_new( match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm.new_type_error(format!( - "super arguement {} was not supplied", - first_arg - ))); + return Err(vm + .new_type_error(format!("super arguement {} was not supplied", first_arg))); } } } else { diff --git a/whats_left.sh b/whats_left.sh index e05051d28..aa79d8d65 100755 --- a/whats_left.sh +++ b/whats_left.sh @@ -8,4 +8,4 @@ python3 not_impl_gen.py cd .. -cargo run -- tests/snippets/whats_left_to_implement.py +cargo +nightly run -- tests/snippets/whats_left_to_implement.py From 94d1b6585c698dde73e3b0026e4fe47aeeb01ed8 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Thu, 9 May 2019 23:50:20 +0300 Subject: [PATCH 516/658] Implement bytearray.__mul__, __imul__ and __rmul__ --- tests/snippets/bytearray.py | 15 +++++++++++ vm/src/obj/objbytearray.rs | 15 +++++++++++ vm/src/obj/objbyteinner.rs | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index d90f52a75..cbecf4936 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -631,3 +631,18 @@ assert ( bytearray(b"they're bill's friends from the UK").title() == bytearray(b"They'Re Bill'S Friends From The Uk") ) + + +# repeat by multiply +a = bytearray(b'abcd') +assert a * 0 == bytearray(b'') +assert a * -1 == bytearray(b'') +assert a * 1 == bytearray(b'abcd') +assert a * 3 == bytearray(b'abcdabcdabcd') +assert 3 * a == bytearray(b'abcdabcdabcd') + +a = bytearray(b'abcd') +a.__imul__(3) +assert a == bytearray(b'abcdabcdabcd') +a.__imul__(0) +assert a == bytearray(b'') diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 4981a6473..84c36148a 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -461,6 +461,21 @@ impl PyByteArrayRef { fn title(self, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytearray(self.inner.borrow().title())) } + + #[pymethod(name = "__mul__")] + fn repeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().repeat(n, vm)?)) + } + + #[pymethod(name = "__rmul__")] + fn rmul(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + self.repeat(n, vm) + } + + #[pymethod(name = "__imul__")] + fn irepeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().irepeat(n, vm) + } } // fn set_value(obj: &PyObjectRef, value: Vec) { diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 16b371309..ae318e628 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -7,6 +7,7 @@ use crate::pyobject::PyRef; use crate::pyobject::PyValue; use crate::pyobject::TryFromObject; use crate::pyobject::{PyIterable, PyObjectRef}; +use core::convert::TryFrom; use core::ops::Range; use num_bigint::BigInt; @@ -999,6 +1000,56 @@ impl PyByteInner { res } + + pub fn repeat(&self, n: PyIntRef, vm: &VirtualMachine) -> PyResult> { + if self.elements.is_empty() { + // We can multiple an empty vector by any integer, even if it doesn't fit in an isize. + return Ok(vec![]); + } + + let n = n.as_bigint().to_isize().ok_or_else(|| { + vm.new_overflow_error("can't multiply bytes that many times".to_string()) + })?; + + if n <= 0 { + Ok(vec![]) + } else { + let n = usize::try_from(n).unwrap(); + + let mut new_value = Vec::with_capacity(n * self.elements.len()); + for _ in 0..n { + new_value.extend(&self.elements); + } + + Ok(new_value) + } + } + + pub fn irepeat(&mut self, n: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + if self.elements.is_empty() { + // We can multiple an empty vector by any integer, even if it doesn't fit in an isize. + return Ok(()); + } + + let n = n.as_bigint().to_isize().ok_or_else(|| { + vm.new_overflow_error("can't multiply bytes that many times".to_string()) + })?; + + if n <= 0 { + self.elements.clear(); + } else { + let n = usize::try_from(n).unwrap(); + + let old = self.elements.clone(); + + self.elements.reserve((n - 1) * old.len()); + for _ in 1..n { + self.elements.extend(&old); + } + } + + Ok(()) + } } pub fn try_as_byte(obj: &PyObjectRef) -> Option> { From 87844ff4342c7faee3ba96e802a876fe4732929e Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Thu, 9 May 2019 23:51:20 +0300 Subject: [PATCH 517/658] Implement bytes.__mul__ and __rmul__ --- tests/snippets/bytes.py | 9 +++++++++ vm/src/obj/objbytes.rs | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a0cc348db..1fcbae3ab 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -588,3 +588,12 @@ assert ( b"they're bill's friends from the UK".title() == b"They'Re Bill'S Friends From The Uk" ) + + +# repeat by multiply +a = b'abcd' +assert a * 0 == b'' +assert a * -1 == b'' +assert a * 1 == b'abcd' +assert a * 3 == b'abcdabcdabcd' +assert 3 * a == b'abcdabcdabcd' diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 9cb44f4f8..102c70d4f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -403,6 +403,16 @@ impl PyBytesRef { fn title(self, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.title())) } + + #[pymethod(name = "__mul__")] + fn repeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.repeat(n, vm)?)) + } + + #[pymethod(name = "__rmul__")] + fn rmul(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + self.repeat(n, vm) + } } #[pyclass] From 04ce9ea1857a74b4c48034ecf6f7e55d88fd0e3f Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 10 May 2019 00:01:27 +0300 Subject: [PATCH 518/658] Implement bytearray.copy --- tests/snippets/bytearray.py | 9 +++++++++ vm/src/obj/objbytearray.rs | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index cbecf4936..8c03a2d01 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -646,3 +646,12 @@ a.__imul__(3) assert a == bytearray(b'abcdabcdabcd') a.__imul__(0) assert a == bytearray(b'') + + +# copy +a = bytearray(b"my bytearray") +b = a.copy() +assert a == b +assert a is not b +b.append(100) +assert a != b diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 84c36148a..b2c29fe11 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -440,6 +440,11 @@ impl PyByteArrayRef { self.inner.borrow_mut().elements.clear(); } + #[pymethod(name = "copy")] + fn copy(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().elements.clone())) + } + #[pymethod(name = "append")] fn append(self, x: PyIntRef, vm: &VirtualMachine) -> Result<(), PyObjectRef> { self.inner From 485ffa01a25266d06e8b4fb8fd399426300d473d Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 10 May 2019 00:08:41 +0300 Subject: [PATCH 519/658] Implement bytearray.extend --- tests/snippets/bytearray.py | 8 ++++++++ vm/src/obj/objbytearray.rs | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 8c03a2d01..4f08b2773 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -655,3 +655,11 @@ assert a == b assert a is not b b.append(100) assert a != b + + +# extend +a = bytearray(b"hello,") +# any iterable of ints should work +a.extend([32, 119, 111, 114]) +a.extend(b"ld") +assert a == bytearray(b"hello, world") diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index b2c29fe11..ee947deb8 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -454,6 +454,20 @@ impl PyByteArrayRef { Ok(()) } + #[pymethod(name = "extend")] + fn extend(self, iterable_of_ints: PyIterable, vm: &VirtualMachine) -> Result<(), PyObjectRef> { + let mut inner = self.inner.borrow_mut(); + + for x in iterable_of_ints.iter(vm)? { + let x = x?; + let x = PyIntRef::try_from_object(vm, x)?; + let x = x.as_bigint().byte_or(vm)?; + inner.elements.push(x); + } + + Ok(()) + } + #[pymethod(name = "pop")] fn pop(self, vm: &VirtualMachine) -> PyResult { let bytes = &mut self.inner.borrow_mut().elements; From 43cd50270f26ef938e55c757c74076540e587c2c Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 10 May 2019 00:25:55 +0300 Subject: [PATCH 520/658] Implement bytearray.insert --- tests/snippets/bytearray.py | 15 +++++++++++++++ vm/src/obj/objbytearray.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 4f08b2773..840c25bf4 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -663,3 +663,18 @@ a = bytearray(b"hello,") a.extend([32, 119, 111, 114]) a.extend(b"ld") assert a == bytearray(b"hello, world") + + +# insert +a = bytearray(b"hello, world") +a.insert(0, 119) +assert a == bytearray(b"whello, world"), a +# -1 is not at the end, but one before +a.insert(-1, 119) +assert a == bytearray(b"whello, worlwd"), a +# inserting before the beginning just inserts at the beginning +a.insert(-1000, 111) +assert a == bytearray(b"owhello, worlwd"), a +# inserting after the end just inserts at the end +a.insert(1000, 111) +assert a == bytearray(b"owhello, worlwdo"), a diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index ee947deb8..2da58cc96 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -15,7 +15,9 @@ use crate::pyobject::{ TryFromObject, }; use crate::vm::VirtualMachine; +use num_traits::ToPrimitive; use std::cell::{Cell, RefCell}; +use std::convert::TryFrom; use super::objiter; use super::objtype::PyClassRef; @@ -468,6 +470,37 @@ impl PyByteArrayRef { Ok(()) } + #[pymethod(name = "insert")] + fn insert(self, index: PyIntRef, x: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let bytes = &mut self.inner.borrow_mut().elements; + let len = isize::try_from(bytes.len()) + .map_err(|_e| vm.new_overflow_error("bytearray too big".to_string()))?; + + let x = x.as_bigint().byte_or(vm)?; + + let mut index = index + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("index too big".to_string()))?; + + if index >= len { + bytes.push(x); + return Ok(()); + } + + if index < 0 { + index += len; + index = index.max(0); + } + + let index = usize::try_from(index) + .map_err(|_e| vm.new_overflow_error("overflow in index calculation".to_string()))?; + + bytes.insert(index, x); + + Ok(()) + } + #[pymethod(name = "pop")] fn pop(self, vm: &VirtualMachine) -> PyResult { let bytes = &mut self.inner.borrow_mut().elements; From e5bae56ccb99f9c7a5ba70a72255f4db48ac1427 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 10 May 2019 00:31:37 +0300 Subject: [PATCH 521/658] Implement bytearray.remove --- tests/snippets/bytearray.py | 7 +++++++ vm/src/obj/objbytearray.rs | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 840c25bf4..cb1a61f83 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -678,3 +678,10 @@ assert a == bytearray(b"owhello, worlwd"), a # inserting after the end just inserts at the end a.insert(1000, 111) assert a == bytearray(b"owhello, worlwdo"), a + + +# remove +a = bytearray(b'abcdabcd') +a.remove(99) # the letter c +# Only the first is removed +assert a == bytearray(b'abdabcd') diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 2da58cc96..244cd63ba 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -322,6 +322,21 @@ impl PyByteArrayRef { Ok(res) } + #[pymethod(name = "remove")] + fn remove(self, x: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let x = x.as_bigint().byte_or(vm)?; + + let bytes = &mut self.inner.borrow_mut().elements; + let pos = bytes + .iter() + .position(|b| *b == x) + .ok_or(vm.new_value_error("value not found in bytearray".to_string()))?; + + bytes.remove(pos); + + Ok(()) + } + #[pymethod(name = "translate")] fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { self.inner.borrow().translate(options, vm) From 32950bd0bc4a08069726ce61fb57a5097972ae89 Mon Sep 17 00:00:00 2001 From: "Y. Sapir" Date: Fri, 10 May 2019 00:33:05 +0300 Subject: [PATCH 522/658] Implement bytearray.reverse --- tests/snippets/bytearray.py | 6 ++++++ vm/src/obj/objbytearray.rs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index cb1a61f83..dff0852c4 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -685,3 +685,9 @@ a = bytearray(b'abcdabcd') a.remove(99) # the letter c # Only the first is removed assert a == bytearray(b'abdabcd') + + +# reverse +a = bytearray(b'hello, world') +a.reverse() +assert a == bytearray(b'dlrow ,olleh') diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 244cd63ba..8b09a53e9 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -543,6 +543,12 @@ impl PyByteArrayRef { fn irepeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().irepeat(n, vm) } + + #[pymethod(name = "reverse")] + fn reverse(self, _vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().elements.reverse(); + Ok(()) + } } // fn set_value(obj: &PyObjectRef, value: Vec) { From 50c119a503002b88cb32c7ad3781ec77444c292a Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Fri, 10 May 2019 14:21:09 +0800 Subject: [PATCH 523/658] fix floor div for int --- tests/snippets/ints.py | 4 ++++ vm/src/obj/objint.rs | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index ef9328d69..be41c3257 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -64,6 +64,10 @@ assert (2).__rtruediv__(1.0) == NotImplemented assert (2).__pow__(3.0) == NotImplemented assert (2).__rpow__(3.0) == NotImplemented +assert 10 // 4 == 2 +assert -10 // 4 == -3 +assert 10 // -4 == -3 +assert -10 // -4 == 2 assert int() == 0 assert int("101", 2) == 5 diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 21f50a182..403fd08ef 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -260,7 +260,8 @@ impl PyInt { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { - Ok(vm.ctx.new_int((&self.value) / v2)) + let modulo = (&self.value % v2 + v2) % v2; + Ok(vm.ctx.new_int((&self.value - modulo) / v2)) } else { Err(vm.new_zero_division_error("integer floordiv by zero".to_string())) } From fe37ecf8d3065a361f56897b7b08475f6f9ea4c2 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 9 May 2019 15:48:42 +0300 Subject: [PATCH 524/658] Define `os_unix_stat_inner` only when required Avoids warn when building on Windows. --- vm/src/stdlib/os.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b52e3762e..e68873767 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -357,6 +357,7 @@ impl StatResultRef { } } +#[cfg(unix)] macro_rules! os_unix_stat_inner { ( $path:expr, $follow_symlinks:expr, $vm:expr) => {{ let metadata = match $follow_symlinks.follow_symlinks { From da3df8a0aee31d33e798c1be1e7de35cee4527a3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 10 May 2019 13:58:10 +0300 Subject: [PATCH 525/658] Add __file__ for imported modules --- tests/snippets/builtin_file.py | 3 +++ tests/snippets/import_file.py | 4 ++++ vm/src/import.rs | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/snippets/builtin_file.py create mode 100644 tests/snippets/import_file.py diff --git a/tests/snippets/builtin_file.py b/tests/snippets/builtin_file.py new file mode 100644 index 000000000..50dfeb931 --- /dev/null +++ b/tests/snippets/builtin_file.py @@ -0,0 +1,3 @@ +from import_file import import_file + +import_file() diff --git a/tests/snippets/import_file.py b/tests/snippets/import_file.py new file mode 100644 index 000000000..3f9eeed70 --- /dev/null +++ b/tests/snippets/import_file.py @@ -0,0 +1,4 @@ +import os + +def import_file(): + assert os.path.basename(__file__) == "import_file.py" diff --git a/vm/src/import.rs b/vm/src/import.rs index f4c52a11e..a03f14623 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -48,12 +48,13 @@ pub fn import_file( content: String, ) -> PyResult { let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); - let code_obj = compile::compile(vm, &content, &compile::Mode::Exec, file_path) + let code_obj = compile::compile(vm, &content, &compile::Mode::Exec, file_path.clone()) .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; + attrs.set_item("__file__", vm.new_str(file_path), vm)?; let module = vm.ctx.new_module(module_name, attrs.clone()); // Store module in cache to prevent infinite loop with mutual importing libs: From fc1a063a2cfd2b2b5082116566cdb7bb0f8c4ec4 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 10 May 2019 14:06:23 +0300 Subject: [PATCH 526/658] Add __file__ for file executed from main --- src/main.rs | 18 +++++++++++++----- tests/snippets/builtin_file.py | 4 ++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3e77ddbc2..72bb16139 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,8 +10,15 @@ extern crate rustyline; use clap::{App, Arg}; use rustpython_parser::error::ParseError; use rustpython_vm::{ - compile, error::CompileError, error::CompileErrorType, frame::Scope, import, obj::objstr, - print_exception, pyobject::PyResult, util, VirtualMachine, + compile, + error::CompileError, + error::CompileErrorType, + frame::Scope, + import, + obj::objstr, + print_exception, + pyobject::{ItemProtocol, PyResult}, + util, VirtualMachine, }; use rustyline::{error::ReadlineError, Editor}; use std::path::PathBuf; @@ -65,11 +72,12 @@ fn main() { } fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult { - let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path) + let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path.clone()) .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj.borrow()); - let vars = vm.ctx.new_scope(); // Keep track of local variables - vm.run_code_obj(code_obj, vars) + let attrs = vm.ctx.new_dict(); + attrs.set_item("__file__", vm.new_str(source_path), vm)?; + vm.run_code_obj(code_obj, Scope::new(None, attrs)) } fn handle_exception(vm: &VirtualMachine, result: PyResult) { diff --git a/tests/snippets/builtin_file.py b/tests/snippets/builtin_file.py index 50dfeb931..2c80d1590 100644 --- a/tests/snippets/builtin_file.py +++ b/tests/snippets/builtin_file.py @@ -1,3 +1,7 @@ +import os + from import_file import import_file import_file() + +assert os.path.basename(__file__) == "builtin_file.py" From f69c79bdc7f12e54589c9762398c607c4aede3ab Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 27 Apr 2019 16:39:41 +0900 Subject: [PATCH 527/658] Add __builtins__ to globals() --- src/main.rs | 1 + tests/snippets/builtins_module.py | 16 ++++++++++++++++ vm/src/frame.rs | 5 +++++ wasm/lib/src/vm_class.rs | 1 + 4 files changed, 23 insertions(+) create mode 100644 tests/snippets/builtins_module.py diff --git a/src/main.rs b/src/main.rs index 72bb16139..e8dd48008 100644 --- a/src/main.rs +++ b/src/main.rs @@ -218,6 +218,7 @@ fn run_shell(vm: &VirtualMachine) -> PyResult { crate_version!() ); let vars = vm.ctx.new_scope(); + vars.init_builtins(vm)?; // Read a single line: let mut input = String::new(); diff --git a/tests/snippets/builtins_module.py b/tests/snippets/builtins_module.py new file mode 100644 index 000000000..b9feffd0a --- /dev/null +++ b/tests/snippets/builtins_module.py @@ -0,0 +1,16 @@ +from testutils import assertRaises + +assert '__builtins__' in globals() +# assert type(__builtins__).__name__ == 'module' +with assertRaises(AttributeError): + __builtins__.__builtins__ + +__builtins__.x = 'new' +assert x == 'new' + +# __builtins__ is deletable but names are alive +del __builtins__ +with assertRaises(NameError): + __builtins__ + +assert print diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 50c317c37..847487306 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -119,6 +119,11 @@ impl Scope { pub fn child_scope(&self, ctx: &PyContext) -> Scope { self.child_scope_with_locals(ctx.new_dict()) } + + pub fn init_builtins(&self, vm: &VirtualMachine) -> PyResult<()> { + let globals = self.globals.clone(); + globals.set_item("__builtins__", vm.builtins.clone(), vm).and(Ok(())) + } } pub trait NameProtocol { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 2c330e6ee..64957e331 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -27,6 +27,7 @@ impl StoredVirtualMachine { fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { let mut vm = VirtualMachine::new(); let scope = vm.ctx.new_scope(); + scope.init_builtins(&vm).unwrap(); if inject_browser_module { setup_browser_module(&vm); } From 509394080bc5262d3c3961ca16919631c9e1eb5b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 4 May 2019 00:52:44 +0900 Subject: [PATCH 528/658] Inject __builtins__ to scope only when it doesn't exist --- benchmarks/bench.rs | 4 ++-- src/main.rs | 5 ++--- tests/snippets/builtins_module.py | 12 ++++++++++++ vm/src/builtins.rs | 15 ++++++++++++--- vm/src/eval.rs | 2 +- vm/src/frame.rs | 19 ++++++++++++++----- vm/src/import.rs | 2 +- vm/src/pyobject.rs | 4 ---- vm/src/vm.rs | 4 ++++ wasm/lib/src/vm_class.rs | 3 +-- 10 files changed, 49 insertions(+), 21 deletions(-) diff --git a/benchmarks/bench.rs b/benchmarks/bench.rs index b475da7cb..256fa47ad 100644 --- a/benchmarks/bench.rs +++ b/benchmarks/bench.rs @@ -99,7 +99,7 @@ fn bench_rustpy_nbody(b: &mut test::Bencher) { }; b.iter(|| { - let scope = vm.ctx.new_scope(); + let scope = vm.new_scope_with_builtins(); let res: PyResult = vm.run_code_obj(code.clone(), scope); assert!(res.is_ok()); }) @@ -118,7 +118,7 @@ fn bench_rustpy_mandelbrot(b: &mut test::Bencher) { }; b.iter(|| { - let scope = vm.ctx.new_scope(); + let scope = vm.new_scope_with_builtins(); let res: PyResult = vm.run_code_obj(code.clone(), scope); assert!(res.is_ok()); }) diff --git a/src/main.rs b/src/main.rs index e8dd48008..2100226c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,7 @@ fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResu // trace!("Code object: {:?}", code_obj.borrow()); let attrs = vm.ctx.new_dict(); attrs.set_item("__file__", vm.new_str(source_path), vm)?; - vm.run_code_obj(code_obj, Scope::new(None, attrs)) + vm.run_code_obj(code_obj, Scope::with_builtins(None, attrs, vm)) } fn handle_exception(vm: &VirtualMachine, result: PyResult) { @@ -217,8 +217,7 @@ fn run_shell(vm: &VirtualMachine) -> PyResult { "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", crate_version!() ); - let vars = vm.ctx.new_scope(); - vars.init_builtins(vm)?; + let vars = vm.new_scope_with_builtins(); // Read a single line: let mut input = String::new(); diff --git a/tests/snippets/builtins_module.py b/tests/snippets/builtins_module.py index b9feffd0a..e3bdcdc16 100644 --- a/tests/snippets/builtins_module.py +++ b/tests/snippets/builtins_module.py @@ -8,6 +8,18 @@ with assertRaises(AttributeError): __builtins__.x = 'new' assert x == 'new' +exec('assert "__builtins__" in globals()', dict()) +exec('assert __builtins__ == 7', {'__builtins__': 7}) +exec('assert not isinstance(__builtins__, dict)') +exec('assert isinstance(__builtins__, dict)', {}) + +namespace = {} +exec('', namespace) +assert namespace['__builtins__'] == __builtins__.__dict__ + +# with assertRaises(NameError): +# exec('print(__builtins__)', {'__builtins__': {}}) + # __builtins__ is deletable but names are alive del __builtins__ with assertRaises(NameError): diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a48a3b441..5b3082b0d 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -218,18 +218,27 @@ fn make_scope( } None => None, }; - let current_scope = vm.current_scope(); let globals = match globals { - Some(dict) => dict.clone().downcast().unwrap(), + Some(dict) => { + let dict: PyDictRef = dict.clone().downcast().unwrap(); + if !dict.contains_key("__builtins__", vm) { + let builtins_dict = vm.builtins.dict.as_ref().unwrap().as_object(); + dict.set_item("__builtins__", builtins_dict.clone(), vm) + .unwrap(); + } + dict + } None => current_scope.globals.clone(), }; + let locals = match locals { Some(dict) => dict.clone().downcast().ok(), None => current_scope.get_only_locals(), }; - Ok(Scope::new(locals, globals)) + let scope = Scope::with_builtins(locals, globals, vm); + Ok(scope) } fn builtin_format( diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 21bc9dadf..a927f8f03 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -24,7 +24,7 @@ mod tests { fn test_print_42() { let source = String::from("print('Hello world')\n"); let mut vm = VirtualMachine::new(); - let vars = vm.ctx.new_scope(); + let vars = vm.new_scope_with_builtins(); let _result = eval(&mut vm, &source, vars, ""); // TODO: check result? diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 847487306..5955660b6 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -98,6 +98,20 @@ impl Scope { Scope { locals, globals } } + pub fn with_builtins( + locals: Option, + globals: PyDictRef, + vm: &VirtualMachine, + ) -> Scope { + if !globals.contains_key("__builtins__", vm) { + globals + .clone() + .set_item("__builtins__", vm.builtins.clone(), vm) + .unwrap(); + } + Scope::new(locals, globals) + } + pub fn get_locals(&self) -> PyDictRef { match self.locals.iter().next() { Some(dict) => dict.clone(), @@ -119,11 +133,6 @@ impl Scope { pub fn child_scope(&self, ctx: &PyContext) -> Scope { self.child_scope_with_locals(ctx.new_dict()) } - - pub fn init_builtins(&self, vm: &VirtualMachine) -> PyResult<()> { - let globals = self.globals.clone(); - globals.set_item("__builtins__", vm.builtins.clone(), vm).and(Ok(())) - } } pub trait NameProtocol { diff --git a/vm/src/import.rs b/vm/src/import.rs index a03f14623..8445e65f9 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -61,7 +61,7 @@ pub fn import_file( sys_modules.set_item(module_name, module.clone(), vm)?; // Execute main code in module: - vm.run_code_obj(code_obj, Scope::new(None, attrs))?; + vm.run_code_obj(code_obj, Scope::with_builtins(None, attrs, vm))?; Ok(module) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 454a1605f..722588b3f 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -648,10 +648,6 @@ impl PyContext { objtype::new(self.type_type(), name, vec![base], PyAttributes::new()).unwrap() } - pub fn new_scope(&self) -> Scope { - Scope::new(None, self.new_dict()) - } - pub fn new_module(&self, name: &str, dict: PyDictRef) -> PyObjectRef { PyObject::new( PyModule { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 6b2f0e0b1..814631015 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -250,6 +250,10 @@ impl VirtualMachine { syntax_error } + pub fn new_scope_with_builtins(&self) -> Scope { + Scope::with_builtins(None, self.ctx.new_dict(), self) + } + pub fn get_none(&self) -> PyObjectRef { self.ctx.none() } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 64957e331..f9040dfcb 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -26,8 +26,7 @@ pub(crate) struct StoredVirtualMachine { impl StoredVirtualMachine { fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { let mut vm = VirtualMachine::new(); - let scope = vm.ctx.new_scope(); - scope.init_builtins(&vm).unwrap(); + let scope = vm.new_scope_with_builtins(); if inject_browser_module { setup_browser_module(&vm); } From b720681c7b684b86dd9ba4de488ea82c7f6dc517 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 04:38:38 +0900 Subject: [PATCH 529/658] PyClassImpl for PyClassMethod --- vm/src/obj/objclassmethod.rs | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index 7a8d90d1c..69d6dba4e 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -1,7 +1,28 @@ use super::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +/// classmethod(function) -> method +/// +/// Convert a function to be a class method. +/// +/// A class method receives the class as implicit first argument, +/// just like an instance method receives the instance. +/// To declare a class method, use this idiom: +/// +/// class C: +/// @classmethod +/// def f(cls, arg1, arg2, ...): +/// ... +/// +/// It can be called either on the class (e.g. C.f()) or on an instance +/// (e.g. C().f()). The instance is ignored except for its class. +/// If a class method is called for a derived class, the derived class +/// object is passed as the implied first argument. +/// +/// Class methods are different than C++ or Java static methods. +/// If you want those, see the staticmethod builtin. +#[pyclass] #[derive(Clone, Debug)] pub struct PyClassMethod { pub callable: PyObjectRef, @@ -16,7 +37,9 @@ impl PyValue for PyClassMethod { } } -impl PyClassMethodRef { +#[pyimpl] +impl PyClassMethod { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, callable: PyObjectRef, @@ -28,7 +51,8 @@ impl PyClassMethodRef { .into_ref_with_type(vm, cls) } - fn get(self, _inst: PyObjectRef, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__get__")] + fn get(&self, _inst: PyObjectRef, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm .ctx .new_bound_method(self.callable.clone(), owner.clone())) @@ -36,9 +60,5 @@ impl PyClassMethodRef { } pub fn init(context: &PyContext) { - let classmethod_type = &context.classmethod_type; - extend_class!(context, classmethod_type, { - "__get__" => context.new_rustfunc(PyClassMethodRef::get), - "__new__" => context.new_rustfunc(PyClassMethodRef::new) - }); + PyClassMethod::extend_class(context, &context.classmethod_type); } From 7564833b6e3769041db391725d754d13a2a2da8a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 05:17:29 +0900 Subject: [PATCH 530/658] Add classmethod.__func__ --- tests/snippets/class.py | 3 +++ vm/src/obj/objclassmethod.rs | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index ded02dbfa..389c22712 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -152,3 +152,6 @@ class T4: assert T4.__doc__ == "test4" assert T4.t1.__doc__ == "t1" + +cm = classmethod(lambda cls: cls) +assert cm.__func__(int) is int diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index 69d6dba4e..10b3106ec 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -57,6 +57,11 @@ impl PyClassMethod { .ctx .new_bound_method(self.callable.clone(), owner.clone())) } + + #[pyproperty(name = "__func__")] + fn func(&self, _vm: &VirtualMachine) -> PyObjectRef { + self.callable.clone() + } } pub fn init(context: &PyContext) { From 2f253ca0545bb71ef480df0c25461de6a118e81d Mon Sep 17 00:00:00 2001 From: zer0 Date: Fri, 10 May 2019 23:31:11 +0300 Subject: [PATCH 531/658] cargo fmt --- vm/src/obj/objstr.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 6bbb9903d..4f6589598 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -11,8 +11,8 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, - PyValue, TryFromObject, TryIntoRef, TypeProtocol, + IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, + PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -832,13 +832,13 @@ impl PyString { } // https://docs.python.org/3/library/stdtypes.html#str.translate - // Make it work for only subscribtable types #[pymethod] fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut result = String::new(); + // It throws a type error if it is not subscribtable + vm.get_method(table.clone(), "__getitem__")?; for c in self.value.chars() { - let args = PyFuncArgs::new(vec![vm.new_int(c as i32)], vec![]); - match vm.call_method(&table, "__getitem__", args) { + match table.get_item(c as i32, vm) { Ok(value) => { if let Some(text) = value.payload::() { result.extend(text.value.chars()); @@ -1145,6 +1145,7 @@ mod tests { let vm = VirtualMachine::new(); let table = vm.context().new_dict(); + // TODO: use table.set_item vm.call_method( table.as_object(), "__setitem__", From b15a58416856f080e86c78c3e2603410016646b3 Mon Sep 17 00:00:00 2001 From: zer0 Date: Fri, 10 May 2019 23:47:11 +0300 Subject: [PATCH 532/658] added small test to snippets --- tests/snippets/strings.py | 3 +++ vm/src/obj/objstr.rs | 26 +++++++------------------- vm/src/obj/objsuper.rs | 6 ++++-- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 3364a293c..7e2cd2ef5 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -157,6 +157,9 @@ assert 'z' > 'b' assert 'z' >= 'b' assert 'a' >= 'a' +# str.translate +assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" + def try_mutate_str(): word = "word" word[0] = 'x' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 4f6589598..683af3484 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1145,25 +1145,13 @@ mod tests { let vm = VirtualMachine::new(); let table = vm.context().new_dict(); - // TODO: use table.set_item - vm.call_method( - table.as_object(), - "__setitem__", - vec![vm.new_int(97), vm.new_str("🎅".to_owned())], - ) - .unwrap(); - vm.call_method( - table.as_object(), - "__setitem__", - vec![vm.new_int(98), vm.get_none()], - ) - .unwrap(); - vm.call_method( - table.as_object(), - "__setitem__", - vec![vm.new_int(99), vm.new_str("xda".to_owned())], - ) - .unwrap(); + table + .set_item(97, vm.new_str("🎅".to_owned()), &vm) + .unwrap(); + table.set_item(98, vm.get_none(), &vm).unwrap(); + table + .set_item(99, vm.new_str("xda".to_owned()), &vm) + .unwrap(); let text = PyString::from("abc"); let translated = text.translate(table.into_object(), &vm).unwrap(); assert_eq!(translated, "🎅xda".to_owned()); diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 94fe0fc7b..194212a49 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -127,8 +127,10 @@ fn super_new( match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { From eab5d42ee6532df0a89dc86e25d382a3a18bf8f2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 05:49:11 +0900 Subject: [PATCH 533/658] Adapt PyClassImpl for PyProperty --- vm/src/obj/objproperty.rs | 173 ++++++++++++++++++++------------------ 1 file changed, 93 insertions(+), 80 deletions(-) diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index 9d7217110..1d28ca6b8 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -7,11 +7,13 @@ use crate::function::OptionalArg; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - IdProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, PyClassImpl, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, }; use crate::vm::VirtualMachine; -/// Read-only property, doesn't have __set__ or __delete__ +// Read-only property, doesn't have __set__ or __delete__ +#[pyclass] #[derive(Debug)] pub struct PyReadOnlyProperty { getter: PyObjectRef, @@ -25,22 +27,56 @@ impl PyValue for PyReadOnlyProperty { pub type PyReadOnlyPropertyRef = PyRef; -impl PyReadOnlyPropertyRef { +#[pyimpl] +impl PyReadOnlyProperty { + #[pymethod(name = "__get__")] fn get( - self, + zelf: PyRef, obj: PyObjectRef, _owner: OptionalArg, vm: &VirtualMachine, ) -> PyResult { if obj.is(vm.ctx.none.as_object()) { - Ok(self.into_object()) + Ok(zelf.into_object()) } else { - vm.invoke(self.getter.clone(), obj) + vm.invoke(zelf.getter.clone(), obj) } } } -/// Fully fledged property +/// Property attribute. +/// +/// fget +/// function to be used for getting an attribute value +/// fset +/// function to be used for setting an attribute value +/// fdel +/// function to be used for del'ing an attribute +/// doc +/// docstring +/// +/// Typical use is to define a managed attribute x: +/// +/// class C(object): +/// def getx(self): return self._x +/// def setx(self, value): self._x = value +/// def delx(self): del self._x +/// x = property(getx, setx, delx, "I'm the 'x' property.") +/// +/// Decorators make defining new properties or modifying existing ones easy: +/// +/// class C(object): +/// @property +/// def x(self): +/// "I am the 'x' property." +/// return self._x +/// @x.setter +/// def x(self, value): +/// self._x = value +/// @x.deleter +/// def x(self): +/// del self._x +#[pyclass] #[derive(Debug)] pub struct PyProperty { getter: Option, @@ -56,7 +92,9 @@ impl PyValue for PyProperty { pub type PyPropertyRef = PyRef; -impl PyPropertyRef { +#[pyimpl] +impl PyProperty { + #[pymethod(name = "__new__")] fn new_property( cls: PyClassRef, fget: OptionalArg, @@ -76,7 +114,7 @@ impl PyPropertyRef { // Descriptor methods // specialised version that doesn't check for None - pub(crate) fn instance_binding_get(self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + pub(crate) fn instance_binding_get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(getter) = self.getter.as_ref() { vm.invoke(getter.clone(), obj) } else { @@ -84,15 +122,16 @@ impl PyPropertyRef { } } + #[pymethod(name = "__get__")] fn get( - self, + zelf: PyRef, obj: PyObjectRef, _owner: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - if let Some(getter) = self.getter.as_ref() { + if let Some(getter) = zelf.getter.as_ref() { if obj.is(vm.ctx.none.as_object()) { - Ok(self.into_object()) + Ok(zelf.into_object()) } else { vm.invoke(getter.clone(), obj) } @@ -101,7 +140,8 @@ impl PyPropertyRef { } } - fn set(self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__set__")] + fn set(&self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(setter) = self.setter.as_ref() { vm.invoke(setter.clone(), vec![obj, value]) } else { @@ -109,7 +149,8 @@ impl PyPropertyRef { } } - fn delete(self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__delete__")] + fn delete(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(deleter) = self.deleter.as_ref() { vm.invoke(deleter.clone(), obj) } else { @@ -119,45 +160,63 @@ impl PyPropertyRef { // Access functions - fn fget(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fget(&self, _vm: &VirtualMachine) -> Option { self.getter.clone() } - fn fset(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fset(&self, _vm: &VirtualMachine) -> Option { self.setter.clone() } - fn fdel(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fdel(&self, _vm: &VirtualMachine) -> Option { self.deleter.clone() } // Python builder functions - fn getter(self, getter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn getter( + zelf: PyRef, + getter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: getter.or_else(|| self.getter.clone()), - setter: self.setter.clone(), - deleter: self.deleter.clone(), + getter: getter.or_else(|| zelf.getter.clone()), + setter: zelf.setter.clone(), + deleter: zelf.deleter.clone(), } - .into_ref_with_type(vm, TypeProtocol::class(&self)) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } - fn setter(self, setter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn setter( + zelf: PyRef, + setter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: self.getter.clone(), - setter: setter.or_else(|| self.setter.clone()), - deleter: self.deleter.clone(), + getter: zelf.getter.clone(), + setter: setter.or_else(|| zelf.setter.clone()), + deleter: zelf.deleter.clone(), } - .into_ref_with_type(vm, TypeProtocol::class(&self)) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } - fn deleter(self, deleter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn deleter( + zelf: PyRef, + deleter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: self.getter.clone(), - setter: self.setter.clone(), - deleter: deleter.or_else(|| self.deleter.clone()), + getter: zelf.getter.clone(), + setter: zelf.setter.clone(), + deleter: deleter.or_else(|| zelf.deleter.clone()), } - .into_ref_with_type(vm, TypeProtocol::class(&self)) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } } @@ -216,52 +275,6 @@ impl<'a> PropertyBuilder<'a> { } pub fn init(context: &PyContext) { - extend_class!(context, &context.readonly_property_type, { - "__get__" => context.new_rustfunc(PyReadOnlyPropertyRef::get), - }); - - let property_doc = - "Property attribute.\n\n \ - fget\n \ - function to be used for getting an attribute value\n \ - fset\n \ - function to be used for setting an attribute value\n \ - fdel\n \ - function to be used for del\'ing an attribute\n \ - doc\n \ - docstring\n\n\ - Typical use is to define a managed attribute x:\n\n\ - class C(object):\n \ - def getx(self): return self._x\n \ - def setx(self, value): self._x = value\n \ - def delx(self): del self._x\n \ - x = property(getx, setx, delx, \"I\'m the \'x\' property.\")\n\n\ - Decorators make defining new properties or modifying existing ones easy:\n\n\ - class C(object):\n \ - @property\n \ - def x(self):\n \"I am the \'x\' property.\"\n \ - return self._x\n \ - @x.setter\n \ - def x(self, value):\n \ - self._x = value\n \ - @x.deleter\n \ - def x(self):\n \ - del self._x"; - - extend_class!(context, &context.property_type, { - "__new__" => context.new_rustfunc(PyPropertyRef::new_property), - "__doc__" => context.new_str(property_doc.to_string()), - - "__get__" => context.new_rustfunc(PyPropertyRef::get), - "__set__" => context.new_rustfunc(PyPropertyRef::set), - "__delete__" => context.new_rustfunc(PyPropertyRef::delete), - - "fget" => context.new_property(PyPropertyRef::fget), - "fset" => context.new_property(PyPropertyRef::fset), - "fdel" => context.new_property(PyPropertyRef::fdel), - - "getter" => context.new_rustfunc(PyPropertyRef::getter), - "setter" => context.new_rustfunc(PyPropertyRef::setter), - "deleter" => context.new_rustfunc(PyPropertyRef::deleter), - }); + PyReadOnlyProperty::extend_class(context, &context.readonly_property_type); + PyProperty::extend_class(context, &context.property_type); } From 268164e957cd1e7afb3be1460a56798de392a753 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 06:02:56 +0900 Subject: [PATCH 534/658] Fix property to take keyword arguments --- tests/snippets/property.py | 6 ++++++ vm/src/obj/objproperty.rs | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/snippets/property.py b/tests/snippets/property.py index c61f13189..da2c1ee63 100644 --- a/tests/snippets/property.py +++ b/tests/snippets/property.py @@ -59,6 +59,8 @@ with assertRaises(AttributeError): with assertRaises(TypeError): property.__new__(object) +# assert p.__doc__ is None + p1 = property("a", "b", "c") @@ -75,3 +77,7 @@ assert p1.setter(None).fset == "b" assert p1.deleter(None).fdel == "c" assert p1.__get__(None, object) is p1 +# assert p1.__doc__ is 'a'.__doc__ + +p2 = property('a', doc='pdoc') +# assert p2.__doc__ == 'pdoc' diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index 1d28ca6b8..9c8394a4c 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -2,9 +2,7 @@ */ -use crate::function::IntoPyNativeFunc; -use crate::function::OptionalArg; -use crate::obj::objstr::PyStringRef; +use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ IdProtocol, PyClassImpl, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, @@ -82,6 +80,7 @@ pub struct PyProperty { getter: Option, setter: Option, deleter: Option, + doc: Option, } impl PyValue for PyProperty { @@ -97,16 +96,31 @@ impl PyProperty { #[pymethod(name = "__new__")] fn new_property( cls: PyClassRef, - fget: OptionalArg, - fset: OptionalArg, - fdel: OptionalArg, - _doc: OptionalArg, + args: PyFuncArgs, vm: &VirtualMachine, ) -> PyResult { + arg_check!( + vm, + args, + required = [], + optional = [(fget, None), (fset, None), (fdel, None), (doc, None)] + ); + + fn into_option(vm: &VirtualMachine, arg: Option<&PyObjectRef>) -> Option { + arg.and_then(|arg| { + if vm.ctx.none().is(arg) { + None + } else { + Some(arg.clone()) + } + }) + } + PyProperty { - getter: fget.into_option(), - setter: fset.into_option(), - deleter: fdel.into_option(), + getter: into_option(vm, fget), + setter: into_option(vm, fset), + deleter: into_option(vm, fdel), + doc: into_option(vm, doc), } .into_ref_with_type(vm, cls) } @@ -187,6 +201,7 @@ impl PyProperty { getter: getter.or_else(|| zelf.getter.clone()), setter: zelf.setter.clone(), deleter: zelf.deleter.clone(), + doc: None, } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } @@ -201,6 +216,7 @@ impl PyProperty { getter: zelf.getter.clone(), setter: setter.or_else(|| zelf.setter.clone()), deleter: zelf.deleter.clone(), + doc: None, } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } @@ -215,6 +231,7 @@ impl PyProperty { getter: zelf.getter.clone(), setter: zelf.setter.clone(), deleter: deleter.or_else(|| zelf.deleter.clone()), + doc: None, } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } @@ -259,6 +276,7 @@ impl<'a> PropertyBuilder<'a> { getter: self.getter.clone(), setter: self.setter.clone(), deleter: None, + doc: None, }; PyObject::new(payload, self.ctx.property_type(), None) From 6e057955213fd01559bb4fb0301d692745c6c6de Mon Sep 17 00:00:00 2001 From: zer0 Date: Sat, 11 May 2019 01:03:22 +0300 Subject: [PATCH 535/658] added str.maketrans --- parser/src/lexer.rs | 6 +++- tests/snippets/strings.py | 4 +++ vm/src/obj/objstr.rs | 61 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index d23198ca8..11d30cbee 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -826,7 +826,11 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + tok_end, + ))); } } } diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 7e2cd2ef5..480500e9f 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -160,6 +160,10 @@ assert 'a' >= 'a' # str.translate assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" +# str.maketrans +assert str.maketrans({"a": "abc", "b": None, "c": 33}) == {97: "abc", 98: None, 99: 33} +assert str.maketrans("hello", "world", "rust") == {103: "w", 101: "o", 108: "l", 111: "d", "r": None, "u": None, "s": None, "t": None} + def try_mutate_str(): word = "word" word[0] = 'x' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 683af3484..a4e6da8c5 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -16,6 +16,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; +use super::objdict::{PyDict, PyDictRef}; use super::objint::{self, PyInt}; use super::objnone::PyNone; use super::objsequence::PySliceableSequence; @@ -838,7 +839,7 @@ impl PyString { // It throws a type error if it is not subscribtable vm.get_method(table.clone(), "__getitem__")?; for c in self.value.chars() { - match table.get_item(c as i32, vm) { + match table.get_item(c as u32, vm) { Ok(value) => { if let Some(text) = value.payload::() { result.extend(text.value.chars()); @@ -864,6 +865,64 @@ impl PyString { } Ok(result) } + + #[pymethod] + fn maketrans( + dict_or_str: PyObjectRef, + to_str: OptionalArg, + none_str: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + if let OptionalArg::Present(to_str) = to_str { + // dict_or_str is str + let from_str = dict_or_str.payload::().ok_or( + vm.new_type_error( + "first maketrans argument must be a string if there is a second argument" + .to_owned(), + ), + )?; + let new_dict = vm.context().new_dict(); + if to_str.len(vm) == from_str.len(vm) { + for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { + new_dict.set_item(c1.to_string(), vm.new_str(c2.to_string()), vm)?; + } + } else { + return Err(vm.new_value_error( + "the first two maketrans arguments must have equal length".to_owned(), + )); + } + if let OptionalArg::Present(none_str) = none_str { + for c in none_str.value.chars() { + new_dict.set_item(c.to_string(), vm.get_none(), vm)?; + } + } + new_dict.into_pyobject(vm) + } else { + // dict_str must be a dict + if let Ok(dict) = dict_or_str.downcast::() { + let new_dict = vm.context().new_dict(); + for (key, val) in dict { + if let Some(num) = key.payload::() { + new_dict.set_item(num.as_bigint().to_i32(), val, vm)?; + } else if let Some(string) = key.payload::() { + if string.len(vm) == 1 { + let num_value = string.value.chars().next().unwrap() as u32; + new_dict.set_item(num_value, val, vm)?; + } else { + return Err(vm.new_value_error( + "string keys in translate table must be of length 1".to_owned(), + )); + } + } + } + new_dict.into_pyobject(vm) + } else { + Err(vm.new_value_error( + "if you give only one argument to maketrans it must be a dict".to_owned(), + )) + } + } + } } impl PyValue for PyString { From c0f14eace3e0530c78eac5c476f3eed41572db77 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 03:12:36 +0900 Subject: [PATCH 536/658] Fix str.isidentifier() and add tests snippets --- Cargo.lock | 1 + tests/snippets/strings.py | 7 +++++++ vm/Cargo.toml | 1 + vm/src/obj/objstr.rs | 23 ++++++++++------------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33e9ddcfd..0303e06e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -849,6 +849,7 @@ dependencies = [ "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 28df3f47f..0cb3782b1 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -164,6 +164,13 @@ assert not ''.isdecimal() assert '123'.isdecimal() assert not '\u00B2'.isdecimal() +assert not ''.isidentifier() +assert 'python'.isidentifier() +assert '_'.isidentifier() +assert '유니코드'.isidentifier() +assert not '😂'.isidentifier() +assert not '123'.isidentifier() + # String Formatting assert "{} {}".format(1,2) == "1 2" assert "{0} {1}".format(2,3) == "2 3" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5af648084..8ac5ebabb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -24,6 +24,7 @@ rustc_version_runtime = "0.1.*" statrs = "0.10.0" caseless = "0.2.1" unicode-segmentation = "1.2.1" +unicode-xid = "0.1.0" lazy_static = "^1.0.1" lexical = "2.0.0" itertools = "^0.8.0" diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index e5e2afe37..7b2fcbe9a 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,3 +1,5 @@ +extern crate unicode_xid; + use std::fmt; use std::hash::{Hash, Hasher}; use std::ops::Range; @@ -7,6 +9,7 @@ use std::string::ToString; use num_traits::ToPrimitive; use unicode_casing::CharExt; use unicode_segmentation::UnicodeSegmentation; +use unicode_xid::UnicodeXID; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; @@ -832,20 +835,14 @@ impl PyString { #[pymethod] fn isidentifier(&self, _vm: &VirtualMachine) -> bool { - let value = &self.value; + let mut chars = self.value.chars(); + let is_identifier_start = match chars.next() { + Some('_') => true, + Some(c) => UnicodeXID::is_xid_start(c), + None => false, + }; // a string is not an identifier if it has whitespace or starts with a number - if !value.chars().any(|c| c.is_ascii_whitespace()) - && !value.chars().nth(0).unwrap().is_digit(10) - { - for c in value.chars() { - if c != "_".chars().nth(0).unwrap() && !c.is_digit(10) && !c.is_alphabetic() { - return false; - } - } - true - } else { - false - } + is_identifier_start && chars.all(|c| UnicodeXID::is_xid_continue(c)) } } From 8294ac54513700ab05ae353e2d80104b41e9b717 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 09:17:23 +0300 Subject: [PATCH 537/658] Add os.getcwd --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e68873767..ad5ecf9d1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -479,6 +479,15 @@ fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResu unimplemented!(); } +fn os_getcwd(vm: &VirtualMachine) -> PyResult { + Ok(env::current_dir() + .map_err(|s| vm.new_os_error(s.to_string()))? + .as_path() + .to_str() + .unwrap() + .to_string()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -534,6 +543,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "stat_result" => stat_result, "stat" => ctx.new_rustfunc(os_stat), "symlink" => ctx.new_rustfunc(os_symlink), + "getcwd" => ctx.new_rustfunc(os_getcwd), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From f0fd371b690464313066435cac482c6a5b7fa555 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 09:29:09 +0300 Subject: [PATCH 538/658] Add os.chdir --- vm/src/stdlib/os.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index ad5ecf9d1..c01b5aaac 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -488,6 +488,10 @@ fn os_getcwd(vm: &VirtualMachine) -> PyResult { .to_string()) } +fn os_chdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + env::set_current_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -544,6 +548,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "stat" => ctx.new_rustfunc(os_stat), "symlink" => ctx.new_rustfunc(os_symlink), "getcwd" => ctx.new_rustfunc(os_getcwd), + "chdir" => ctx.new_rustfunc(os_chdir), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From c7e9bdd4640de5345c04e97965caba3c80eacf16 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 09:29:24 +0300 Subject: [PATCH 539/658] Add tests for chdir and getcwd --- tests/snippets/stdlib_os.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 26a101ecb..39d5e7bdc 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -65,6 +65,14 @@ class TestWithTempDir(): pass +class TestWithTempCurrentDir(): + def __enter__(self): + self.prev_cwd = os.getcwd() + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.prev_cwd) + + FILE_NAME = "test1" FILE_NAME2 = "test2" SYMLINK_FILE = "symlink" @@ -161,3 +169,8 @@ with TestWithTempDir() as tmpdir: assert os.path.basename(fname) == FILE_NAME assert os.path.dirname(fname) == tmpdir + + with TestWithTempCurrentDir(): + os.chdir(tmpdir) + assert os.getcwd() == tmpdir + os.path.exists(FILE_NAME) From b01e960da711dfc612a83205ed389ea7b5b830d1 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 11 May 2019 19:15:54 +1200 Subject: [PATCH 540/658] Add st_{a,m,c}time to os.stat --- tests/snippets/stdlib_os.py | 7 ++ vm/src/stdlib/os.rs | 144 ++++++++++++++++++++++++++---------- 2 files changed, 111 insertions(+), 40 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 26a101ecb..9317a8e82 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -143,6 +143,13 @@ with TestWithTempDir() as tmpdir: print(stat_res.st_gid) print(stat_res.st_size) assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + # test that it all of these times are greater than the 10 May 2019, when this test was written + assert stat_res.st_atime > 1557500000 + assert stat_res.st_ctime > 1557500000 + assert stat_res.st_mtime > 1557500000 # stat default is follow_symlink=True os.stat(fname).st_ino == os.stat(symlink_file).st_ino diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e68873767..e1b44839d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,7 +1,8 @@ use std::cell::RefCell; use std::fs::File; use std::fs::OpenOptions; -use std::io::{ErrorKind, Read, Write}; +use std::io::{self, ErrorKind, Read, Write}; +use std::time::{Duration, SystemTime}; use std::{env, fs}; use num_traits::cast::ToPrimitive; @@ -261,7 +262,7 @@ impl DirEntryRef { .is_symlink()) } - fn stat(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + fn stat(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { os_stat(self.path(vm).try_into_ref(vm)?, follow_symlinks, vm) } } @@ -317,6 +318,9 @@ struct StatResult { st_uid: u32, st_gid: u32, st_size: u64, + st_atime: f64, + st_ctime: f64, + st_mtime: f64, } impl PyValue for StatResult { @@ -355,47 +359,93 @@ impl StatResultRef { fn st_size(self, _vm: &VirtualMachine) -> u64 { self.st_size } + + fn st_atime(self, _vm: &VirtualMachine) -> f64 { + self.st_atime + } + + fn st_ctime(self, _vm: &VirtualMachine) -> f64 { + self.st_ctime + } + + fn st_mtime(self, _vm: &VirtualMachine) -> f64 { + self.st_mtime + } +} + +// Copied code from Duration::as_secs_f64 as it's still unstable +fn duration_as_secs_f64(duration: Duration) -> f64 { + (duration.as_secs() as f64) + (duration.subsec_nanos() as f64) / (1_000_000_000 as f64) +} + +fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { + match sys_time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(duration) => duration_as_secs_f64(duration), + Err(err) => -duration_as_secs_f64(err.duration()), + } +} + +fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { + let duration = Duration::new(secs as u64, nanos as u32); + duration_as_secs_f64(duration) } #[cfg(unix)] macro_rules! os_unix_stat_inner { - ( $path:expr, $follow_symlinks:expr, $vm:expr) => {{ - let metadata = match $follow_symlinks.follow_symlinks { - true => fs::metadata($path), - false => fs::symlink_metadata($path), - }; - let meta = metadata.map_err(|s| $vm.new_os_error(s.to_string()))?; + ( $path:expr, $follow_symlinks:expr, $vm:expr ) => {{ + fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { + let meta = match follow_symlinks { + true => fs::metadata(path)?, + false => fs::symlink_metadata(path)?, + }; - Ok(StatResult { - st_mode: meta.st_mode(), - st_ino: meta.st_ino(), - st_dev: meta.st_dev(), - st_nlink: meta.st_nlink(), - st_uid: meta.st_uid(), - st_gid: meta.st_gid(), - st_size: meta.st_size(), + Ok(StatResult { + st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), + st_size: meta.st_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_nanos(meta.st_ctime(), meta.st_ctime_nsec()), + }) } - .into_ref($vm) - .into_object()) + + get_stats(&$path.value, $follow_symlinks.follow_symlinks) + .map_err(|s| $vm.new_os_error(s.to_string())) }}; } #[cfg(target_os = "linux")] -fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { +fn os_stat( + path: PyStringRef, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { use std::os::linux::fs::MetadataExt; - os_unix_stat_inner!(&path.value, follow_symlinks, vm) + os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "macos")] -fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { +fn os_stat( + path: PyStringRef, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { use std::os::macos::fs::MetadataExt; - os_unix_stat_inner!(&path.value, follow_symlinks, vm) + os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "android")] -fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { +fn os_stat( + path: PyStringRef, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { use std::os::android::fs::MetadataExt; - os_unix_stat_inner!(&path.value, follow_symlinks, vm) + os_unix_stat_inner!(path, follow_symlinks, vm) } // Copied from CPython fileutils.c @@ -420,24 +470,35 @@ fn attributes_to_mode(attr: u32) -> u32 { } #[cfg(windows)] -fn os_stat(path: PyStringRef, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { +fn os_stat( + path: PyStringRef, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { use std::os::windows::fs::MetadataExt; - let metadata = match follow_symlinks.follow_symlinks { - true => fs::metadata(&path.value), - false => fs::symlink_metadata(&path.value), - }; - let meta = metadata.map_err(|s| vm.new_os_error(s.to_string()))?; - Ok(StatResult { - st_mode: attributes_to_mode(meta.file_attributes()), - st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. - st_uid: 0, // 0 on windows - st_gid: 0, // 0 on windows - st_size: meta.file_size(), + + fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { + let meta = match follow_symlinks { + true => fs::metadata(path)?, + false => fs::symlink_metadata(path)?, + }; + + Ok(StatResult { + st_mode: attributes_to_mode(meta.file_attributes()), + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_uid: 0, // 0 on windows + st_gid: 0, // 0 on windows + st_size: meta.file_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_unix_epoch(meta.created()?), + }) } - .into_ref(vm) - .into_object()) + + get_stats(&path.value, follow_symlinks.follow_symlinks) + .map_err(|s| vm.new_os_error(s.to_string())) } #[cfg(not(any( @@ -510,6 +571,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "st_uid" => ctx.new_property(StatResultRef::st_uid), "st_gid" => ctx.new_property(StatResultRef::st_gid), "st_size" => ctx.new_property(StatResultRef::st_size), + "st_atime" => ctx.new_property(StatResultRef::st_atime), + "st_ctime" => ctx.new_property(StatResultRef::st_ctime), + "st_mtime" => ctx.new_property(StatResultRef::st_mtime), }); py_module!(vm, "_os", { From 152c1654d6e5138f626f0614594638fc77054d89 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 11:43:11 +0300 Subject: [PATCH 541/658] Add sys.platform --- tests/snippets/sysmod.py | 2 ++ vm/src/sysmodule.rs | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index bfcf17201..b7cdc22ec 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -2,3 +2,5 @@ import sys print(sys.argv) assert sys.argv[0].endswith('.py') + +assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "win32" or sys.platform == "unknown" diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a5bf2a6f4..4174fcf10 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -62,6 +62,19 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR }; let path = ctx.new_list(path_list); + let platform = if cfg!(target_os = "linux") { + "linux".to_string() + } else if cfg!(target_os = "macos") { + "darwin".to_string() + } else if cfg!(target_os = "windows") { + "win32".to_string() + } else if cfg!(target_os = "android") { + // Linux as well. see https://bugs.python.org/issue32637 + "linux".to_string() + } else { + "unknown".to_string() + }; + let sys_doc = "This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. @@ -145,6 +158,7 @@ settrace() -- set the global debug tracing function "_getframe" => ctx.new_rustfunc(getframe), "modules" => modules.clone(), "warnoptions" => ctx.new_list(vec![]), + "platform" => ctx.new_str(platform), }); modules.set_item("sys", module.clone(), vm).unwrap(); From 7490724b65536140fe11fd51a13a3642a6b0b32c Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 10 May 2019 11:35:12 +0300 Subject: [PATCH 542/658] Add `itertools` module skeleton --- tests/snippets/stdlib_itertools.py | 1 + vm/src/stdlib/itertools.rs | 7 +++++++ vm/src/stdlib/mod.rs | 2 ++ 3 files changed, 10 insertions(+) create mode 100644 tests/snippets/stdlib_itertools.py create mode 100644 vm/src/stdlib/itertools.rs diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py new file mode 100644 index 000000000..994a79fbb --- /dev/null +++ b/tests/snippets/stdlib_itertools.py @@ -0,0 +1 @@ +import itertools diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs new file mode 100644 index 000000000..f37c9c5f8 --- /dev/null +++ b/vm/src/stdlib/itertools.rs @@ -0,0 +1,7 @@ +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + py_module!(vm, "itertools", { + }) +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 0933b7192..bd59ef9c6 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,5 +1,6 @@ mod ast; mod dis; +mod itertools; pub(crate) mod json; mod keyword; mod math; @@ -34,6 +35,7 @@ pub fn get_module_inits() -> HashMap { Box::new(ast::make_module) as StdlibInitFunc, ); modules.insert("dis".to_string(), Box::new(dis::make_module)); + modules.insert("itertools".to_string(), Box::new(itertools::make_module)); modules.insert("json".to_string(), Box::new(json::make_module)); modules.insert("keyword".to_string(), Box::new(keyword::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); From ce514d2aa5dcc30f8fa4e744adc8eb537b7776e3 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 10 May 2019 17:27:28 +0300 Subject: [PATCH 543/658] Add `itertools.count` --- tests/snippets/stdlib_itertools.py | 54 ++++++++++++++++++++++++ vm/src/stdlib/itertools.rs | 68 +++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index 994a79fbb..cfda2fb68 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -1 +1,55 @@ import itertools + +# count + +# default arguments +c = itertools.count() +assert next(c) == 0 +assert next(c) == 1 +assert next(c) == 2 + +# positional +c = itertools.count(2, 3) +assert next(c) == 2 +assert next(c) == 5 +assert next(c) == 8 + +# backwards +c = itertools.count(1, -10) +assert next(c) == 1 +assert next(c) == -9 +assert next(c) == -19 + +# step = 0 +c = itertools.count(5, 0) +assert next(c) == 5 +assert next(c) == 5 + +# itertools.count TODOs: kwargs and floats + +# step kwarg +# c = itertools.count(step=5) +# assert next(c) == 0 +# assert next(c) == 5 + +# start kwarg +# c = itertools.count(start=10) +# assert next(c) == 10 + +# float start +# c = itertools.count(0.5) +# assert next(c) == 0.5 +# assert next(c) == 1.5 +# assert next(c) == 2.5 + +# float step +# c = itertools.count(1, 0.5) +# assert next(c) == 1 +# assert next(c) == 1.5 +# assert next(c) == 2 + +# float start + step +# c = itertools.count(0.5, 0.5) +# assert next(c) == 0.5 +# assert next(c) == 1 +# assert next(c) == 1.5 diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index f37c9c5f8..a7a955fa5 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -1,7 +1,73 @@ -use crate::pyobject::PyObjectRef; +use std::cell::RefCell; +use std::ops::AddAssign; + +use num_bigint::BigInt; + +use crate::function::OptionalArg; +use crate::obj::objint::{PyInt, PyIntRef}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass] +#[derive(Debug)] +struct PyItertoolsCount { + cur: RefCell, + step: BigInt, +} + +impl PyValue for PyItertoolsCount { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "count") + } +} + +#[pyimpl] +impl PyItertoolsCount { + #[pymethod(name = "__new__")] + fn new( + _cls: PyClassRef, + start: OptionalArg, + step: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let start = match start.into_option() { + Some(int) => int.as_bigint().clone(), + None => BigInt::from(0), + }; + let step = match step.into_option() { + Some(int) => int.as_bigint().clone(), + None => BigInt::from(1), + }; + + Ok(PyItertoolsCount { + cur: RefCell::new(start), + step: step, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, _vm: &VirtualMachine) -> PyResult { + let result = self.cur.borrow().clone(); + AddAssign::add_assign(&mut self.cur.borrow_mut() as &mut BigInt, &self.step); + Ok(PyInt::new(result)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let count = ctx.new_class("count", ctx.object()); + PyItertoolsCount::extend_class(ctx, &count); + py_module!(vm, "itertools", { + "count" => count, }) } From f7531d8b084ed37f1e95c7f0c30752b6a341facb Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 10 May 2019 23:54:35 +0300 Subject: [PATCH 544/658] Add `binascii` module skeleton --- tests/snippets/stdlib_binascii.py | 1 + vm/src/stdlib/binascii.rs | 7 +++++++ vm/src/stdlib/mod.rs | 2 ++ 3 files changed, 10 insertions(+) create mode 100644 tests/snippets/stdlib_binascii.py create mode 100644 vm/src/stdlib/binascii.rs diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py new file mode 100644 index 000000000..35c5e9f0a --- /dev/null +++ b/tests/snippets/stdlib_binascii.py @@ -0,0 +1 @@ +import binascii diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs new file mode 100644 index 000000000..b0675a1a9 --- /dev/null +++ b/vm/src/stdlib/binascii.rs @@ -0,0 +1,7 @@ +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + py_module!(vm, "binascii", { + }) +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 0933b7192..c17a270ac 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,4 +1,5 @@ mod ast; +mod binascii; mod dis; pub(crate) mod json; mod keyword; @@ -33,6 +34,7 @@ pub fn get_module_inits() -> HashMap { "ast".to_string(), Box::new(ast::make_module) as StdlibInitFunc, ); + modules.insert("binascii".to_string(), Box::new(binascii::make_module)); modules.insert("dis".to_string(), Box::new(dis::make_module)); modules.insert("json".to_string(), Box::new(json::make_module)); modules.insert("keyword".to_string(), Box::new(keyword::make_module)); From 8dd664ef2013c4bb0c16cf71da2f89a8272f074c Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 11 May 2019 00:29:42 +0300 Subject: [PATCH 545/658] Add `binascii.hexlify` --- tests/snippets/stdlib_binascii.py | 13 +++++++++++++ vm/src/stdlib/binascii.rs | 28 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index 35c5e9f0a..cee356462 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -1 +1,14 @@ import binascii +from testutils import assertRaises + + +# hexlify tests +h = binascii.hexlify + +assert h(b"abc") == b"616263" +assert h(1000 * b"x") == 1000 * b"78" +# bytearray not supported yet +# assert h(bytearray(b"a")) = b"61" + +with assertRaises(TypeError): + h("a") diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index b0675a1a9..166376b57 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -1,7 +1,33 @@ -use crate::pyobject::PyObjectRef; +use crate::function::PyFuncArgs; +use crate::obj::objbytes; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +fn hex_nibble(n: u8) -> u8 { + match n { + 0..=9 => b'0' + n, + 10..=15 => b'a' + n, + _ => unreachable!(), + } +} + +fn binascii_hexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(data, Some(vm.ctx.bytes_type()))]); + + let bytes = objbytes::get_value(data); + let mut hex = Vec::::with_capacity(bytes.len() * 2); + for b in bytes.iter() { + hex.push(hex_nibble(b >> 4)); + hex.push(hex_nibble(b & 0xf)); + } + + Ok(vm.ctx.new_bytes(hex)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "binascii", { + "hexlify" => ctx.new_rustfunc(binascii_hexlify), }) } From 070529d8c7b2e3f4fb9d6cfb04a89db96ac9ac54 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 11 May 2019 02:16:32 +0300 Subject: [PATCH 546/658] Add `binascii.unhexlify` --- tests/snippets/stdlib_binascii.py | 19 ++++++++++++++++++ vm/src/stdlib/binascii.rs | 33 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index cee356462..418fad36d 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -12,3 +12,22 @@ assert h(1000 * b"x") == 1000 * b"78" with assertRaises(TypeError): h("a") + + +# unhexlify tests +uh = binascii.unhexlify + +assert uh(b"616263") == b"abc" +assert uh(1000 * b"78") == 1000 * b"x" +x = 1000 * b"1234" +assert uh(h(x)) == x +assert uh(b"ABCDEF") == b"\xab\xcd\xef" + +# unhexlify on strings not supported yet +# assert uh("abcd") == b"\xab\xcd" + +with assertRaises(ValueError): + uh(b"a") # Odd-length string + +with assertRaises(ValueError): + uh(b"nn") # Non-hexadecimal digit found diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 166376b57..3297c5e1a 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -24,10 +24,43 @@ fn binascii_hexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(hex)) } +fn unhex_nibble(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +fn binascii_unhexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: allow 'str' hexstrings as well + arg_check!(vm, args, required = [(hexstr, Some(vm.ctx.bytes_type()))]); + + let hex_bytes = &objbytes::get_value(hexstr); + if hex_bytes.len() % 2 != 0 { + return Err(vm.new_value_error("Odd-length string".to_string())); + } + + let mut unhex = Vec::::with_capacity(hex_bytes.len() / 2); + for i in (0..hex_bytes.len()).step_by(2) { + let n1 = unhex_nibble(hex_bytes[i]); + let n2 = unhex_nibble(hex_bytes[i + 1]); + if n1.is_some() && n2.is_some() { + unhex.push(n1.unwrap() << 4 | n2.unwrap()); + } else { + return Err(vm.new_value_error("Non-hexadecimal digit found".to_string())); + } + } + + Ok(vm.ctx.new_bytes(unhex)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "binascii", { "hexlify" => ctx.new_rustfunc(binascii_hexlify), + "unhexlify" => ctx.new_rustfunc(binascii_unhexlify), }) } From 322aff70df0d2dece219c178d32ea9e1893c4641 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 11 May 2019 02:19:10 +0300 Subject: [PATCH 547/658] Add `binascii.b2a_hex`, `binascii.a2b_hex` --- tests/snippets/stdlib_binascii.py | 2 ++ vm/src/stdlib/binascii.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index 418fad36d..e73e162bd 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -9,6 +9,7 @@ assert h(b"abc") == b"616263" assert h(1000 * b"x") == 1000 * b"78" # bytearray not supported yet # assert h(bytearray(b"a")) = b"61" +assert binascii.b2a_hex(b"aa") == b"6161" with assertRaises(TypeError): h("a") @@ -22,6 +23,7 @@ assert uh(1000 * b"78") == 1000 * b"x" x = 1000 * b"1234" assert uh(h(x)) == x assert uh(b"ABCDEF") == b"\xab\xcd\xef" +assert binascii.a2b_hex(b"6161") == b"aa" # unhexlify on strings not supported yet # assert uh("abcd") == b"\xab\xcd" diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 3297c5e1a..97fb76f8e 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -61,6 +61,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { py_module!(vm, "binascii", { "hexlify" => ctx.new_rustfunc(binascii_hexlify), + "b2a_hex" => ctx.new_rustfunc(binascii_hexlify), "unhexlify" => ctx.new_rustfunc(binascii_unhexlify), + "a2b_hex" => ctx.new_rustfunc(binascii_unhexlify), }) } From 12e831380d867b9a4fbf09010b40146908610987 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 10 May 2019 18:36:17 +0300 Subject: [PATCH 548/658] Add `itertools.repeat` --- tests/snippets/stdlib_itertools.py | 31 +++++++++++++- vm/src/stdlib/itertools.rs | 66 +++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index cfda2fb68..61c031dd7 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -1,6 +1,9 @@ import itertools -# count +from testutils import assertRaises + + +# itertools.count tests # default arguments c = itertools.count() @@ -53,3 +56,29 @@ assert next(c) == 5 # assert next(c) == 0.5 # assert next(c) == 1 # assert next(c) == 1.5 + + +# itertools.repeat tests + +# no times +r = itertools.repeat(5) +assert next(r) == 5 +assert next(r) == 5 +assert next(r) == 5 + +# times +r = itertools.repeat(1, 2) +assert next(r) == 1 +assert next(r) == 1 +with assertRaises(StopIteration): + next(r) + +# timees = 0 +r = itertools.repeat(1, 0) +with assertRaises(StopIteration): + next(r) + +# negative times +r = itertools.repeat(1, -1) +with assertRaises(StopIteration): + next(r) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index a7a955fa5..3856fe92f 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -1,10 +1,12 @@ use std::cell::RefCell; -use std::ops::AddAssign; +use std::cmp::Ordering; +use std::ops::{AddAssign, SubAssign}; use num_bigint::BigInt; use crate::function::OptionalArg; use crate::obj::objint::{PyInt, PyIntRef}; +use crate::obj::objiter::new_stop_iteration; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; @@ -61,13 +63,75 @@ impl PyItertoolsCount { } } +#[pyclass] +#[derive(Debug)] +struct PyItertoolsRepeat { + object: PyObjectRef, + times: Option>, +} + +impl PyValue for PyItertoolsRepeat { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "repeat") + } +} + +#[pyimpl] +impl PyItertoolsRepeat { + #[pymethod(name = "__new__")] + fn new( + _cls: PyClassRef, + object: PyObjectRef, + times: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let times = match times.into_option() { + Some(int) => Some(RefCell::new(int.as_bigint().clone())), + None => None, + }; + + Ok(PyItertoolsRepeat { + object: object.clone(), + times: times, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.times.is_some() { + match self.times.as_ref().unwrap().borrow().cmp(&BigInt::from(0)) { + Ordering::Less | Ordering::Equal => return Err(new_stop_iteration(vm)), + _ => (), + }; + + SubAssign::sub_assign( + &mut self.times.as_ref().unwrap().borrow_mut() as &mut BigInt, + &BigInt::from(1), + ); + } + + Ok(self.object.clone()) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let count = ctx.new_class("count", ctx.object()); PyItertoolsCount::extend_class(ctx, &count); + let repeat = ctx.new_class("repeat", ctx.object()); + PyItertoolsRepeat::extend_class(ctx, &repeat); + py_module!(vm, "itertools", { "count" => count, + "repeat" => repeat, }) } From d1c95d1c513a206f02d6b71eedba40a1671ddc02 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 17:17:00 +0300 Subject: [PATCH 549/658] Remove patch in posixpath.py --- Lib/posixpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index ae54f2387..ecb4e5a8f 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -444,7 +444,7 @@ def _joinrealpath(path, rest, seen): return path, True -supports_unicode_filenames = (hasattr(sys, "platform") and sys.platform == 'darwin') +supports_unicode_filenames = (sys.platform == 'darwin') def relpath(path, start=None): """Return a relative version of a path""" From b54354860f865f4acb175bda9bbb94fc26def6e3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 18:06:06 +0300 Subject: [PATCH 550/658] Move fspath to _os --- Lib/os.py | 31 ------------------------------- vm/src/stdlib/os.rs | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index c768946c5..cb49481df 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -133,34 +133,3 @@ def getenv(key, default=None): The optional second argument can specify an alternate default. key, default and the result are str.""" return environ.get(key, default) - - -def fspath(path): - """Return the path representation of a path-like object. - - If str or bytes is passed in, it is returned unchanged. Otherwise the - os.PathLike interface is used to get the path representation. If the - path representation is not str or bytes, TypeError is raised. If the - provided path is not str, bytes, or os.PathLike, TypeError is raised. - """ - if isinstance(path, (str, bytes)): - return path - - # Work from the object's type to match method resolution of other magic - # methods. - path_type = type(path) - try: - path_repr = path_type.__fspath__(path) - except AttributeError: - if hasattr(path_type, '__fspath__'): - raise - else: - raise TypeError("expected str, bytes or os.PathLike object, " - "not " + path_type.__name__) - if isinstance(path_repr, (str, bytes)): - return path_repr - else: - raise TypeError("expected {}.__fspath__() to return str or bytes, " - "not {}".format(path_type.__name__, - type(path_repr).__name__)) - diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c01b5aaac..a69a97cc2 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -14,9 +14,10 @@ use crate::obj::objint::PyIntRef; use crate::obj::objiter; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; +use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, + ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -492,6 +493,19 @@ fn os_chdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { env::set_current_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } +fn os_fspath(path: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::issubclass(&path.class(), &vm.ctx.str_type()) + || objtype::issubclass(&path.class(), &vm.ctx.bytes_type()) + { + Ok(path) + } else { + Err(vm.new_type_error(format!( + "expected str or bytes object, not {}", + path.class() + ))) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -549,6 +563,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "symlink" => ctx.new_rustfunc(os_symlink), "getcwd" => ctx.new_rustfunc(os_getcwd), "chdir" => ctx.new_rustfunc(os_chdir), + "fspath" => ctx.new_rustfunc(os_fspath), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), From d96d04f0fde2bb4b567bac75d70ebf0c53cf8069 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 11 May 2019 18:08:47 +0300 Subject: [PATCH 551/658] Add os.fspath tests --- tests/snippets/stdlib_os.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 39d5e7bdc..32cb7d4a9 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -49,6 +49,10 @@ else: assert os.altsep == None assert os.pathsep == ":" +assert os.fspath("Testing") == "Testing" +assert os.fspath(b"Testing") == b"Testing" +assert_raises(TypeError, lambda: os.fspath([1,2,3])) + class TestWithTempDir(): def __enter__(self): if os.name == "nt": From 4070baa6ac71a1a6731622e68fd1baea40fd5b3f Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 12 May 2019 07:51:14 +1200 Subject: [PATCH 552/658] Add fsync and more tests for file times --- tests/snippets/stdlib_os.py | 30 ++++++++++++++++++++++++++++++ vm/src/stdlib/os.rs | 11 +++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 9317a8e82..30b48d003 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -88,6 +88,9 @@ with TestWithTempDir() as tmpdir: assert os.read(fd, len(CONTENT3)) == CONTENT3 os.close(fd) + # wait a little bit to ensure that the file times aren't the same + time.sleep(0.1) + fname2 = os.path.join(tmpdir, FILE_NAME2) with open(fname2, "wb"): pass @@ -151,6 +154,33 @@ with TestWithTempDir() as tmpdir: assert stat_res.st_ctime > 1557500000 assert stat_res.st_mtime > 1557500000 + stat_file2 = os.stat(fname2) + print(stat_file2.st_ctime) + assert stat_file2.st_ctime > stat_res.st_ctime + + # wait a little bit to ensures that the access/modify time is different + time.sleep(0.1) + + old_atime = stat_res.st_atime + old_mtime = stat_res.st_mtime + + fd = os.open(fname, os.O_RDWR) + os.write(fd, CONTENT) + os.fsync(fd) + + os.read(fd, 1) + os.fsync(fd) + os.close(fd) + + # retrieve update file stats + stat_res = os.stat(fname) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + assert stat_res.st_atime > old_atime, "Access time should be update" + assert stat_res.st_mtime > old_mtime, "Modified time should be update" + assert stat_res.st_atime > stat_res.st_mtime + # stat default is follow_symlink=True os.stat(fname).st_ino == os.stat(symlink_file).st_ino os.stat(fname).st_mode == os.stat(symlink_file).st_mode diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e1b44839d..93d40b38b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -89,6 +89,7 @@ pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let handle = match objint::get_value(mode).to_u16().unwrap() { 0 => OpenOptions::new().read(true).open(&fname), 1 => OpenOptions::new().write(true).open(&fname), + 2 => OpenOptions::new().read(true).write(true).open(&fname), 512 => OpenOptions::new().write(true).create(true).open(&fname), _ => OpenOptions::new().read(true).open(&fname), } @@ -124,6 +125,15 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } +fn os_fsync(fd: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let file = rust_file(fd.as_bigint().to_i64().unwrap()); + file.sync_all() + .map_err(|s| vm.new_os_error(s.to_string()))?; + // Avoid closing the fd + raw_file_number(file); + Ok(()) +} + fn os_read(fd: PyIntRef, n: PyIntRef, vm: &VirtualMachine) -> PyResult { let mut buffer = vec![0u8; n.as_bigint().to_usize().unwrap()]; let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); @@ -580,6 +590,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "fsync" => ctx.new_rustfunc(os_fsync), "read" => ctx.new_rustfunc(os_read), "write" => ctx.new_rustfunc(os_write), "remove" => ctx.new_rustfunc(os_remove), From f2703511b959973ec7cfdc4f7eee6c8e1b2807c5 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 12 May 2019 08:26:37 +1200 Subject: [PATCH 553/658] Fix dead code warning on windows and disable access time tests as file access time on windows has quite bad resolution. --- tests/snippets/stdlib_os.py | 7 +++++-- vm/src/stdlib/os.rs | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 30b48d003..96906308a 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -177,9 +177,12 @@ with TestWithTempDir() as tmpdir: print(stat_res.st_atime) print(stat_res.st_ctime) print(stat_res.st_mtime) - assert stat_res.st_atime > old_atime, "Access time should be update" + if os.name != "nt": + # access time on windows has a resolution ranging from 1 hour to 1 day + # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime + assert stat_res.st_atime > old_atime, "Access time should be update" + assert stat_res.st_atime > stat_res.st_mtime assert stat_res.st_mtime > old_mtime, "Modified time should be update" - assert stat_res.st_atime > stat_res.st_mtime # stat default is follow_symlink=True os.stat(fname).st_ino == os.stat(symlink_file).st_ino diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 93d40b38b..6fd2daf72 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -395,6 +395,7 @@ fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { } } +#[cfg(unix)] fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { let duration = Duration::new(secs as u64, nanos as u32); duration_as_secs_f64(duration) From 645c6dec48e956d35683970bb922f60a495e88f5 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 12 May 2019 08:55:39 +1200 Subject: [PATCH 554/658] Add wait between file modification and access --- tests/snippets/stdlib_os.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 96906308a..7d8d84a9d 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -158,7 +158,7 @@ with TestWithTempDir() as tmpdir: print(stat_file2.st_ctime) assert stat_file2.st_ctime > stat_res.st_ctime - # wait a little bit to ensures that the access/modify time is different + # wait a little bit to ensures that the access/modify time will change time.sleep(0.1) old_atime = stat_res.st_atime @@ -168,6 +168,9 @@ with TestWithTempDir() as tmpdir: os.write(fd, CONTENT) os.fsync(fd) + # wait a little bit to ensures that the access/modify time is different + time.sleep(0.1) + os.read(fd, 1) os.fsync(fd) os.close(fd) From 268cb8d21776e110f1b9ef288592dc465e78ec57 Mon Sep 17 00:00:00 2001 From: zer0 Date: Sat, 11 May 2019 23:58:42 +0300 Subject: [PATCH 555/658] fix str.maketrans 2 and 3 arguments cases --- tests/snippets/strings.py | 2 +- vm/src/obj/objstr.rs | 4 ++-- whats_left.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 480500e9f..531e8f83c 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -162,7 +162,7 @@ assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" # str.maketrans assert str.maketrans({"a": "abc", "b": None, "c": 33}) == {97: "abc", 98: None, 99: 33} -assert str.maketrans("hello", "world", "rust") == {103: "w", 101: "o", 108: "l", 111: "d", "r": None, "u": None, "s": None, "t": None} +assert str.maketrans("hello", "world", "rust") == {104: 119, 101: 111, 108: 108, 111: 100, 114: None, 117: None, 115: None, 116: None} def try_mutate_str(): word = "word" diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a4e6da8c5..6062d1d17 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -884,7 +884,7 @@ impl PyString { let new_dict = vm.context().new_dict(); if to_str.len(vm) == from_str.len(vm) { for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { - new_dict.set_item(c1.to_string(), vm.new_str(c2.to_string()), vm)?; + new_dict.set_item(c1 as u32, vm.new_int(c2 as u32), vm)?; } } else { return Err(vm.new_value_error( @@ -893,7 +893,7 @@ impl PyString { } if let OptionalArg::Present(none_str) = none_str { for c in none_str.value.chars() { - new_dict.set_item(c.to_string(), vm.get_none(), vm)?; + new_dict.set_item(c as u32, vm.get_none(), vm)?; } } new_dict.into_pyobject(vm) diff --git a/whats_left.sh b/whats_left.sh index aa79d8d65..66e64f1a3 100755 --- a/whats_left.sh +++ b/whats_left.sh @@ -8,4 +8,4 @@ python3 not_impl_gen.py cd .. -cargo +nightly run -- tests/snippets/whats_left_to_implement.py +cargo run -- tests/snippets/whats_left_to_implement.py \ No newline at end of file From b94923314db7990bf11f21bff8de261cde6152ae Mon Sep 17 00:00:00 2001 From: zer0 Date: Sun, 12 May 2019 00:00:21 +0300 Subject: [PATCH 556/658] removed import warning --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 6062d1d17..a87aad20e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -16,7 +16,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objdict::{PyDict, PyDictRef}; +use super::objdict::PyDict; use super::objint::{self, PyInt}; use super::objnone::PyNone; use super::objsequence::PySliceableSequence; From 764215151d1aabfeff608c9503209a8c57ae6eda Mon Sep 17 00:00:00 2001 From: zer0 Date: Sun, 12 May 2019 00:40:43 +0300 Subject: [PATCH 557/658] more idiomatic code --- vm/src/obj/objstr.rs | 100 +++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a87aad20e..c084ca0ce 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -835,19 +835,19 @@ impl PyString { // https://docs.python.org/3/library/stdtypes.html#str.translate #[pymethod] fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut result = String::new(); + let mut translated = String::new(); // It throws a type error if it is not subscribtable vm.get_method(table.clone(), "__getitem__")?; for c in self.value.chars() { match table.get_item(c as u32, vm) { Ok(value) => { if let Some(text) = value.payload::() { - result.extend(text.value.chars()); + translated.extend(text.value.chars()); } else if let Some(_) = value.payload::() { // Do Nothing } else if let Some(bigint) = value.payload::() { match bigint.as_bigint().to_u32().and_then(std::char::from_u32) { - Some(ch) => result.push(ch as char), + Some(ch) => translated.push(ch as char), None => { return Err(vm.new_value_error(format!( "character mapping must be in range(0x110000)" @@ -860,10 +860,10 @@ impl PyString { )); } } - _ => result.push(c), + _ => translated.push(c), } } - Ok(result) + Ok(translated) } #[pymethod] @@ -873,53 +873,54 @@ impl PyString { none_str: OptionalArg, vm: &VirtualMachine, ) -> PyResult { + let new_dict = vm.context().new_dict(); if let OptionalArg::Present(to_str) = to_str { - // dict_or_str is str - let from_str = dict_or_str.payload::().ok_or( - vm.new_type_error( - "first maketrans argument must be a string if there is a second argument" - .to_owned(), - ), - )?; - let new_dict = vm.context().new_dict(); - if to_str.len(vm) == from_str.len(vm) { - for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { - new_dict.set_item(c1 as u32, vm.new_int(c2 as u32), vm)?; - } - } else { - return Err(vm.new_value_error( - "the first two maketrans arguments must have equal length".to_owned(), - )); - } - if let OptionalArg::Present(none_str) = none_str { - for c in none_str.value.chars() { - new_dict.set_item(c as u32, vm.get_none(), vm)?; - } - } - new_dict.into_pyobject(vm) - } else { - // dict_str must be a dict - if let Ok(dict) = dict_or_str.downcast::() { - let new_dict = vm.context().new_dict(); - for (key, val) in dict { - if let Some(num) = key.payload::() { - new_dict.set_item(num.as_bigint().to_i32(), val, vm)?; - } else if let Some(string) = key.payload::() { - if string.len(vm) == 1 { - let num_value = string.value.chars().next().unwrap() as u32; - new_dict.set_item(num_value, val, vm)?; - } else { - return Err(vm.new_value_error( - "string keys in translate table must be of length 1".to_owned(), - )); + match dict_or_str.downcast::() { + Ok(from_str) => { + if to_str.len(vm) == from_str.len(vm) { + for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { + new_dict.set_item(c1 as u32, vm.new_int(c2 as u32), vm)?; } + if let OptionalArg::Present(none_str) = none_str { + for c in none_str.value.chars() { + new_dict.set_item(c as u32, vm.get_none(), vm)?; + } + } + new_dict.into_pyobject(vm) + } else { + Err(vm.new_value_error( + "the first two maketrans arguments must have equal length".to_owned(), + )) } } - new_dict.into_pyobject(vm) - } else { - Err(vm.new_value_error( + _ => Err(vm.new_type_error( + "first maketrans argument must be a string if there is a second argument" + .to_owned(), + )), + } + } else { + // dict_str must be a dict + match dict_or_str.downcast::() { + Ok(dict) => { + for (key, val) in dict { + if let Some(num) = key.payload::() { + new_dict.set_item(num.as_bigint().to_i32(), val, vm)?; + } else if let Some(string) = key.payload::() { + if string.len(vm) == 1 { + let num_value = string.value.chars().next().unwrap() as u32; + new_dict.set_item(num_value, val, vm)?; + } else { + return Err(vm.new_value_error( + "string keys in translate table must be of length 1".to_owned(), + )); + } + } + } + new_dict.into_pyobject(vm) + } + _ => Err(vm.new_value_error( "if you give only one argument to maketrans it must be a dict".to_owned(), - )) + )), } } } @@ -1217,4 +1218,9 @@ mod tests { let translated = text.translate(vm.new_int(3), &vm); println!("{:?}", translated); } + + #[test] + fn str_maketrans() { + let vm = VirtualMachine::new(); + } } From 90711f10a09f62062a358041fbf3ecf7c6f49506 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 12 May 2019 13:14:27 +1200 Subject: [PATCH 558/658] Accept tuple for first arg in str.startswith and str.endswith --- tests/snippets/strings.py | 7 ++++++ vm/src/builtins.rs | 50 +++++++++++++++++++-------------------- vm/src/function.rs | 26 ++++++++++++++++++++ vm/src/obj/objstr.rs | 44 +++++++++++++++++++++++++--------- 4 files changed, 90 insertions(+), 37 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 0cb3782b1..4a480cbf5 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -42,9 +42,15 @@ assert a.lower() == 'hallo' assert a.upper() == 'HALLO' assert a.split('al') == ['H', 'lo'] assert a.startswith('H') +assert a.startswith(('H', 1)) +assert a.startswith(('A', 'H')) assert not a.startswith('f') +assert not a.startswith(('A', 'f')) assert a.endswith('llo') +assert a.endswith(('lo', 1)) +assert a.endswith(('A', 'lo')) assert not a.endswith('on') +assert not a.endswith(('A', 'll')) assert a.zfill(8) == '000Hallo' assert a.isalnum() assert not a.isdigit() @@ -144,6 +150,7 @@ assert '___a__'.find('a', 3, 4) == 3 assert '___a__'.find('a', 4, 3) == -1 assert 'abcd'.startswith('b', 1) +assert 'abcd'.startswith(('b', 'z'), 1) assert not 'abcd'.startswith('b', -4) assert 'abcd'.startswith('b', -3) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a48a3b441..e9056e2ea 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -17,11 +17,10 @@ use crate::obj::objdict::PyDictRef; use crate::obj::objint::{self, PyIntRef}; use crate::obj::objiter; use crate::obj::objstr::{self, PyString, PyStringRef}; -use crate::obj::objtuple::PyTuple; -use crate::obj::objtype::{self, PyClass, PyClassRef}; +use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; -use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; +use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -317,29 +316,18 @@ fn builtin_id(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_input -fn type_test( - vm: &VirtualMachine, - typ: PyObjectRef, - test: impl Fn(&PyClassRef) -> PyResult, - test_name: &str, -) -> PyResult { - match_class!(typ, - cls @ PyClass => test(&cls), - tuple @ PyTuple => { - for cls_obj in tuple.elements.borrow().iter() { - let cls = PyClassRef::try_from_object(vm, cls_obj.clone())?; - if test(&cls)? { - return Ok(true); - } - } - Ok(false) - }, - _ => Err(vm.new_type_error(format!("{}() arg 2 must be a type or tuple of types", test_name))) - ) -} - fn builtin_isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult { - type_test(vm, typ, |cls| vm.isinstance(&obj, cls), "isinstance") + single_or_tuple_any( + typ, + |cls: PyClassRef| vm.isinstance(&obj, &cls), + |o| { + format!( + "isinstance() arg 2 must be a type or tuple of types, not {}", + o.class() + ) + }, + vm, + ) } fn builtin_issubclass( @@ -347,7 +335,17 @@ fn builtin_issubclass( typ: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - type_test(vm, typ, |cls| vm.issubclass(&subclass, cls), "issubclass") + single_or_tuple_any( + typ, + |cls: PyClassRef| vm.issubclass(&subclass, &cls), + |o| { + format!( + "issubclass() arg 2 must be a class or tuple of classes, not {}", + o.class() + ) + }, + vm, + ) } fn builtin_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/function.rs b/vm/src/function.rs index d3f26950f..170c395f6 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::mem; use std::ops::RangeInclusive; +use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{isinstance, PyClassRef}; use crate::pyobject::{ IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -523,3 +524,28 @@ into_py_native_func_tuple!((a, A), (b, B)); into_py_native_func_tuple!((a, A), (b, B), (c, C)); into_py_native_func_tuple!((a, A), (b, B), (c, C), (d, D)); into_py_native_func_tuple!((a, A), (b, B), (c, C), (d, D), (e, E)); + +/// Tests that the predicate is True on a single value, or if the value is a tuple a tuple, then +/// test that any of the values contained within the tuples satisfies the predicate. Type parameter +/// T specifies the type that is expected, if the input value is not of that type or a tuple of +/// values of that type, then a TypeError is raised. +pub fn single_or_tuple_any) -> PyResult>( + obj: PyObjectRef, + predicate: F, + message: fn(&PyObjectRef) -> String, + vm: &VirtualMachine, +) -> PyResult { + match_class!(obj, + obj @ T => predicate(obj), + tuple @ PyTuple => { + for obj in tuple.elements.borrow().iter() { + let inner_val = PyRef::::try_from_object(vm, obj.clone())?; + if predicate(inner_val)? { + return Ok(true); + } + } + Ok(false) + }, + obj => Err(vm.new_type_error(message(&obj))) + ) +} diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 7b2fcbe9a..621cbf5bc 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -12,7 +12,7 @@ use unicode_segmentation::UnicodeSegmentation; use unicode_xid::UnicodeXID; use crate::format::{FormatParseError, FormatPart, FormatString}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{single_or_tuple_any, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, @@ -342,30 +342,52 @@ impl PyString { #[pymethod] fn endswith( &self, - suffix: PyStringRef, + suffix: PyObjectRef, start: OptionalArg, end: OptionalArg, - _vm: &VirtualMachine, - ) -> bool { + vm: &VirtualMachine, + ) -> PyResult { if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].ends_with(&suffix.value) + let value = &self.value[start..end]; + single_or_tuple_any( + suffix, + |s: PyStringRef| Ok(value.ends_with(&s.value)), + |o| { + format!( + "endswith first arg must be str or a tuple of str, not {}", + o.class(), + ) + }, + vm, + ) } else { - false + Ok(false) } } #[pymethod] fn startswith( &self, - prefix: PyStringRef, + prefix: PyObjectRef, start: OptionalArg, end: OptionalArg, - _vm: &VirtualMachine, - ) -> bool { + vm: &VirtualMachine, + ) -> PyResult { if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].starts_with(&prefix.value) + let value = &self.value[start..end]; + single_or_tuple_any( + prefix, + |s: PyStringRef| Ok(value.starts_with(&s.value)), + |o| { + format!( + "startswith first arg must be str or a tuple of str, not {}", + o.class(), + ) + }, + vm, + ) } else { - false + Ok(false) } } From 04c017d2682cf6c8975a6b5fbd3f4c22e0e44cd3 Mon Sep 17 00:00:00 2001 From: zer0 Date: Sun, 12 May 2019 12:47:23 +0300 Subject: [PATCH 559/658] unit testing for str.maketrans --- vm/src/obj/objstr.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index c084ca0ce..d81b69f93 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1201,26 +1201,28 @@ mod tests { } #[test] - fn str_translate() { + fn str_maketrans_and_translate() { let vm = VirtualMachine::new(); let table = vm.context().new_dict(); table - .set_item(97, vm.new_str("🎅".to_owned()), &vm) + .set_item("a", vm.new_str("🎅".to_owned()), &vm) .unwrap(); - table.set_item(98, vm.get_none(), &vm).unwrap(); + table.set_item("b", vm.get_none(), &vm).unwrap(); table - .set_item(99, vm.new_str("xda".to_owned()), &vm) + .set_item("c", vm.new_str("xda".to_owned()), &vm) .unwrap(); + let translated = PyString::maketrans( + table.into_object(), + OptionalArg::Missing, + OptionalArg::Missing, + &vm, + ) + .unwrap(); let text = PyString::from("abc"); - let translated = text.translate(table.into_object(), &vm).unwrap(); + let translated = text.translate(translated, &vm).unwrap(); assert_eq!(translated, "🎅xda".to_owned()); let translated = text.translate(vm.new_int(3), &vm); - println!("{:?}", translated); - } - - #[test] - fn str_maketrans() { - let vm = VirtualMachine::new(); + assert_eq!(translated.unwrap_err().class().name, "TypeError".to_owned()); } } From e311b4f156e5ebb7970459b3e66c69b30144a0e6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 04:26:55 +0900 Subject: [PATCH 560/658] Fix exec to populate locals only when globals() is not given --- tests/snippets/test_exec.py | 20 ++++++++++++++++++++ vm/src/builtins.rs | 17 +++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/snippets/test_exec.py b/tests/snippets/test_exec.py index 61932c24c..506882e16 100644 --- a/tests/snippets/test_exec.py +++ b/tests/snippets/test_exec.py @@ -44,3 +44,23 @@ assert g['x'] == x exec("del x") assert 'x' not in g + +assert 'g' in globals() +assert 'g' in locals() +exec("assert 'g' in globals()") +exec("assert 'g' in locals()") +exec("assert 'g' not in globals()", {}) +exec("assert 'g' not in locals()", {}) + +del g + +def f(): + g = 1 + assert 'g' not in globals() + assert 'g' in locals() + exec("assert 'g' not in globals()") + exec("assert 'g' in locals()") + exec("assert 'g' not in globals()", {}) + exec("assert 'g' not in locals()", {}) + +f() diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 5b3082b0d..068ff43ff 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -171,7 +171,7 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { vm, args, required = [(source, None)], - optional = [(globals, None), (locals, Some(vm.ctx.dict_type()))] + optional = [(globals, None), (locals, None)] ); let scope = make_scope(vm, globals, locals)?; @@ -219,6 +219,16 @@ fn make_scope( None => None, }; let current_scope = vm.current_scope(); + let locals = match locals { + Some(dict) => dict.clone().downcast().ok(), + None => { + if globals.is_some() { + None + } else { + current_scope.get_only_locals() + } + } + }; let globals = match globals { Some(dict) => { let dict: PyDictRef = dict.clone().downcast().unwrap(); @@ -232,11 +242,6 @@ fn make_scope( None => current_scope.globals.clone(), }; - let locals = match locals { - Some(dict) => dict.clone().downcast().ok(), - None => current_scope.get_only_locals(), - }; - let scope = Scope::with_builtins(locals, globals, vm); Ok(scope) } From ddf676569acad211fc06ff3ba9f082073a4a7d45 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 07:16:45 +0900 Subject: [PATCH 561/658] Add sys.builtin_module_names --- tests/snippets/sysmod.py | 3 +++ vm/src/sysmodule.rs | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index b7cdc22ec..423ecf08d 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -4,3 +4,6 @@ print(sys.argv) assert sys.argv[0].endswith('.py') assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "win32" or sys.platform == "unknown" + +assert isinstance(sys.builtin_module_names, tuple) +assert 'sys' in sys.builtin_module_names diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 4174fcf10..8c6bafe74 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -4,7 +4,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* @@ -144,9 +144,14 @@ setprofile() -- set the global profiling function setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; + let mut module_names: Vec<_> = vm.stdlib_inits.borrow().keys().cloned().collect(); + module_names.push("sys".to_string()); + module_names.push("builtins".to_string()); + module_names.sort(); let modules = ctx.new_dict(); extend_module!(vm, module, { "argv" => argv(ctx), + "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), "intern" => ctx.new_rustfunc(sys_intern), From 72043dac99c2f44949252edd1ab3673489a7ab77 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 02:23:28 +0900 Subject: [PATCH 562/658] Fix builtin hash() --- vm/src/builtins.rs | 3 +-- vm/src/vm.rs | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 53d760cc8..3250bef80 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -303,8 +303,7 @@ fn builtin_hasattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> fn builtin_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - - vm.call_method(obj, "__hash__", vec![]) + vm._hash(obj).and_then(|v| Ok(vm.new_int(v))) } // builtin_help diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 814631015..eab6fca8c 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -23,6 +23,7 @@ use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGenerator; +use crate::obj::objint; use crate::obj::objiter; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; @@ -36,6 +37,7 @@ use crate::pyobject::{ use crate::stdlib; use crate::sysmodule; use num_bigint::BigInt; +use num_traits::ToPrimitive; // use objects::objects; @@ -912,6 +914,19 @@ impl VirtualMachine { }) } + pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { + const PRIME: usize = 0x1fff_ffff_ffff_ffff; + let hash_obj = self.call_method(obj, "__hash__", vec![])?; + if objtype::isinstance(&hash_obj, &self.ctx.int_type()) { + let hash_int = objint::get_value(&hash_obj); + Ok(hash_int + .to_usize() + .unwrap_or_else(|| (hash_int % PRIME).to_usize().unwrap())) + } else { + Err(self.new_type_error("__hash__ method should return an integer".to_string())) + } + } + // https://docs.python.org/3/reference/expressions.html#membership-test-operations fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { let iter = objiter::get_iter(self, &haystack)?; From 2c77dc208ff4719fc61ebb3e6d7ce37bd66d14bd Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 07:01:39 +0900 Subject: [PATCH 563/658] Add float.__hash__ --- tests/snippets/dict.py | 2 ++ tests/snippets/floats.py | 15 +++++++++ vm/src/lib.rs | 1 + vm/src/obj/objfloat.rs | 6 ++++ vm/src/pyhash.rs | 67 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+) create mode 100644 vm/src/pyhash.rs diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 270d2c07e..f7506a205 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -198,3 +198,5 @@ z = {'c': 3, 'd': 3, 'e': 3} w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} + +assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 82f2b86e3..c116b8010 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -98,6 +98,21 @@ assert float(b'2.99e-23') == 2.99e-23 assert_raises(ValueError, lambda: float('foo')) assert_raises(OverflowError, lambda: float(2**10000)) +# check eq and hash for small numbers + +assert 1.0 == 1 +assert 1.0 == True +assert 0.0 == 0 +assert 0.0 == False +assert hash(1.0) == hash(1) +assert hash(1.0) == hash(True) +assert hash(0.0) == hash(0) +assert hash(0.0) == hash(False) +assert hash(1.0) != hash(1.0000000001) + +assert 5.0 in {3, 4, 5} +assert {-1: 2}[-1.0] == 2 + # check that magic methods are implemented for ints and floats assert 1.0.__add__(1.0) == 2.0 diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 8b14454fe..2e5fcb3f4 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -55,6 +55,7 @@ pub mod frame; pub mod function; pub mod import; pub mod obj; +mod pyhash; pub mod pyobject; pub mod stdlib; mod symboltable; diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index add61a684..8d983c3ae 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -4,6 +4,7 @@ use super::objstr; use super::objtype; use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, @@ -429,6 +430,11 @@ impl PyFloat { zelf } + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_float(self.value) + } + #[pyproperty(name = "real")] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { zelf diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs new file mode 100644 index 000000000..81c605584 --- /dev/null +++ b/vm/src/pyhash.rs @@ -0,0 +1,67 @@ +use std::hash::{Hash, Hasher}; + +use crate::pyobject::PyObjectRef; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +pub type PyHash = i64; +pub type PyUHash = u64; + +pub const BITS: usize = 61; +pub const MODULUS: PyUHash = (1 << BITS) - 1; +// pub const CUTOFF: usize = 7; + +pub const INF: PyHash = 314159; +pub const NAN: PyHash = 0; + +pub fn hash_float(value: f64) -> PyHash { + // cpython _Py_HashDouble + if !value.is_finite() { + return if value.is_infinite() { + if value > 0.0 { + INF + } else { + -INF + } + } else { + NAN + }; + } + + let frexp = if 0.0 == value { + (value, 0i32) + } else { + let bits = value.to_bits(); + let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; + let mantissa_bits = bits & (0x000fffffffffffff) | (1022 << 52); + (f64::from_bits(mantissa_bits), exponent) + }; + + // process 28 bits at a time; this should work well both for binary + // and hexadecimal floating point. + let mut m = frexp.0; + let mut e = frexp.1; + let mut x: PyUHash = 0; + while m != 0.0 { + x = ((x << 28) & MODULUS) | x >> (BITS - 28); + m *= 268435456.0; // 2**28 + e -= 28; + let y = m as PyUHash; // pull out integer part + m -= y as f64; + x += y; + if x >= MODULUS { + x -= MODULUS; + } + } + + // adjust for the exponent; first reduce it modulo BITS + const BITS32: i32 = BITS as i32; + e = if e >= 0 { + e % BITS32 + } else { + BITS32 - 1 - ((-1 - e) % BITS32) + }; + x = ((x << e) & MODULUS) | x >> (BITS32 - e); + + x as PyHash * value.signum() as PyHash +} From 3bebe448961c641bfd66ead4a080aab360549960 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 07:02:59 +0900 Subject: [PATCH 564/658] Fix int.__hash__ --- vm/src/obj/objint.rs | 11 ++++++----- vm/src/vm.rs | 12 ++++-------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 403fd08ef..9c1ccf921 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,5 +1,4 @@ use std::fmt; -use std::hash::{Hash, Hasher}; use num_bigint::{BigInt, Sign}; use num_integer::Integer; @@ -7,6 +6,7 @@ use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -400,10 +400,11 @@ impl PyInt { } #[pymethod(name = "__hash__")] - fn hash(&self, _vm: &VirtualMachine) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() + pub fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + match self.value.to_i64() { + Some(value) => (value % pyhash::MODULUS as i64), + None => (&self.value % pyhash::MODULUS).to_i64().unwrap(), + } } #[pymethod(name = "__abs__")] diff --git a/vm/src/vm.rs b/vm/src/vm.rs index eab6fca8c..6cd1165db 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -23,13 +23,14 @@ use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGenerator; -use crate::obj::objint; +use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, @@ -37,7 +38,6 @@ use crate::pyobject::{ use crate::stdlib; use crate::sysmodule; use num_bigint::BigInt; -use num_traits::ToPrimitive; // use objects::objects; @@ -914,14 +914,10 @@ impl VirtualMachine { }) } - pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { - const PRIME: usize = 0x1fff_ffff_ffff_ffff; + pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { let hash_obj = self.call_method(obj, "__hash__", vec![])?; if objtype::isinstance(&hash_obj, &self.ctx.int_type()) { - let hash_int = objint::get_value(&hash_obj); - Ok(hash_int - .to_usize() - .unwrap_or_else(|| (hash_int % PRIME).to_usize().unwrap())) + Ok(hash_obj.payload::().unwrap().hash(self)) } else { Err(self.new_type_error("__hash__ method should return an integer".to_string())) } From c768b25673a9d95665405559c7ed06d8c8d8b715 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 07:32:25 +0900 Subject: [PATCH 565/658] Refine hash types and refactor hash impl --- vm/src/obj/objbytearray.rs | 2 +- vm/src/obj/objbyteinner.rs | 16 +++++----------- vm/src/obj/objbytes.rs | 4 ++-- vm/src/obj/objdict.rs | 2 +- vm/src/obj/objlist.rs | 2 +- vm/src/obj/objobject.rs | 5 +++-- vm/src/obj/objset.rs | 2 +- vm/src/obj/objstr.rs | 8 +++----- vm/src/obj/objtuple.rs | 13 +++---------- vm/src/pyhash.rs | 18 ++++++++++++++++++ 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 8b09a53e9..e5078bdbb 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -135,7 +135,7 @@ impl PyByteArrayRef { } #[pymethod(name = "__hash__")] - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type: bytearray".to_string())) } diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index ae318e628..4e51c4170 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -2,6 +2,7 @@ use crate::obj::objint::PyIntRef; use crate::obj::objnone::PyNoneRef; use crate::obj::objslice::PySliceRef; use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; use crate::pyobject::Either; use crate::pyobject::PyRef; use crate::pyobject::PyValue; @@ -12,17 +13,12 @@ use core::ops::Range; use num_bigint::BigInt; use crate::function::OptionalArg; - -use crate::vm::VirtualMachine; - use crate::pyobject::{PyResult, TypeProtocol}; - -use crate::obj::objstr::{PyString, PyStringRef}; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use crate::vm::VirtualMachine; use super::objint; use super::objsequence::{is_valid_slice_arg, PySliceableSequence}; +use super::objstr::{PyString, PyStringRef}; use crate::obj::objint::PyInt; use num_integer::Integer; @@ -379,10 +375,8 @@ impl PyByteInner { } } - pub fn hash(&self) -> usize { - let mut hasher = DefaultHasher::new(); - self.elements.hash(&mut hasher); - hasher.finish() as usize + pub fn hash(&self) -> pyhash::PyHash { + pyhash::hash_value(&self.elements) } pub fn add(&self, other: PyByteInner) -> Vec { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 102c70d4f..dd2e1518c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,8 +1,8 @@ use crate::obj::objint::PyIntRef; - use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStringRef; use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; use crate::pyobject::Either; use crate::vm::VirtualMachine; @@ -125,7 +125,7 @@ impl PyBytesRef { } #[pymethod(name = "__hash__")] - fn hash(self, _vm: &VirtualMachine) -> usize { + fn hash(self, _vm: &VirtualMachine) -> pyhash::PyHash { self.inner.hash() } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 87cf3010b..396c711f4 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -298,7 +298,7 @@ impl PyDictRef { Ok(PyDict { entries }.into_ref(vm)) } - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 6b00b6828..88842f9b7 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -387,7 +387,7 @@ impl PyListRef { Ok(s) } - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 31e52fc99..77830a402 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -5,6 +5,7 @@ use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -55,8 +56,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py vm.ctx.not_implemented() } -fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 { - zelf.get_id() as u64 +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> pyhash::PyHash { + zelf.get_id() as pyhash::PyHash } fn object_setattr( diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b4f3bfabc..41bb06547 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -759,7 +759,7 @@ impl TryFromObject for SetIterable { } } -fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { +fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ab7525e74..cf42a1a5c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,7 +1,6 @@ extern crate unicode_xid; use std::fmt; -use std::hash::{Hash, Hasher}; use std::ops::Range; use std::str::FromStr; use std::string::ToString; @@ -13,6 +12,7 @@ use unicode_xid::UnicodeXID; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{single_or_tuple_any, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, @@ -172,10 +172,8 @@ impl PyString { } #[pymethod(name = "__hash__")] - fn hash(&self, _vm: &VirtualMachine) -> usize { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() as usize + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_value(&self.value) } #[pymethod(name = "__len__")] diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 42999189f..795e17d62 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -1,13 +1,12 @@ use std::cell::{Cell, RefCell}; use std::fmt; -use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; +use crate::pyhash; use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; use super::objiter; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, @@ -129,14 +128,8 @@ impl PyTupleRef { } } - fn hash(self, vm: &VirtualMachine) -> PyResult { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - for element in self.elements.borrow().iter() { - let hash_result = vm.call_method(element, "__hash__", vec![])?; - let element_hash = objint::get_value(&hash_result); - element_hash.hash(&mut hasher); - } - Ok(hasher.finish()) + fn hash(self, vm: &VirtualMachine) -> PyResult { + pyhash::hash_iter(self.elements.borrow().iter(), vm) } fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs index 81c605584..f87337359 100644 --- a/vm/src/pyhash.rs +++ b/vm/src/pyhash.rs @@ -65,3 +65,21 @@ pub fn hash_float(value: f64) -> PyHash { x as PyHash * value.signum() as PyHash } + +pub fn hash_value(data: &T) -> PyHash { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + hasher.finish() as PyHash +} + +pub fn hash_iter<'a, I: std::iter::Iterator>( + iter: I, + vm: &VirtualMachine, +) -> PyResult { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for element in iter { + let item_hash = vm._hash(&element)?; + item_hash.hash(&mut hasher); + } + Ok(hasher.finish() as PyHash) +} From 7a64f3e797231d0f6c5ff8d71fe7b03532d2d177 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 07:32:45 +0900 Subject: [PATCH 566/658] Fix dict hash impl --- vm/src/dictdatatype.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index b0bd4894f..a1aa9e3c3 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,8 +1,6 @@ use crate::obj::objbool; -use crate::obj::objint; use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -use num_traits::ToPrimitive; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 @@ -218,8 +216,7 @@ enum LookupResult { } fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash = vm.call_method(key, "__hash__", vec![])?; - Ok(objint::get_value(&hash).to_usize().unwrap()) + Ok(vm._hash(key)? as usize) } /// Invoke __eq__ on two keys From 64afd5ccf8f9f76e7dcb23ca81bb54bebfdf6ba5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 17:56:11 +0900 Subject: [PATCH 567/658] Add os.supports_* placeholders --- tests/snippets/stdlib_os.py | 5 + vm/src/obj/objset.rs | 2 +- vm/src/stdlib/os.rs | 185 ++++++++++++++++++++++++++++++------ 3 files changed, 164 insertions(+), 28 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 61fa42855..e24ac6503 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -217,3 +217,8 @@ with TestWithTempDir() as tmpdir: os.chdir(tmpdir) assert os.getcwd() == tmpdir os.path.exists(FILE_NAME) + +# supports +assert isinstance(os.supports_fd, set) +assert isinstance(os.supports_dir_fd, set) +assert isinstance(os.supports_follow_symlinks, set) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b4f3bfabc..7593edf50 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -472,7 +472,7 @@ impl PySetRef { Ok(vm.new_str(s)) } - fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + pub fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().add(&item, vm) } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 4394bdb89..1e80146ea 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,14 +7,13 @@ use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::PyFuncArgs; +use crate::function::{IntoPyNativeFunc, PyFuncArgs}; use crate::obj::objbytes::PyBytesRef; use crate::obj::objdict::PyDictRef; -use crate::obj::objint; -use crate::obj::objint::PyIntRef; +use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objiter; -use crate::obj::objstr; -use crate::obj::objstr::PyStringRef; +use crate::obj::objset::PySet; +use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, @@ -61,6 +60,14 @@ pub fn raw_file_number(handle: File) -> i64 { unimplemented!(); } +fn make_path(_vm: &VirtualMachine, path: PyStringRef, dir_fd: &DirFd) -> PyStringRef { + if dir_fd.dir_fd.is_some() { + unimplemented!(); + } else { + path + } +} + pub fn os_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(fileno, Some(vm.ctx.int_type()))]); @@ -81,10 +88,19 @@ pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [ (name, Some(vm.ctx.str_type())), (mode, Some(vm.ctx.int_type())) - ] + ], + optional = [(dir_fd, Some(vm.ctx.int_type()))] ); - let fname = objstr::get_value(&name); + let name = name.clone().downcast::().unwrap(); + let dir_fd = if let Some(obj) = dir_fd { + DirFd { + dir_fd: Some(obj.clone().downcast::().unwrap()), + } + } else { + DirFd::default() + }; + let fname = &make_path(vm, name, &dir_fd).value; let handle = match objint::get_value(mode).to_u16().unwrap() { 0 => OpenOptions::new().read(true).open(&fname), @@ -159,11 +175,13 @@ fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_remove(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::remove_file(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_mkdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::create_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } @@ -171,7 +189,8 @@ fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { fs::create_dir_all(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_rmdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::remove_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } @@ -219,6 +238,12 @@ impl PyValue for DirEntry { } } +#[derive(FromArgs, Default)] +struct DirFd { + #[pyarg(keyword_only, default = "None")] + dir_fd: Option, +} + #[derive(FromArgs)] struct FollowSymlinks { #[pyarg(keyword_only, default = "true")] @@ -272,8 +297,13 @@ impl DirEntryRef { .is_symlink()) } - fn stat(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - os_stat(self.path(vm).try_into_ref(vm)?, follow_symlinks, vm) + fn stat( + self, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + os_stat(self.path(vm).try_into_ref(vm)?, dir_fd, follow_symlinks, vm) } } @@ -432,30 +462,36 @@ macro_rules! os_unix_stat_inner { #[cfg(target_os = "linux")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::linux::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "macos")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::macos::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "android")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::android::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } @@ -483,6 +519,7 @@ fn attributes_to_mode(attr: u32) -> u32 { #[cfg(windows)] fn os_stat( path: PyStringRef, + _dir_fd: DirFd, // TODO: error follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { @@ -523,13 +560,24 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } #[cfg(unix)] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { use std::os::unix::fs as unix_fs; + let dst = make_path(vm, dst, &dir_fd); unix_fs::symlink(&src.value, &dst.value).map_err(|s| vm.new_os_error(s.to_string())) } #[cfg(windows)] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + _dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { use std::os::windows::fs as win_fs; let ret = match fs::metadata(&dst.value) { Ok(meta) => { @@ -547,7 +595,12 @@ fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResu } #[cfg(all(not(unix), not(windows)))] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { unimplemented!(); } @@ -600,36 +653,114 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "st_mtime" => ctx.new_property(StatResultRef::st_mtime), }); - py_module!(vm, "_os", { - "open" => ctx.new_rustfunc(os_open), + struct SupportFunc<'a> { + name: &'a str, + func_obj: PyObjectRef, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + }; + impl<'a> SupportFunc<'a> { + fn new( + vm: &VirtualMachine, + name: &'a str, + func: F, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + ) -> Self + where + F: IntoPyNativeFunc, + { + Self { + name: name, + func_obj: vm.ctx.new_rustfunc(func), + fd: fd, + dir_fd: dir_fd, + follow_symlinks: follow_symlinks, + } + } + } + let support_funcs = vec![ + SupportFunc::new(vm, "open", os_open, None, Some(false), None), + // access Some Some None + SupportFunc::new(vm, "chdir", os_chdir, Some(false), None, None), + // chflags Some, None Some + // chmod Some Some Some + // chown Some Some Some + // chroot Some None None + SupportFunc::new(vm, "listdir", os_listdir, Some(false), None, None), + SupportFunc::new(vm, "mkdir", os_mkdir, Some(false), Some(false), None), + // mkfifo Some Some None + // mknod Some Some None + // pathconf Some None None + // readlink Some Some None + SupportFunc::new(vm, "remove", os_remove, Some(false), Some(false), None), + // rename Some Some None + // replace Some Some None + SupportFunc::new(vm, "rmdir", os_rmdir, Some(false), Some(false), None), + SupportFunc::new(vm, "scandir", os_scandir, Some(false), None, None), + SupportFunc::new(vm, "stat", os_stat, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "symlink", os_symlink, None, Some(false), None), + // truncate Some None None + SupportFunc::new(vm, "unlink", os_remove, Some(false), Some(false), None), + // utime Some Some Some + ]; + let supports_fd = PySet::default().into_ref(vm); + let supports_dir_fd = PySet::default().into_ref(vm); + let supports_follow_symlinks = PySet::default().into_ref(vm); + + let module = py_module!(vm, "_os", { "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), "fsync" => ctx.new_rustfunc(os_fsync), "read" => ctx.new_rustfunc(os_read), "write" => ctx.new_rustfunc(os_write), - "remove" => ctx.new_rustfunc(os_remove), - "unlink" => ctx.new_rustfunc(os_remove), - "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), - "rmdir" => ctx.new_rustfunc(os_rmdir), - "listdir" => ctx.new_rustfunc(os_listdir), "putenv" => ctx.new_rustfunc(os_putenv), "unsetenv" => ctx.new_rustfunc(os_unsetenv), "environ" => environ, "name" => ctx.new_str(os_name), - "scandir" => ctx.new_rustfunc(os_scandir), "ScandirIter" => scandir_iter, "DirEntry" => dir_entry, "stat_result" => stat_result, - "stat" => ctx.new_rustfunc(os_stat), - "symlink" => ctx.new_rustfunc(os_symlink), "getcwd" => ctx.new_rustfunc(os_getcwd), - "chdir" => ctx.new_rustfunc(os_chdir), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), "O_NONBLOCK" => ctx.new_int(4), "O_APPEND" => ctx.new_int(8), "O_CREAT" => ctx.new_int(512) - }) + }); + + for support in support_funcs { + if support.fd.unwrap_or(false) { + supports_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.dir_fd.unwrap_or(false) { + supports_dir_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.follow_symlinks.unwrap_or(false) { + supports_follow_symlinks + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + vm.set_attr(&module, support.name, support.func_obj) + .unwrap(); + } + + extend_module!(vm, module, { + "supports_fd" => supports_fd.into_object(), + "supports_dir_fd" => supports_dir_fd.into_object(), + "supports_follow_symlinks" => supports_follow_symlinks.into_object(), + }); + + module } From 98d90c830b0c8b3e2545d0dc7895bc47c6f37172 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 12 May 2019 23:16:23 +0300 Subject: [PATCH 568/658] Add `itertools.takewhile` --- tests/snippets/stdlib_itertools.py | 44 ++++++++++++++++++++ vm/src/stdlib/itertools.rs | 66 +++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index 61c031dd7..92fbe4a29 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -82,3 +82,47 @@ with assertRaises(StopIteration): r = itertools.repeat(1, -1) with assertRaises(StopIteration): next(r) + + +# itertools.takewhile tests + +from itertools import takewhile as tw + +t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# not iterable +with assertRaises(TypeError): + tw(lambda n: n < 1, 1) + +# not callable +t = tw(5, [1, 2]) +with assertRaises(TypeError): + next(t) + +# non-bool predicate +t = tw(lambda n: n, [1, 2, 0]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# bad predicate prototype +t = tw(lambda: True, [1]) +with assertRaises(TypeError): + next(t) + +# StopIteration before attempting to call (bad) predicate +t = tw(lambda: True, []) +with assertRaises(StopIteration): + next(t) + +# doesn't try again after the first predicate failure +t = tw(lambda n: n < 1, [1, 0]) +with assertRaises(StopIteration): + next(t) +with assertRaises(StopIteration): + next(t) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 3856fe92f..e97c53bfa 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -5,8 +5,9 @@ use std::ops::{AddAssign, SubAssign}; use num_bigint::BigInt; use crate::function::OptionalArg; +use crate::obj::objbool; use crate::obj::objint::{PyInt, PyIntRef}; -use crate::obj::objiter::new_stop_iteration; +use crate::obj::objiter::{call_next, get_iter, new_stop_iteration}; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; @@ -121,6 +122,65 @@ impl PyItertoolsRepeat { } } +#[pyclass] +#[derive(Debug)] +struct PyItertoolsTakewhile { + predicate: PyObjectRef, + iterable: PyObjectRef, + stop_flag: RefCell, +} + +impl PyValue for PyItertoolsTakewhile { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "takewhile") + } +} + +#[pyimpl] +impl PyItertoolsTakewhile { + #[pymethod(name = "__new__")] + fn new( + _cls: PyClassRef, + predicate: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsTakewhile { + predicate: predicate, + iterable: iter, + stop_flag: RefCell::new(false), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if *self.stop_flag.borrow() { + return Err(new_stop_iteration(vm)); + } + + // might be StopIteration or anything else, which is propaged upwwards + let obj = call_next(vm, &self.iterable)?; + + let verdict = vm.invoke(self.predicate.clone(), vec![obj.clone()])?; + let verdict = objbool::boolval(vm, verdict)?; + if verdict { + Ok(obj) + } else { + *self.stop_flag.borrow_mut() = true; + Err(new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -130,8 +190,12 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let repeat = ctx.new_class("repeat", ctx.object()); PyItertoolsRepeat::extend_class(ctx, &repeat); + let takewhile = ctx.new_class("takewhile", ctx.object()); + PyItertoolsTakewhile::extend_class(ctx, &takewhile); + py_module!(vm, "itertools", { "count" => count, "repeat" => repeat, + "takewhile" => takewhile, }) } From 29fff5ab4d2ca6ea558fbbb40ac493252ce02529 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Mon, 13 May 2019 11:41:46 +0900 Subject: [PATCH 569/658] Remove types module --- vm/src/stdlib/mod.rs | 2 -- vm/src/stdlib/types.rs | 39 --------------------------------------- 2 files changed, 41 deletions(-) delete mode 100644 vm/src/stdlib/types.rs diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 830aa2dca..149f37d9e 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -14,7 +14,6 @@ mod string; mod thread; mod time_module; mod tokenize; -mod types; mod weakref; use std::collections::HashMap; @@ -49,7 +48,6 @@ pub fn get_module_inits() -> HashMap { modules.insert("_thread".to_string(), Box::new(thread::make_module)); modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); - modules.insert("types".to_string(), Box::new(types::make_module)); modules.insert("_weakref".to_string(), Box::new(weakref::make_module)); // disable some modules on WASM diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs deleted file mode 100644 index e224a93c6..000000000 --- a/vm/src/stdlib/types.rs +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Dynamic type creation and names for built in types. - */ - -use crate::function::OptionalArg; -use crate::obj::objdict::PyDict; -use crate::obj::objstr::PyStringRef; -use crate::obj::objtype; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; -use crate::VirtualMachine; - -fn types_new_class( - name: PyStringRef, - bases: OptionalArg>, - vm: &VirtualMachine, -) -> PyResult { - // TODO kwds and exec_body parameter - - let bases = match bases { - OptionalArg::Present(bases) => bases, - OptionalArg::Missing => PyIterable::try_from_object(vm, vm.ctx.new_tuple(vec![]))?, - }; - let dict = PyDict::default().into_ref(vm); - objtype::type_new_class(vm, vm.ctx.type_type(), name, bases, dict) -} - -pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - - py_module!(vm, "types", { - "new_class" => ctx.new_rustfunc(types_new_class), - "FunctionType" => ctx.function_type(), - "MethodType" => ctx.bound_method_type(), - "LambdaType" => ctx.function_type(), - "CodeType" => ctx.code_type(), - "FrameType" => ctx.frame_type() - }) -} From b048a66bc330ea9f9fc9194e8c9cba48a4b953c1 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Mon, 13 May 2019 11:42:16 +0900 Subject: [PATCH 570/658] Copy types.py from CPython --- Lib/types.py | 295 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 Lib/types.py diff --git a/Lib/types.py b/Lib/types.py new file mode 100644 index 000000000..2e0513ca1 --- /dev/null +++ b/Lib/types.py @@ -0,0 +1,295 @@ +""" +Define names for built-in types that aren't directly accessible as a builtin. +""" +import sys + +# Iterators in Python aren't a matter of type but of protocol. A large +# and changing number of builtin types implement *some* flavor of +# iterator. Don't check the type! Use hasattr to check for both +# "__iter__" and "__next__" attributes instead. + +def _f(): pass +FunctionType = type(_f) +LambdaType = type(lambda: None) # Same as FunctionType +CodeType = type(_f.__code__) +MappingProxyType = type(type.__dict__) +SimpleNamespace = type(sys.implementation) + +def _g(): + yield 1 +GeneratorType = type(_g()) + +async def _c(): pass +_c = _c() +CoroutineType = type(_c) +_c.close() # Prevent ResourceWarning + +async def _ag(): + yield +_ag = _ag() +AsyncGeneratorType = type(_ag) + +class _C: + def _m(self): pass +MethodType = type(_C()._m) + +BuiltinFunctionType = type(len) +BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + +WrapperDescriptorType = type(object.__init__) +MethodWrapperType = type(object().__str__) +MethodDescriptorType = type(str.join) +ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + +ModuleType = type(sys) + +try: + raise TypeError +except TypeError: + tb = sys.exc_info()[2] + TracebackType = type(tb) + FrameType = type(tb.tb_frame) + tb = None; del tb + +# For Jython, the following two types are identical +GetSetDescriptorType = type(FunctionType.__code__) +MemberDescriptorType = type(FunctionType.__globals__) + +del sys, _f, _g, _C, _c, _ag # Not for export + + +# Provide a PEP 3115 compliant mechanism for class creation +def new_class(name, bases=(), kwds=None, exec_body=None): + """Create a class object dynamically using the appropriate metaclass.""" + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) + if exec_body is not None: + exec_body(ns) + if resolved_bases is not bases: + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + shift = 0 + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entries__"): + continue + new_base = base.__mro_entries__(bases) + updated = True + if not isinstance(new_base, tuple): + raise TypeError("__mro_entries__ must return a tuple") + else: + new_bases[i+shift:i+shift+1] = new_base + shift += len(new_base) - 1 + if not updated: + return bases + return tuple(new_bases) + +def prepare_class(name, bases=(), kwds=None): + """Call the __prepare__ method of the appropriate metaclass. + + Returns (metaclass, namespace, kwds) as a 3-tuple + + *metaclass* is the appropriate metaclass + *namespace* is the prepared class namespace + *kwds* is an updated copy of the passed in kwds argument with any + 'metaclass' entry removed. If no kwds argument is passed in, this will + be an empty dict. + """ + if kwds is None: + kwds = {} + else: + kwds = dict(kwds) # Don't alter the provided mapping + if 'metaclass' in kwds: + meta = kwds.pop('metaclass') + else: + if bases: + meta = type(bases[0]) + else: + meta = type + if isinstance(meta, type): + # when meta is a type, we first determine the most-derived metaclass + # instead of invoking the initial candidate directly + meta = _calculate_meta(meta, bases) + if hasattr(meta, '__prepare__'): + ns = meta.__prepare__(name, bases, **kwds) + else: + ns = {} + return meta, ns, kwds + +def _calculate_meta(meta, bases): + """Calculate the most derived metaclass.""" + winner = meta + for base in bases: + base_meta = type(base) + if issubclass(winner, base_meta): + continue + if issubclass(base_meta, winner): + winner = base_meta + continue + # else: + raise TypeError("metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases") + return winner + +class DynamicClassAttribute: + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + This allows one to have properties active on an instance, and have virtual + attributes on the class with the same name (see Enum for an example). + + """ + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + # next two lines make DynamicClassAttribute act the same as property + self.__doc__ = doc or fget.__doc__ + self.overwrite_doc = doc is None + # support for abstract methods + self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False)) + + def __get__(self, instance, ownerclass=None): + if instance is None: + if self.__isabstractmethod__: + return self + raise AttributeError() + elif self.fget is None: + raise AttributeError("unreadable attribute") + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError("can't set attribute") + self.fset(instance, value) + + def __delete__(self, instance): + if self.fdel is None: + raise AttributeError("can't delete attribute") + self.fdel(instance) + + def getter(self, fget): + fdoc = fget.__doc__ if self.overwrite_doc else None + result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def setter(self, fset): + result = type(self)(self.fget, fset, self.fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def deleter(self, fdel): + result = type(self)(self.fget, self.fset, fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + +class _GeneratorWrapper: + # TODO: Implement this in C. + def __init__(self, gen): + self.__wrapped = gen + self.__isgen = gen.__class__ is GeneratorType + self.__name__ = getattr(gen, '__name__', None) + self.__qualname__ = getattr(gen, '__qualname__', None) + def send(self, val): + return self.__wrapped.send(val) + def throw(self, tp, *rest): + return self.__wrapped.throw(tp, *rest) + def close(self): + return self.__wrapped.close() + @property + def gi_code(self): + return self.__wrapped.gi_code + @property + def gi_frame(self): + return self.__wrapped.gi_frame + @property + def gi_running(self): + return self.__wrapped.gi_running + @property + def gi_yieldfrom(self): + return self.__wrapped.gi_yieldfrom + cr_code = gi_code + cr_frame = gi_frame + cr_running = gi_running + cr_await = gi_yieldfrom + def __next__(self): + return next(self.__wrapped) + def __iter__(self): + if self.__isgen: + return self.__wrapped + return self + __await__ = __iter__ + +def coroutine(func): + """Convert regular generator function to a coroutine.""" + + if not callable(func): + raise TypeError('types.coroutine() expects a callable') + + if (func.__class__ is FunctionType and + getattr(func, '__code__', None).__class__ is CodeType): + + co_flags = func.__code__.co_flags + + # Check if 'func' is a coroutine function. + # (0x180 == CO_COROUTINE | CO_ITERABLE_COROUTINE) + if co_flags & 0x180: + return func + + # Check if 'func' is a generator function. + # (0x20 == CO_GENERATOR) + if co_flags & 0x20: + # TODO: Implement this in C. + co = func.__code__ + func.__code__ = CodeType( + co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, + co.co_stacksize, + co.co_flags | 0x100, # 0x100 == CO_ITERABLE_COROUTINE + co.co_code, + co.co_consts, co.co_names, co.co_varnames, co.co_filename, + co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, + co.co_cellvars) + return func + + # The following code is primarily to support functions that + # return generator-like objects (for instance generators + # compiled with Cython). + + # Delay functools and _collections_abc import for speeding up types import. + import functools + import _collections_abc + @functools.wraps(func) + def wrapped(*args, **kwargs): + coro = func(*args, **kwargs) + if (coro.__class__ is CoroutineType or + coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100): + # 'coro' is a native coroutine object or an iterable coroutine + return coro + if (isinstance(coro, _collections_abc.Generator) and + not isinstance(coro, _collections_abc.Coroutine)): + # 'coro' is either a pure Python generator iterator, or it + # implements collections.abc.Generator (and does not implement + # collections.abc.Coroutine). + return _GeneratorWrapper(coro) + # 'coro' is either an instance of collections.abc.Coroutine or + # some other object -- pass it through. + return coro + + return wrapped + + +__all__ = [n for n in globals() if n[:1] != '_'] From b76cb680e9680be2054f78b4da6326d02634768e Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Mon, 13 May 2019 11:44:12 +0900 Subject: [PATCH 571/658] Comment out unsupported code from types.py --- Lib/types.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Lib/types.py b/Lib/types.py index 2e0513ca1..4ef461961 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -13,21 +13,21 @@ FunctionType = type(_f) LambdaType = type(lambda: None) # Same as FunctionType CodeType = type(_f.__code__) MappingProxyType = type(type.__dict__) -SimpleNamespace = type(sys.implementation) +# SimpleNamespace = type(sys.implementation) def _g(): yield 1 GeneratorType = type(_g()) -async def _c(): pass -_c = _c() -CoroutineType = type(_c) -_c.close() # Prevent ResourceWarning +# async def _c(): pass +# _c = _c() +# CoroutineType = type(_c) +# _c.close() # Prevent ResourceWarning -async def _ag(): - yield -_ag = _ag() -AsyncGeneratorType = type(_ag) +# async def _ag(): +# yield +# _ag = _ag() +# AsyncGeneratorType = type(_ag) class _C: def _m(self): pass @@ -43,19 +43,19 @@ ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) ModuleType = type(sys) -try: - raise TypeError -except TypeError: - tb = sys.exc_info()[2] - TracebackType = type(tb) - FrameType = type(tb.tb_frame) - tb = None; del tb +# try: +# raise TypeError +# except TypeError: +# tb = sys.exc_info()[2] +# TracebackType = type(tb) +# FrameType = type(tb.tb_frame) +# tb = None; del tb # For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) -MemberDescriptorType = type(FunctionType.__globals__) +# MemberDescriptorType = type(FunctionType.__globals__) -del sys, _f, _g, _C, _c, _ag # Not for export +del sys, _f, _g, _C # Not for export # Provide a PEP 3115 compliant mechanism for class creation From 5cff843782c598a16fa3a748a2d27c2f593f43f5 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Mon, 13 May 2019 19:41:06 +0900 Subject: [PATCH 572/658] Implement re.escape --- tests/snippets/test_re.py | 2 ++ vm/src/stdlib/re.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tests/snippets/test_re.py b/tests/snippets/test_re.py index 2463f380b..2bc7308e6 100644 --- a/tests/snippets/test_re.py +++ b/tests/snippets/test_re.py @@ -11,3 +11,5 @@ print(mo) # assert isinstance(mo, re.Match) assert mo.start() == 1 assert mo.end() == 5 + +assert re.escape('python.exe') == 'python\\.exe' diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index b9f921502..f42738980 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -79,6 +79,10 @@ fn re_compile(pattern: PyStringRef, vm: &VirtualMachine) -> PyResult { make_regex(vm, &pattern.value) } +fn re_escape(pattern: PyStringRef, _vm: &VirtualMachine) -> String { + regex::escape(&pattern.value) +} + impl PyRegexRef { fn match_(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { do_match(vm, &self, &text.value) @@ -113,6 +117,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { py_module!(vm, "re", { "compile" => ctx.new_rustfunc(re_compile), + "escape" => ctx.new_rustfunc(re_escape), "Match" => match_type, "match" => ctx.new_rustfunc(re_match), "Pattern" => pattern_type, From 189bf171ab42fe08d6b90385d88d05a3545475ef Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Tue, 14 May 2019 12:24:44 +0900 Subject: [PATCH 573/658] Remove extern crate unneeded for Rust 2018 --- vm/src/lib.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 8b14454fe..f21d5dcf4 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -20,13 +20,6 @@ extern crate lexical; #[macro_use] extern crate log; // extern crate env_logger; -extern crate num_bigint; -extern crate num_complex; -extern crate num_integer; -extern crate num_traits; -extern crate serde; -extern crate serde_json; -extern crate statrs; extern crate rustpython_parser; #[macro_use] From 68a1a9f6480214e1f3c541ce5e4dd543643d4d61 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Tue, 14 May 2019 16:56:50 +0800 Subject: [PATCH 574/658] add to_bytes for int --- tests/snippets/ints.py | 7 +++++ vm/src/obj/objint.rs | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index be41c3257..b947797ac 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -79,6 +79,13 @@ assert int.from_bytes(b'\x00\x10', 'little') == 4096 assert int.from_bytes(b'\xfc\x00', 'big', signed=True) == -1024 assert int.from_bytes(b'\xfc\x00', 'big', signed=False) == 64512 +assert (1024).to_bytes(4, 'big') == b'\x00\x00\x04\x00' +assert (1024).to_bytes(2, 'little', signed=True) == b'\x00\x04' +assert (-1024).to_bytes(4, 'big', signed=True) == b'\xff\xff\xfc\x00' +assert (-1024).to_bytes(4, 'little', signed=True) == b'\x00\xfc\xff\xff' +assert (2147483647).to_bytes(8, 'big', signed=False) == b'\x00\x00\x00\x00\x7f\xff\xff\xff' +assert (-2147483648).to_bytes(8, 'little', signed=True) == b'\x00\x00\x00\x80\xff\xff\xff\xff' + with assertRaises(TypeError): int(base=2) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 403fd08ef..a8c74aaf7 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -14,6 +14,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use super::objbyteinner::PyByteInner; +use super::objbytes::PyBytes; use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -526,7 +527,74 @@ impl PyInt { } Ok(x) } + #[pymethod] + fn to_bytes( + &self, + length: PyIntRef, + byteorder: PyStringRef, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let mut signed = false; + let value = self.as_bigint(); + for (key, value) in kwargs.into_iter() { + if key == "signed" { + signed = match_class!(value, + b @ PyInt => !b.as_bigint().is_zero(), + _ => false, + ); + } + } + if value.sign() == Sign::Minus && signed == false { + return Err(vm.new_overflow_error("can't convert negative int to unsigned".to_string())); + } + let byte_len; + if let Some(temp) = length.as_bigint().to_usize() { + byte_len = temp; + } else { + return Err(vm.new_value_error("length parameter is illegal".to_string())); + } + + let mut origin_bytes = match byteorder.value.as_str() { + "big" => match signed { + true => value.to_signed_bytes_be(), + false => value.to_bytes_be().1, + }, + "little" => match signed { + true => value.to_signed_bytes_le(), + false => value.to_bytes_le().1, + }, + _ => { + return Err( + vm.new_value_error("byteorder must be either 'little' or 'big'".to_string()) + ); + } + }; + let origin_len = origin_bytes.len(); + if origin_len > byte_len { + return Err(vm.new_value_error("int too big to convert".to_string())); + } + + let mut append_bytes = match value.sign() { + Sign::Minus => vec![255u8; byte_len - origin_len], + _ => vec![0u8; byte_len - origin_len], + }; + let mut bytes = vec![]; + match byteorder.value.as_str() { + "big" => { + bytes = append_bytes; + bytes.append(&mut origin_bytes); + } + "little" => { + bytes = origin_bytes; + bytes.append(&mut append_bytes); + } + _ => (), + } + + Ok(PyBytes::new(bytes)) + } #[pyproperty] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf From 13187fad5c10162b7282df39b8b6d528c7d8a2fb Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 14 May 2019 03:30:25 +0900 Subject: [PATCH 575/658] Fix dictdatatype::Dict eq to regard __bool__ --- vm/src/dictdatatype.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index a1aa9e3c3..a075ec937 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -222,7 +222,7 @@ fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { /// Invoke __eq__ on two keys fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { let result = vm._eq(key1.clone(), key2.clone())?; - Ok(objbool::get_value(&result)) + objbool::boolval(vm, result) } #[cfg(test)] From cc9dcbc9f811fe9cbc49ba2e1fea76e2de9d6792 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 02:26:30 +0900 Subject: [PATCH 576/658] Refactor dictdatatype::Dict --- tests/snippets/dict.py | 17 ++++++++ vm/src/dictdatatype.rs | 93 +++++++++++++++++++++++++++--------------- vm/src/obj/objdict.rs | 14 +++---- 3 files changed, 81 insertions(+), 43 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f7506a205..4545f7f58 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -200,3 +200,20 @@ w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) + +class A: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, A) +class B: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, B) + +s = {1: 0, A(): 1, B(): 2} +assert len(s) == 3 +assert s[1] == 0 +assert s[A()] == 1 +assert s[B()] == 2 diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index a075ec937..858cf43e1 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,16 +1,25 @@ use crate::obj::objbool; +use crate::pyhash; use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 /// And: http://code.activestate.com/recipes/578375/ -use std::collections::HashMap; +use std::collections::{hash_map::DefaultHasher, HashMap}; +use std::hash::{Hash, Hasher}; + +/// hash value of an object returned by __hash__ +type HashValue = pyhash::PyHash; +/// index calculated by resolving collision +type HashIndex = pyhash::PyHash; +/// entry index mapped in indices +type EntryIndex = usize; #[derive(Clone)] pub struct Dict { size: usize, - indices: HashMap, + indices: HashMap, entries: Vec>>, } @@ -26,7 +35,7 @@ impl Default for Dict { #[derive(Clone)] struct DictEntry { - hash: usize, + hash: HashValue, key: PyObjectRef, value: T, } @@ -38,12 +47,27 @@ pub struct DictSize { } impl Dict { - pub fn new() -> Self { - Dict { - size: 0, - indices: HashMap::new(), - entries: Vec::new(), - } + fn unchecked_push( + &mut self, + hash_index: HashIndex, + hash_value: HashValue, + key: &PyObjectRef, + value: T, + ) { + let entry = DictEntry { + hash: hash_value, + key: key.clone(), + value, + }; + let entry_index = self.entries.len(); + self.entries.push(Some(entry)); + self.indices.insert(hash_index, entry_index); + self.size += 1; + } + + fn unchecked_delete(&mut self, entry_index: EntryIndex) { + self.entries[entry_index] = None; + self.size -= 1; } /// Store a key @@ -63,29 +87,21 @@ impl Dict { hash_value, } => { // New key: - let entry = DictEntry { - hash: hash_value, - key: key.clone(), - value, - }; - let index = self.entries.len(); - self.entries.push(Some(entry)); - self.indices.insert(hash_index, index); - self.size += 1; + self.unchecked_push(hash_index, hash_value, key, value); Ok(()) } } } pub fn contains(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - if let LookupResult::Existing(_index) = self.lookup(vm, key)? { + if let LookupResult::Existing(_) = self.lookup(vm, key)? { Ok(true) } else { Ok(false) } } - fn unchecked_get(&self, index: usize) -> T { + fn unchecked_get(&self, index: EntryIndex) -> T { if let Some(entry) = &self.entries[index] { entry.value.clone() } else { @@ -110,9 +126,7 @@ impl Dict { /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { - if let LookupResult::Existing(index) = self.lookup(vm, key)? { - self.entries[index] = None; - self.size -= 1; + if self.delete_if_exists(vm, key)? { Ok(()) } else { let key_repr = vm.to_pystr(key)?; @@ -120,6 +134,15 @@ impl Dict { } } + pub fn delete_if_exists(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + if let LookupResult::Existing(entry_index) = self.lookup(vm, key)? { + self.unchecked_delete(entry_index); + Ok(true) + } else { + Ok(false) + } + } + pub fn len(&self) -> usize { self.size } @@ -135,7 +158,7 @@ impl Dict { } } - pub fn next_entry(&self, position: &mut usize) -> Option<(&PyObjectRef, &T)> { + pub fn next_entry(&self, position: &mut EntryIndex) -> Option<(&PyObjectRef, &T)> { while *position < self.entries.len() { if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { *position += 1; @@ -152,9 +175,9 @@ impl Dict { /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash_value = calc_hash(vm, key)?; + let hash_value = collection_hash(vm, key)?; let perturb = hash_value; - let mut hash_index: usize = hash_value; + let mut hash_index: HashIndex = hash_value; loop { if self.indices.contains_key(&hash_index) { // Now we have an index, lets check the key. @@ -197,8 +220,7 @@ impl Dict { pub fn pop(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { if let LookupResult::Existing(index) = self.lookup(vm, key)? { let value = self.unchecked_get(index); - self.entries[index] = None; - self.size -= 1; + self.unchecked_delete(index); Ok(value) } else { let key_repr = vm.to_pystr(key)?; @@ -209,14 +231,17 @@ impl Dict { enum LookupResult { NewIndex { - hash_value: usize, - hash_index: usize, + hash_value: HashValue, + hash_index: HashIndex, }, // return not found, index into indices - Existing(usize), // Existing record, index into entries + Existing(EntryIndex), // Existing record, index into entries } -fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - Ok(vm._hash(key)? as usize) +fn collection_hash(vm: &VirtualMachine, object: &PyObjectRef) -> PyResult { + let raw_hash = vm._hash(object)?; + let mut hasher = DefaultHasher::new(); + raw_hash.hash(&mut hasher); + Ok(hasher.finish() as HashValue) } /// Invoke __eq__ on two keys @@ -232,7 +257,7 @@ mod tests { #[test] fn test_insert() { let mut vm = VirtualMachine::new(); - let mut dict = Dict::new(); + let mut dict = Dict::default(); assert_eq!(0, dict.len()); let key1 = vm.new_bool(true); diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 396c711f4..87e4b1919 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -264,15 +264,11 @@ impl PyDictRef { fn popitem(self, vm: &VirtualMachine) -> PyResult { let mut entries = self.entries.borrow_mut(); - let (key, value) = match entries.next_entry(&mut 0) { - Some((key, value)) => (key.clone(), value.clone()), - None => { - return Err(vm.new_key_error("popitem(): dictionary is empty".to_string())); - } - }; - - entries.delete(vm, &key)?; - Ok(vm.ctx.new_tuple(vec![key, value])) + if let Some((key, value)) = entries.pop_front() { + Ok(vm.ctx.new_tuple(vec![key, value])) + } else { + Err(vm.new_key_error("popitem(): dictionary is empty".to_string())) + } } /// Take a python dictionary and convert it to attributes. From 81186ea5b7df966e678e9ab52c2e61a13f301704 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 9 May 2019 02:27:32 +0900 Subject: [PATCH 577/658] Fix set() hash collision based on dictdatatype::Dict --- tests/snippets/set.py | 35 ++++++- vm/src/dictdatatype.rs | 36 ++++++++ vm/src/obj/objset.rs | 205 ++++++++++++----------------------------- 3 files changed, 128 insertions(+), 148 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index ba20fbae6..65a568f9d 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -103,7 +103,7 @@ a = set([1,2]) b = a.pop() assert b in [1,2] c = a.pop() -assert (c in [1,2] and c != b) +assert (c in [1,2] and c != b) assert_raises(KeyError, lambda: a.pop()) a = set([1,2,3]) @@ -268,3 +268,36 @@ assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) + +class A: + def __hash__(self): + return 1 +class B: + def __hash__(self): + return 1 + +s = {1, A(), B()} +assert len(s) == 3 + +s = {True} +s.add(1.0) +assert str(s) == '{True}' + +class EqObject: + def __init__(self, eq): + self.eq = eq + def __eq__(self, other): + return self.eq + def __hash__(self): + return bool(self.eq) + +assert 'x' == (EqObject('x') == EqObject('x')) +s = {EqObject('x')} +assert EqObject('x') in s +assert '[]' == (EqObject('[]') == EqObject('[]')) +s = {EqObject([])} +assert EqObject([]) not in s +x = object() +assert x == (EqObject(x) == EqObject(x)) +s = {EqObject(x)} +assert EqObject(x) in s diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 858cf43e1..788587266 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -143,6 +143,22 @@ impl Dict { } } + pub fn delete_or_insert( + &mut self, + vm: &VirtualMachine, + key: &PyObjectRef, + value: T, + ) -> PyResult<()> { + match self.lookup(vm, &key)? { + LookupResult::Existing(entry_index) => self.unchecked_delete(entry_index), + LookupResult::NewIndex { + hash_value, + hash_index, + } => self.unchecked_push(hash_index, hash_value, &key, value), + }; + Ok(()) + } + pub fn len(&self) -> usize { self.size } @@ -173,6 +189,14 @@ impl Dict { position.size != self.size || self.entries.len() != position.entries_size } + pub fn keys<'a>(&'a self) -> Box + 'a> { + Box::new( + self.entries + .iter() + .filter_map(|v| v.as_ref().map_or(None, |v| Some(v.key.clone()))), + ) + } + /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { let hash_value = collection_hash(vm, key)?; @@ -227,6 +251,18 @@ impl Dict { Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } + + pub fn pop_front(&mut self) -> Option<(PyObjectRef, T)> { + let mut entry_index = 0; + match self.next_entry(&mut entry_index) { + Some((key, value)) => { + let item = (key.clone(), value.clone()); + self.unchecked_delete(entry_index - 1); + Some(item) + } + None => None, + } + } } enum LookupResult { diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index aed3f8351..e404fa924 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -3,10 +3,9 @@ */ use std::cell::{Cell, RefCell}; -use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; -use std::hash::{Hash, Hasher}; +use crate::dictdatatype; use crate::function::OptionalArg; use crate::pyobject::{ PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, @@ -14,12 +13,12 @@ use crate::pyobject::{ }; use crate::vm::{ReprGuard, VirtualMachine}; -use super::objbool; -use super::objint; use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; +pub type SetContentType = dictdatatype::Dict<()>; + #[derive(Default)] pub struct PySet { inner: RefCell, @@ -60,42 +59,32 @@ impl PyValue for PyFrozenSet { #[derive(Default, Clone)] struct PySetInner { - elements: HashMap, + content: SetContentType, } impl PySetInner { fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - for item in iterable.iter(vm)? { - insert_into_set(vm, &mut elements, &item?)?; - } - elements + let mut set = PySetInner::default(); + if let OptionalArg::Present(iterable) = iterable { + for item in iterable.iter(vm)? { + set.add(&item?, vm)?; } - }; - - Ok(PySetInner { elements }) + } + Ok(set) } fn len(&self) -> usize { - self.elements.len() + self.content.len() } + fn copy(&self) -> PySetInner { PySetInner { - elements: self.elements.clone(), + content: self.content.clone(), } } - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for element in self.elements.iter() { - let value = vm._eq(needle.clone(), element.1.clone())?; - if objbool::get_value(&value) { - return Ok(true); - } - } - Ok(false) + fn contains(&self, needle: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.contains(vm, needle) } fn _compare_inner( @@ -123,8 +112,8 @@ impl PySetInner { if size_func(get_zelf(swap).len(), get_other(swap).len()) { return Ok(vm.new_bool(false)); } - for element in get_other(swap).elements.iter() { - if !get_zelf(swap).contains(element.1.clone(), vm)? { + for key in get_other(swap).content.keys() { + if !get_zelf(swap).contains(&key, vm)? { return Ok(vm.new_bool(false)); } } @@ -177,46 +166,38 @@ impl PySetInner { } fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.clone(); + let mut set = self.clone(); for item in other.iter(vm)? { - insert_into_set(vm, &mut elements, &item?)?; + set.add(&item?, vm)?; } - Ok(PySetInner { elements }) + Ok(set) } fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = HashMap::new(); + let mut set = PySetInner::default(); for item in other.iter(vm)? { let obj = item?; - if self.contains(obj.clone(), vm)? { - insert_into_set(vm, &mut elements, &obj)?; + if self.contains(&obj, vm)? { + set.add(&obj, vm)?; } } - Ok(PySetInner { elements }) + Ok(set) } fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.clone(); + let mut set = self.copy(); for item in other.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { - remove_from_set(vm, &mut elements, &obj)?; - } + set.content.delete_if_exists(vm, &item?)?; } - Ok(PySetInner { elements }) + Ok(set) } fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut new_inner = self.clone(); for item in other.iter(vm)? { - let obj = item?; - if !self.contains(obj.clone(), vm)? { - new_inner.add(&obj, vm)?; - } else { - new_inner.remove(&obj, vm)?; - } + new_inner.content.delete_or_insert(vm, &item?, ())? } Ok(new_inner) @@ -224,8 +205,7 @@ impl PySetInner { fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { for item in other.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { + if self.contains(&item?, vm)? { return Ok(false); } } @@ -233,7 +213,7 @@ impl PySetInner { } fn iter(&self, vm: &VirtualMachine) -> PyListIterator { - let items = self.elements.values().cloned().collect(); + let items = self.content.keys().collect(); let set_list = vm.ctx.new_list(items); PyListIterator { position: Cell::new(0), @@ -243,52 +223,43 @@ impl PySetInner { fn repr(&self, vm: &VirtualMachine) -> PyResult { let mut str_parts = vec![]; - for elem in self.elements.values() { - let part = vm.to_repr(elem)?; + for key in self.content.keys() { + let part = vm.to_repr(&key)?; str_parts.push(part.value.clone()); } Ok(format!("{{{}}}", str_parts.join(", "))) } - fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - insert_into_set(vm, &mut self.elements, &item) + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.insert(vm, item, ()) } - fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - remove_from_set(vm, &mut self.elements, &item) + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.delete(vm, item) } - fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, &mut self.elements, &item, &discard) + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.delete_if_exists(vm, item) } fn clear(&mut self) { - self.elements.clear(); + self.content.clear() } fn pop(&mut self, vm: &VirtualMachine) -> PyResult { - let elements = &mut self.elements; - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), + if let Some((key, _)) = self.content.pop_front() { + Ok(key) + } else { + Err(vm.new_key_error("pop from an empty set".to_string())) } } - fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { for item in iterable.iter(vm)? { - insert_into_set(vm, &mut self.elements, &item?)?; + self.add(&item?, vm)?; } - Ok(vm.get_none()) + Ok(()) } fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { @@ -296,7 +267,7 @@ impl PySetInner { self.clear(); for item in iterable.iter(vm)? { let obj = item?; - if temp_inner.contains(obj.clone(), vm)? { + if temp_inner.contains(&obj, vm)? { self.add(&obj, vm)?; } } @@ -305,10 +276,7 @@ impl PySetInner { fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { for item in iterable.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { - self.remove(&obj, vm)?; - } + self.content.delete_if_exists(vm, &item?)?; } Ok(vm.get_none()) } @@ -319,12 +287,7 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult { for item in iterable.iter(vm)? { - let obj = item?; - if !self.contains(obj.clone(), vm)? { - self.add(&obj, vm)?; - } else { - self.remove(&obj, vm)?; - } + self.content.delete_or_insert(vm, &item?, ())?; } Ok(vm.get_none()) } @@ -353,7 +316,7 @@ impl PySetRef { } fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().contains(needle, vm) + self.inner.borrow().contains(&needle, vm) } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -472,16 +435,18 @@ impl PySetRef { Ok(vm.new_str(s)) } - pub fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().add(&item, vm) + pub fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().add(&item, vm)?; + Ok(()) } - fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().remove(&item, vm) } - fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().discard(&item, vm) + fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().discard(&item, vm)?; + Ok(()) } fn clear(self, _vm: &VirtualMachine) { @@ -564,7 +529,7 @@ impl PyFrozenSetRef { } fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.contains(needle, vm) + self.inner.contains(&needle, vm) } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -684,60 +649,6 @@ impl PyFrozenSetRef { } } -fn perform_action_with_hash( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, - f: &Fn(&VirtualMachine, &mut HashMap, u64, &PyObjectRef) -> PyResult, -) -> PyResult { - let hash: PyObjectRef = vm.call_method(item, "__hash__", vec![])?; - - let hash_value = objint::get_value(&hash); - let mut hasher = DefaultHasher::new(); - hash_value.hash(&mut hasher); - let key = hasher.finish(); - f(vm, elements, key, item) -} - -fn insert_into_set( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, -) -> PyResult { - fn insert( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - elements.insert(key, value.clone()); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, elements, item, &insert) -} - -fn remove_from_set( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, -) -> PyResult { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, elements, item, &remove) -} - struct SetIterable { iterable: PyIterable, } From 87fc4cc470196ae37ea3d6d434d9b1e59ee67646 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 04:58:53 +0900 Subject: [PATCH 578/658] Add complex.__hash__ --- tests/snippets/builtin_complex.py | 8 ++++++++ vm/src/obj/objcomplex.rs | 9 +++++++++ vm/src/pyhash.rs | 8 ++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 8d883a9dd..06d46a980 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -86,6 +86,14 @@ assert bool(complex(0, 0)) is False assert bool(complex(0, 1)) is True assert bool(complex(1, 0)) is True +# __hash__ + +assert hash(complex(1)) == hash(float(1)) == hash(int(1)) +assert hash(complex(-1)) == hash(float(-1)) == hash(int(-1)) +assert hash(complex(3.14)) == hash(float(3.14)) +assert hash(complex(-float('inf'))) == hash(-float('inf')) +assert hash(1j) != hash(1) + # numbers.Complex a = complex(3, 4) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 677763213..a349354a2 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -2,6 +2,7 @@ use num_complex::Complex64; use num_traits::Zero; use crate::function::OptionalArg; +use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; @@ -248,4 +249,12 @@ impl PyComplex { let value = Complex64::new(real, imag); PyComplex { value }.into_ref_with_type(vm, cls) } + + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + let re_hash = pyhash::hash_float(self.value.re); + let im_hash = pyhash::hash_float(self.value.im); + + re_hash + im_hash * pyhash::IMAG + } } diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs index f87337359..1f0f3d121 100644 --- a/vm/src/pyhash.rs +++ b/vm/src/pyhash.rs @@ -7,12 +7,16 @@ use crate::vm::VirtualMachine; pub type PyHash = i64; pub type PyUHash = u64; +/// Prime multiplier used in string and various other hashes. +pub const MULTIPLIER: PyHash = 1000003; // 0xf4243 +/// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 pub const BITS: usize = 61; pub const MODULUS: PyUHash = (1 << BITS) - 1; -// pub const CUTOFF: usize = 7; - pub const INF: PyHash = 314159; pub const NAN: PyHash = 0; +pub const IMAG: PyHash = MULTIPLIER; + +// pub const CUTOFF: usize = 7; pub fn hash_float(value: f64) -> PyHash { // cpython _Py_HashDouble From 83c5a3d33064c95b592b8f9d59033208cfa99c30 Mon Sep 17 00:00:00 2001 From: ypyf Date: Thu, 16 May 2019 20:35:40 +0800 Subject: [PATCH 579/658] improve error message for type_new --- vm/src/obj/objtype.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index c66e57354..29c66ffd1 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -248,7 +248,7 @@ pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let (typ, name, bases, dict) = args.bind(vm)?; type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { - Err(vm.new_type_error(format!(": type_new: {:?}", args))) + Err(vm.new_type_error("type() takes 1 or 3 arguments".to_string())) } } From d223af645b8bace49842717e23792b2a0fb6e3d2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 05:40:35 +0900 Subject: [PATCH 580/658] Add math.frexp --- tests/snippets/math_module.py | 11 ++++++++++- vm/src/obj/objfloat.rs | 16 ++++++++++++++++ vm/src/pyhash.rs | 10 ++-------- vm/src/stdlib/math.rs | 19 +++++++++++++++++++ 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/snippets/math_module.py b/tests/snippets/math_module.py index 1064610f1..dc7fc0706 100644 --- a/tests/snippets/math_module.py +++ b/tests/snippets/math_module.py @@ -1,5 +1,5 @@ import math -from testutils import assertRaises +from testutils import assertRaises, assert_raises # assert(math.exp(2) == math.exp(2.0)) # assert(math.exp(True) == math.exp(1.0)) @@ -78,3 +78,12 @@ with assertRaises(TypeError): math.ceil(object()) with assertRaises(TypeError): math.floor(object()) + +assert str(math.frexp(0.0)) == str((+0.0, 0)) +assert str(math.frexp(-0.0)) == str((-0.0, 0)) +assert math.frexp(1) == (0.5, 1) +assert math.frexp(1.5) == (0.75, 1) + +assert math.frexp(float('inf')) == (float('inf'), 0) +assert str(math.frexp(float('nan'))) == str((float('nan'), 0)) +assert_raises(TypeError, lambda: math.frexp(None)) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 8d983c3ae..45e8d6624 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -298,6 +298,11 @@ impl PyFloat { ) } + #[pymethod(name = "__pos__")] + fn pos(&self, _vm: &VirtualMachine) -> f64 { + self.value + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value @@ -475,6 +480,17 @@ impl PyFloat { } } +pub fn ufrexp(value: f64) -> (f64, i32) { + if 0.0 == value { + (0.0, 0i32) + } else { + let bits = value.to_bits(); + let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; + let mantissa_bits = bits & (0x000fffffffffffff) | (1022 << 52); + (f64::from_bits(mantissa_bits), exponent) + } +} + pub type PyFloatRef = PyRef; // Retrieve inner float value: diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs index f87337359..3d2d0b0d1 100644 --- a/vm/src/pyhash.rs +++ b/vm/src/pyhash.rs @@ -1,5 +1,6 @@ use std::hash::{Hash, Hasher}; +use crate::obj::objfloat; use crate::pyobject::PyObjectRef; use crate::pyobject::PyResult; use crate::vm::VirtualMachine; @@ -28,14 +29,7 @@ pub fn hash_float(value: f64) -> PyHash { }; } - let frexp = if 0.0 == value { - (value, 0i32) - } else { - let bits = value.to_bits(); - let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; - let mantissa_bits = bits & (0x000fffffffffffff) | (1022 << 52); - (f64::from_bits(mantissa_bits), exponent) - }; + let frexp = objfloat::ufrexp(value); // process 28 bits at a time; this should work well both for binary // and hexadecimal floating point. diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 58b3ee953..c8ee63247 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -209,6 +209,23 @@ fn math_floor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn math_frexp(value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + objfloat::try_float(&value, vm)?.map_or_else( + || Err(vm.new_type_error(format!("must be real number, not {}", value.class()))), + |value| { + let (m, e) = if value.is_finite() { + let (m, e) = objfloat::ufrexp(value); + (m * value.signum(), e) + } else { + (value, 0) + }; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(m), vm.ctx.new_int(e)])) + }, + ) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -256,6 +273,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "gamma" => ctx.new_rustfunc(math_gamma), "lgamma" => ctx.new_rustfunc(math_lgamma), + "frexp" => ctx.new_rustfunc(math_frexp), + // Rounding functions: "trunc" => ctx.new_rustfunc(math_trunc), "ceil" => ctx.new_rustfunc(math_ceil), From ceca2ed98e9755a751fd1a1566f85c935a28128b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 06:17:50 +0900 Subject: [PATCH 581/658] Add float.fromhex --- Cargo.lock | 76 ++++++++++++++++++++++++++++++++++++++++ tests/snippets/floats.py | 6 ++++ vm/Cargo.toml | 1 + vm/src/obj/objfloat.rs | 12 +++++++ 4 files changed, 95 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 0303e06e9..1194fd899 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,6 +323,31 @@ dependencies = [ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hexf" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hexf-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hexf-parse" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "humantime" version = "1.2.0" @@ -553,6 +578,19 @@ name = "precomputed-hash" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro-hack" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.4.27" @@ -575,6 +613,11 @@ name = "quick-error" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "quote" version = "0.6.11" @@ -829,6 +872,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1014,6 +1058,16 @@ name = "strsim" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "0.15.29" @@ -1024,6 +1078,14 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synstructure" version = "0.10.1" @@ -1147,6 +1209,11 @@ name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -1372,6 +1439,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e54653cc32d838771a36532647afad59c4bf7155745eeeec406f71fd5d7e7538" +"checksum hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "22eadcfadba76a730b2764eaa577d045f35e0ef5174b9c5b46adf1ee42b85e12" +"checksum hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79296f72d53a89096cbc9a88c9547ee8dfe793388674620e2207593d370550ac" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" @@ -1401,9 +1471,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" "checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +"checksum proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b773f824ff2a495833f85fcdddcf85e096949971decada2e93249fa2c6c3d32f" +"checksum proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f674ccc446da486175527473ec8aa064f980b0966bbf767ee743a5dff6244a7" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" "checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" "checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -1446,7 +1519,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eea1eee654ef80933142157fdad9dd8bc43cf7c74e999e369263496f04ff4da" "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" @@ -1464,6 +1539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index c116b8010..3d19ac2de 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -196,3 +196,9 @@ assert str(1.123456789) == '1.123456789' a = .5 assert a == 0.5 +assert float.fromhex('0x0.0p+0') == 0.0 +assert float.fromhex('-0x0.0p+0') == -0.0 +assert float.fromhex('0x1.000000p+0') == 1.0 +assert float.fromhex('-0x1.800000p+0') == -1.5 +assert float.fromhex('inf') == float('inf') +assert math.isnan(float.fromhex('nan')) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 8ac5ebabb..522bcc50f 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -28,6 +28,7 @@ unicode-xid = "0.1.0" lazy_static = "^1.0.1" lexical = "2.0.0" itertools = "^0.8.0" +hexf = "0.1.0" # TODO: release and publish to crates.io [dependencies.unicode-casing] diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 8d983c3ae..bc8c014d9 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -3,6 +3,7 @@ use super::objint; use super::objstr; use super::objtype; use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; use crate::pyhash; use crate::pyobject::{ @@ -10,6 +11,7 @@ use crate::pyobject::{ TypeProtocol, }; use crate::vm::VirtualMachine; +use hexf; use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; use num_traits::{ToPrimitive, Zero}; @@ -473,6 +475,16 @@ impl PyFloat { let denom = vm.ctx.new_int(ratio.denom().clone()); Ok(vm.ctx.new_tuple(vec![numer, denom])) } + + #[pymethod] + fn fromhex(repr: PyStringRef, vm: &VirtualMachine) -> PyResult { + hexf::parse_hexf64(&repr.value, false).or_else(|_| match repr.value.as_ref() { + "nan" => Ok(std::f64::NAN), + "inf" => Ok(std::f64::INFINITY), + "-inf" => Ok(std::f64::NEG_INFINITY), + _ => Err(vm.new_value_error("invalid hexadecimal floating-point string".to_string())), + }) + } } pub type PyFloatRef = PyRef; From ca912a168d692a0ded187cb7e2aad080a9e0f936 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 17 May 2019 02:35:59 +0900 Subject: [PATCH 582/658] Add float.to_hex --- tests/snippets/floats.py | 12 +++++++++++ vm/src/obj/objfloat.rs | 45 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 3d19ac2de..101d58cc9 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -202,3 +202,15 @@ assert float.fromhex('0x1.000000p+0') == 1.0 assert float.fromhex('-0x1.800000p+0') == -1.5 assert float.fromhex('inf') == float('inf') assert math.isnan(float.fromhex('nan')) + +assert (0.0).hex() == '0x0.0p+0' +assert (-0.0).hex() == '-0x0.0p+0' +assert (1.0).hex() == '0x1.0000000000000p+0' +assert (-1.5).hex() == '-0x1.8000000000000p+0' +assert float('inf').hex() == 'inf' +assert float('-inf').hex() == '-inf' +assert float('nan').hex() == 'nan' + +#for _ in range(10000): +# f = random.random() * random.randint(0, 0x10000000000000000) +# assert f == float.fromhex(f.hex()) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index bc8c014d9..d240961bb 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -14,7 +14,7 @@ use crate::vm::VirtualMachine; use hexf; use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; -use num_traits::{ToPrimitive, Zero}; +use num_traits::{float::Float, ToPrimitive, Zero}; /// Convert a string or number to a floating point number, if possible. #[pyclass(name = "float")] @@ -485,6 +485,49 @@ impl PyFloat { _ => Err(vm.new_value_error("invalid hexadecimal floating-point string".to_string())), }) } + + #[pymethod] + fn hex(&self, _vm: &VirtualMachine) -> String { + to_hex(self.value) + } +} + +fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt), + value if value.is_infinite() => format!("{}inf", sign_fmt), + value if value.is_nan() => "nan".to_string(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}0x{:x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::thread_rng().gen::<[u64; 1]>(); + let f = f64::from_bits(bytes[0]); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } } pub type PyFloatRef = PyRef; From b76da2ad0533bd59bd0a4666ce8e8ec4234b0471 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Fri, 17 May 2019 18:58:59 +0900 Subject: [PATCH 583/658] Avoid timeout in WASM test --- wasm/tests/.travis-runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/tests/.travis-runner.sh b/wasm/tests/.travis-runner.sh index add96c6b2..40e1dee50 100755 --- a/wasm/tests/.travis-runner.sh +++ b/wasm/tests/.travis-runner.sh @@ -20,4 +20,4 @@ export PATH=$PATH:$PWD/geckodriver pip install pipenv (cd wasm/tests; pipenv install) -(cd wasm/demo; npm install; npm run ci) +(cd wasm/demo; npm install; npm run build; npm run ci) From cb0367e708d75dab214723fcb60e8e46c810bd83 Mon Sep 17 00:00:00 2001 From: rbrtberglund Date: Sat, 18 May 2019 16:14:43 +0200 Subject: [PATCH 584/658] Add exit/quit builtin function --- vm/src/builtins.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 3250bef80..f87ea8e7e 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -613,6 +613,24 @@ impl Printer for std::io::StdoutLock<'_> { } } +pub fn builtin_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [], optional = [(object, None)]); + match object { + Some(value) => match i32::try_from_object(&vm, value.clone()) { + Ok(code) => std::process::exit(code), + _ => { + let stdout = io::stdout(); + let mut printer: Box = Box::new(stdout.lock()); + printer.write(vm, value.clone())?; + printer.write(vm, "\n".into_pyobject(vm).unwrap())?; + printer.flush(vm)?; + } + }, + _ => {} + } + std::process::exit(0); +} + pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { let stdout = io::stdout(); @@ -823,6 +841,8 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "tuple" => ctx.tuple_type(), "type" => ctx.type_type(), "zip" => ctx.zip_type(), + "exit" => ctx.new_rustfunc(builtin_exit), + "quit" => ctx.new_rustfunc(builtin_exit), "__import__" => ctx.new_rustfunc(builtin_import), // Constants From 7f310bd81f4e6a8f951cd269e2602fce965b9d4f Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 18 May 2019 23:53:40 +0300 Subject: [PATCH 585/658] Add `itertools.starmap` --- tests/snippets/stdlib_itertools.py | 10 +++++++ vm/src/stdlib/itertools.rs | 48 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index 92fbe4a29..410f39732 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -84,6 +84,16 @@ with assertRaises(StopIteration): next(r) +# itertools.starmap tests +starmap = itertools.starmap + +assert list(starmap(pow, zip(range(3), range(1,7)))) == [0**1, 1**2, 2**3] +assert list(starmap(pow, [])) == [] +assert list(starmap(pow, [iter([4,5])])) == [4**5] +with assertRaises(TypeError): + starmap(pow) + + # itertools.takewhile tests from itertools import takewhile as tw diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index e97c53bfa..4e6addd2e 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -122,6 +122,51 @@ impl PyItertoolsRepeat { } } +#[pyclass(name = "starmap")] +#[derive(Debug)] +struct PyItertoolsStarmap { + function: PyObjectRef, + iter: PyObjectRef, +} + +impl PyValue for PyItertoolsStarmap { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "starmap") + } +} + +#[pyimpl] +impl PyItertoolsStarmap { + #[pymethod(name = "__new__")] + fn new( + _cls: PyClassRef, + function: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsStarmap { + function: function, + iter: iter, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let obj = call_next(vm, &self.iter)?; + + vm.invoke(self.function.clone(), vm.extract_elements(&obj)?) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + #[pyclass] #[derive(Debug)] struct PyItertoolsTakewhile { @@ -190,12 +235,15 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let repeat = ctx.new_class("repeat", ctx.object()); PyItertoolsRepeat::extend_class(ctx, &repeat); + let starmap = PyItertoolsStarmap::make_class(ctx); + let takewhile = ctx.new_class("takewhile", ctx.object()); PyItertoolsTakewhile::extend_class(ctx, &takewhile); py_module!(vm, "itertools", { "count" => count, "repeat" => repeat, + "starmap" => starmap, "takewhile" => takewhile, }) } From 2ca96c8ae1282a6a5a5a36af57d0a280f366e3f9 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 19 May 2019 10:12:33 +1200 Subject: [PATCH 586/658] Add sys.flags by creating a pystruct_sequence macro --- derive/src/lib.rs | 9 +++ derive/src/pyclass.rs | 127 +++++++++++++++++++++++++++++++++++---- tests/snippets/sysmod.py | 5 ++ vm/src/obj/objtuple.rs | 6 ++ vm/src/sysmodule.rs | 29 ++++++++- 5 files changed, 162 insertions(+), 14 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 72ab44d4c..c88ca6769 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "128"] + extern crate proc_macro; #[macro_use] @@ -38,3 +40,10 @@ pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { let item = parse_macro_input!(item as Item); result_to_tokens(pyclass::impl_pyimpl(attr, item)) } + +#[proc_macro_attribute] +pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + result_to_tokens(pyclass::impl_pystruct_sequence(attr, item)) +} diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index a2cf57b4a..021ee9292 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -2,7 +2,9 @@ use super::Diagnostic; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use std::collections::HashMap; -use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; +use syn::{ + Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, Meta, MethodSig, NestedMeta, +}; enum ClassItem { Method { @@ -266,16 +268,12 @@ pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result Result { - let (item, ident, attrs) = match item { - Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), - Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), - other => bail_span!( - other, - "#[pyclass] can only be on a struct or enum declaration" - ), - }; - +fn generate_class_def( + ident: &Ident, + attr_name: &'static str, + attr: AttributeArgs, + attrs: &Vec, +) -> Result { let mut class_name = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { @@ -284,7 +282,11 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result = #doc; @@ -324,3 +325,103 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result { + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), + other => bail_span!( + other, + "#[pyclass] can only be on a struct or enum declaration" + ), + }; + + let class_def = generate_class_def(&ident, "pyclass", attr, &attrs)?; + + let ret = quote! { + #item + #class_def + }; + Ok(ret) +} + +pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result { + if let Item::Struct(struc) = item { + let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?; + let mut methods = Vec::new(); + let mut method_references = Vec::new(); + let mut field_names = Vec::new(); + for (i, field) in struc.fields.iter().enumerate() { + let idx = Index::from(i); + if let Some(ref field_name) = field.ident { + let method_name = format!("get_{}", field_name); + let method_ident = Ident::new(&method_name, field_name.span()); + + let method = quote! { + fn #method_ident( + zelf: &::rustpython_vm::obj::objtuple::PyTuple, + _vm: &::rustpython_vm::vm::VirtualMachine) + -> ::rustpython_vm::pyobject::PyObjectRef { + zelf.fast_getitem(#idx) + } + }; + methods.push(method); + let field_name_str = field_name.to_string(); + let method_reference = quote! { + class.set_str_attr( + #field_name_str, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(Self::#method_ident) + .create(), + ); + }; + method_references.push(method_reference); + field_names.push(quote!(#field_name)); + } else { + field_names.push(quote!(#idx)); + } + } + + let ty = &struc.ident; + let ret = quote! { + #struc + #class_def + impl #ty { + fn into_struct_sequence(&self, + vm: &::rustpython_vm::vm::VirtualMachine, + cls: ::rustpython_vm::obj::objtype::PyClassRef, + ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { + let tuple: ::rustpython_vm::obj::objtuple::PyTuple = + vec![#(::rustpython_vm::pyobject::IntoPyObject + ::into_pyobject(self.#field_names, vm)? + ),*].into(); + ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) + } + + #(#methods)* + } + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#method_references)* + } + + fn make_class( + ctx: &::rustpython_vm::pyobject::PyContext + ) -> ::rustpython_vm::obj::objtype::PyClassRef { + let py_class = ctx.new_class(::NAME, ctx.tuple_type()); + Self::extend_class(ctx, &py_class); + py_class + } + } + }; + Ok(ret) + } else { + bail_span!( + item, + "#[pystruct_sequence] can only be on a struct declaration" + ) + } +} diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index 423ecf08d..7de155e17 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -7,3 +7,8 @@ assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "w assert isinstance(sys.builtin_module_names, tuple) assert 'sys' in sys.builtin_module_names + +assert isinstance(sys.flags, tuple) +assert type(sys.flags).__name__ == "flags" +assert type(sys.flags.optimize) is int +assert sys.flags[3] == sys.flags.optimize diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 795e17d62..e4b828f3c 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -40,6 +40,12 @@ impl PyValue for PyTuple { } } +impl PyTuple { + pub fn fast_getitem(&self, idx: usize) -> PyObjectRef { + self.elements.borrow()[idx].clone() + } +} + pub type PyTupleRef = PyRef; impl PyTupleRef { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 8c6bafe74..6400c77f3 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -4,7 +4,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{IntoPyObject, ItemProtocol, PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* @@ -27,6 +27,26 @@ fn getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult PyResult { arg_check!(vm, args, required = [(object, None)]); let size = Rc::strong_count(&object); @@ -48,6 +68,12 @@ fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; + let flags_type = SysFlags::make_class(ctx); + // TODO parse command line arguments and environment variables to populate SysFlags + let flags = SysFlags::default() + .into_struct_sequence(vm, flags_type) + .unwrap(); + let path_list = match env::var_os("PYTHONPATH") { Some(paths) => env::split_paths(&paths) .map(|path| { @@ -152,6 +178,7 @@ settrace() -- set the global debug tracing function extend_module!(vm, module, { "argv" => argv(ctx), "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), + "flags" => flags, "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), "intern" => ctx.new_rustfunc(sys_intern), From 8b657ab81e0dbefc5a87a26669d1842d9b5908f9 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 19 May 2019 10:32:07 +1200 Subject: [PATCH 587/658] Add documentation for sys.flags --- derive/src/pyclass.rs | 1 + vm/src/sysmodule.rs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 021ee9292..21de60d1a 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -367,6 +367,7 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result, vm: &VirtualMachine) -> PyResult Date: Sun, 19 May 2019 00:42:55 +0200 Subject: [PATCH 588/658] applied suggestions to improve exit function implementation --- vm/src/builtins.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index f87ea8e7e..a1b52a17b 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -613,18 +613,11 @@ impl Printer for std::io::StdoutLock<'_> { } } -pub fn builtin_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [], optional = [(object, None)]); +pub fn builtin_exit(object: OptionalArg, vm: &VirtualMachine) -> PyResult<()> { match object { - Some(value) => match i32::try_from_object(&vm, value.clone()) { + OptionalArg::Present(object) => match i32::try_from_object(&vm, object.clone()) { Ok(code) => std::process::exit(code), - _ => { - let stdout = io::stdout(); - let mut printer: Box = Box::new(stdout.lock()); - printer.write(vm, value.clone())?; - printer.write(vm, "\n".into_pyobject(vm).unwrap())?; - printer.flush(vm)?; - } + _ => println!("{}", vm.to_str(&object)?.as_str()), }, _ => {} } From b9814ce3c71aae23c2605cae75a47070a5cda79f Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Sun, 19 May 2019 12:00:36 +0900 Subject: [PATCH 589/658] Update rustyline to 4.1.0 to fix nightly build --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1194fd899..f81dfa142 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,7 +473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -838,7 +838,7 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", - "rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -913,14 +913,14 @@ dependencies = [ [[package]] name = "rustyline" -version = "2.1.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1457,7 +1457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" "checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" "checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30" -"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" +"checksum nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46f0f3210768d796e8fa79ec70ee6af172dacbe7147f5e69be5240a47778302b" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" "checksum num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "57450397855d951f1a41305e54851b1a7b8f5d2e349543a02a2effe25459f718" @@ -1500,7 +1500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rustc-demangle 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "adacaae16d02b6ec37fdc7acfcddf365978de76d1983d3ee22afc260e1ca9619" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6de8ecd7fad7731f306f69b6e10ec5a3178c61e464dcc06979427aa4cc891145" -"checksum rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6010155119d53aac4f5b987cb8f6ea913d0d64d9b237da36f8f96a90cb3f5385" +"checksum rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b" "checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" diff --git a/Cargo.toml b/Cargo.toml index ef2838a1c..71db1d839 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ env_logger="0.5.10" clap = "2.31.2" rustpython_parser = {path = "parser"} rustpython_vm = {path = "vm"} -rustyline = "2.1.0" +rustyline = "4.1.0" xdg = "2.2.0" [dev-dependencies.cpython] From 60c9d5004ab128c243e11db5c4203350e739760e Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 19 May 2019 15:01:22 +1200 Subject: [PATCH 590/658] Add SimpleNamespace and sys.implementation --- Lib/types.py | 2 +- tests/snippets/stdlib_types.py | 10 ++++++++++ tests/snippets/sysmod.py | 3 +++ vm/src/macros.rs | 13 +++++++++++++ vm/src/obj/mod.rs | 1 + vm/src/obj/objnamespace.rs | 32 ++++++++++++++++++++++++++++++++ vm/src/pyobject.rs | 13 +++++++++++++ vm/src/sysmodule.rs | 7 +++++++ 8 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/snippets/stdlib_types.py create mode 100644 vm/src/obj/objnamespace.rs diff --git a/Lib/types.py b/Lib/types.py index 4ef461961..a7034901c 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -13,7 +13,7 @@ FunctionType = type(_f) LambdaType = type(lambda: None) # Same as FunctionType CodeType = type(_f.__code__) MappingProxyType = type(type.__dict__) -# SimpleNamespace = type(sys.implementation) +SimpleNamespace = type(sys.implementation) def _g(): yield 1 diff --git a/tests/snippets/stdlib_types.py b/tests/snippets/stdlib_types.py new file mode 100644 index 000000000..b8011bf40 --- /dev/null +++ b/tests/snippets/stdlib_types.py @@ -0,0 +1,10 @@ +import types + +from testutils import assertRaises + +ns = types.SimpleNamespace(a=2, b='Rust') + +assert ns.a == 2 +assert ns.b == "Rust" +with assertRaises(AttributeError): + _ = ns.c diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index 423ecf08d..409d961ed 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -7,3 +7,6 @@ assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "w assert isinstance(sys.builtin_module_names, tuple) assert 'sys' in sys.builtin_module_names + +assert isinstance(sys.implementation.name, str) +assert isinstance(sys.implementation.cache_tag, str) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 333673985..c3267ff15 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -157,6 +157,19 @@ macro_rules! extend_class { } } +#[macro_export] +macro_rules! py_namespace { + ( $vm:expr, { $($name:expr => $value:expr),* $(,)* }) => { + { + let namespace = $vm.ctx.new_namespace(); + $( + $vm.set_attr(&namespace, $name, $value).unwrap(); + )* + namespace + } + } +} + /// Macro to match on the built-in class of a Python object. /// /// Like `match`, `match_class!` must be exhaustive, so a default arm with diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index c9401d8c4..ad7af74d5 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -23,6 +23,7 @@ pub mod objmap; pub mod objmappingproxy; pub mod objmemory; pub mod objmodule; +pub mod objnamespace; pub mod objnone; pub mod objobject; pub mod objproperty; diff --git a/vm/src/obj/objnamespace.rs b/vm/src/obj/objnamespace.rs new file mode 100644 index 000000000..e79b561c0 --- /dev/null +++ b/vm/src/obj/objnamespace.rs @@ -0,0 +1,32 @@ +use crate::function::KwArgs; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +/// A simple attribute-based namespace. +/// +/// SimpleNamespace(**kwargs) +#[pyclass(name = "SimpleNamespace")] +#[derive(Debug)] +pub struct PyNamespace; + +impl PyValue for PyNamespace { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.namespace_type() + } +} + +#[pyimpl] +impl PyNamespace { + #[pymethod(name = "__init__")] + fn init(zelf: PyRef, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult<()> { + for (name, value) in kwargs.into_iter() { + vm.set_attr(zelf.as_object(), name, value)?; + } + Ok(()) + } +} + +pub fn init(context: &PyContext) { + PyNamespace::extend_class(context, &context.namespace_type); +} diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 722588b3f..afaad09b6 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -40,6 +40,7 @@ use crate::obj::objmap; use crate::obj::objmappingproxy; use crate::obj::objmemory; use crate::obj::objmodule::{self, PyModule}; +use crate::obj::objnamespace::{self, PyNamespace}; use crate::obj::objnone::{self, PyNone, PyNoneRef}; use crate::obj::objobject; use crate::obj::objproperty; @@ -158,6 +159,7 @@ pub struct PyContext { pub property_type: PyClassRef, pub readonly_property_type: PyClassRef, pub module_type: PyClassRef, + pub namespace_type: PyClassRef, pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, pub weakproxy_type: PyClassRef, @@ -250,6 +252,7 @@ impl PyContext { let dict_type = create_type("dict", &type_type, &object_type); let module_type = create_type("module", &type_type, &object_type); + let namespace_type = create_type("SimpleNamespace", &type_type, &object_type); let classmethod_type = create_type("classmethod", &type_type, &object_type); let staticmethod_type = create_type("staticmethod", &type_type, &object_type); let function_type = create_type("function", &type_type, &object_type); @@ -366,6 +369,7 @@ impl PyContext { readonly_property_type, generator_type, module_type, + namespace_type, bound_method_type, weakref_type, weakproxy_type, @@ -407,6 +411,7 @@ impl PyContext { objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); + objnamespace::init(&context); objmappingproxy::init(&context); exceptions::init(&context); context @@ -464,6 +469,10 @@ impl PyContext { self.module_type.clone() } + pub fn namespace_type(&self) -> PyClassRef { + self.namespace_type.clone() + } + pub fn set_type(&self) -> PyClassRef { self.set_type.clone() } @@ -658,6 +667,10 @@ impl PyContext { ) } + pub fn new_namespace(&self) -> PyObjectRef { + PyObject::new(PyNamespace, self.namespace_type(), Some(self.new_dict())) + } + pub fn new_rustfunc(&self, f: F) -> PyObjectRef where F: IntoPyNativeFunc, diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 8c6bafe74..aefdb117b 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -48,6 +48,12 @@ fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; + // TODO Add crate version to this namespace + let implementation = py_namespace!(vm, { + "name" => ctx.new_str("RustPython".to_string()), + "cache_tag" => ctx.new_str("rustpython-01".to_string()), + }); + let path_list = match env::var_os("PYTHONPATH") { Some(paths) => env::split_paths(&paths) .map(|path| { @@ -154,6 +160,7 @@ settrace() -- set the global debug tracing function "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "implementation" => implementation, "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, From 52039845dd59019372c3ab4952df7950eef84ae2 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 19 May 2019 15:23:51 +1200 Subject: [PATCH 591/658] Use closure for generated properties in struct sequences. --- derive/src/pyclass.rs | 134 +++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 73 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 21de60d1a..d216b9357 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -346,83 +346,71 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result { - if let Item::Struct(struc) = item { - let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?; - let mut methods = Vec::new(); - let mut method_references = Vec::new(); - let mut field_names = Vec::new(); - for (i, field) in struc.fields.iter().enumerate() { - let idx = Index::from(i); - if let Some(ref field_name) = field.ident { - let method_name = format!("get_{}", field_name); - let method_ident = Ident::new(&method_name, field_name.span()); - - let method = quote! { - fn #method_ident( - zelf: &::rustpython_vm::obj::objtuple::PyTuple, - _vm: &::rustpython_vm::vm::VirtualMachine) - -> ::rustpython_vm::pyobject::PyObjectRef { - zelf.fast_getitem(#idx) - } - }; - methods.push(method); - let field_name_str = field_name.to_string(); - // TODO add doc to the generated property - let method_reference = quote! { - class.set_str_attr( - #field_name_str, - ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) - .add_getter(Self::#method_ident) - .create(), - ); - }; - method_references.push(method_reference); - field_names.push(quote!(#field_name)); - } else { - field_names.push(quote!(#idx)); - } - } - - let ty = &struc.ident; - let ret = quote! { - #struc - #class_def - impl #ty { - fn into_struct_sequence(&self, - vm: &::rustpython_vm::vm::VirtualMachine, - cls: ::rustpython_vm::obj::objtype::PyClassRef, - ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { - let tuple: ::rustpython_vm::obj::objtuple::PyTuple = - vec![#(::rustpython_vm::pyobject::IntoPyObject - ::into_pyobject(self.#field_names, vm)? - ),*].into(); - ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) - } - - #(#methods)* - } - impl ::rustpython_vm::pyobject::PyClassImpl for #ty { - fn impl_extend_class( - ctx: &::rustpython_vm::pyobject::PyContext, - class: &::rustpython_vm::obj::objtype::PyClassRef, - ) { - #(#method_references)* - } - - fn make_class( - ctx: &::rustpython_vm::pyobject::PyContext - ) -> ::rustpython_vm::obj::objtype::PyClassRef { - let py_class = ctx.new_class(::NAME, ctx.tuple_type()); - Self::extend_class(ctx, &py_class); - py_class - } - } - }; - Ok(ret) + let struc = if let Item::Struct(struc) = item { + struc } else { bail_span!( item, "#[pystruct_sequence] can only be on a struct declaration" ) + }; + let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?; + let mut properties = Vec::new(); + let mut field_names = Vec::new(); + for (i, field) in struc.fields.iter().enumerate() { + let idx = Index::from(i); + if let Some(ref field_name) = field.ident { + let field_name_str = field_name.to_string(); + // TODO add doc to the generated property + let property = quote! { + class.set_str_attr( + #field_name_str, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(|zelf: &::rustpython_vm::obj::objtuple::PyTuple, + _vm: &::rustpython_vm::vm::VirtualMachine| + zelf.fast_getitem(#idx)) + .create(), + ); + }; + properties.push(property); + field_names.push(quote!(#field_name)); + } else { + field_names.push(quote!(#idx)); + } } + + let ty = &struc.ident; + let ret = quote! { + #struc + #class_def + impl #ty { + fn into_struct_sequence(&self, + vm: &::rustpython_vm::vm::VirtualMachine, + cls: ::rustpython_vm::obj::objtype::PyClassRef, + ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { + let tuple: ::rustpython_vm::obj::objtuple::PyTuple = + vec![#(::rustpython_vm::pyobject::IntoPyObject + ::into_pyobject(self.#field_names, vm)? + ),*].into(); + ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) + } + } + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#properties)* + } + + fn make_class( + ctx: &::rustpython_vm::pyobject::PyContext + ) -> ::rustpython_vm::obj::objtype::PyClassRef { + let py_class = ctx.new_class(::NAME, ctx.tuple_type()); + Self::extend_class(ctx, &py_class); + py_class + } + } + }; + Ok(ret) } From 964256439bf2c49e3795766fff81535e500f3d84 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 19 May 2019 15:25:52 +1200 Subject: [PATCH 592/658] Update vm/src/obj/objnamespace.rs Co-Authored-By: coolreader18 <33094578+coolreader18@users.noreply.github.com> --- vm/src/obj/objnamespace.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objnamespace.rs b/vm/src/obj/objnamespace.rs index e79b561c0..f8bfc9369 100644 --- a/vm/src/obj/objnamespace.rs +++ b/vm/src/obj/objnamespace.rs @@ -19,7 +19,7 @@ impl PyValue for PyNamespace { #[pyimpl] impl PyNamespace { #[pymethod(name = "__init__")] - fn init(zelf: PyRef, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult<()> { + fn init(zelf: PyRef, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult<()> { for (name, value) in kwargs.into_iter() { vm.set_attr(zelf.as_object(), name, value)?; } From 36412c57a4ad17d17011d634f2b6f745533280e3 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Wed, 15 May 2019 00:29:23 +0900 Subject: [PATCH 593/658] Implement pwd module --- Cargo.lock | 11 ++++++ vm/Cargo.toml | 1 + vm/src/stdlib/mod.rs | 2 ++ vm/src/stdlib/pwd.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 vm/src/stdlib/pwd.rs diff --git a/Cargo.lock b/Cargo.lock index f81dfa142..66b1f01af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,15 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pwd" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "python3-sys" version = "0.2.1" @@ -882,6 +891,7 @@ dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1474,6 +1484,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b773f824ff2a495833f85fcdddcf85e096949971decada2e93249fa2c6c3d32f" "checksum proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f674ccc446da486175527473ec8aa064f980b0966bbf767ee743a5dff6244a7" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" +"checksum pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5dd32d8bece608e144ca20251e714ed107cdecdabb20c2d383cfc687825106a5" "checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 522bcc50f..cdd7bbb30 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -19,6 +19,7 @@ serde = "1.0.66" serde_derive = "1.0.66" serde_json = "1.0.26" byteorder = "1.2.6" +pwd = "1" regex = "1" rustc_version_runtime = "0.1.*" statrs = "0.10.0" diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 149f37d9e..9abc8b26c 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod json; mod keyword; mod math; mod platform; +mod pwd; mod pystruct; mod random; mod re; @@ -41,6 +42,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("keyword".to_string(), Box::new(keyword::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); modules.insert("platform".to_string(), Box::new(platform::make_module)); + modules.insert("pwd".to_string(), Box::new(pwd::make_module)); modules.insert("re".to_string(), Box::new(re::make_module)); modules.insert("random".to_string(), Box::new(random::make_module)); modules.insert("string".to_string(), Box::new(string::make_module)); diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs new file mode 100644 index 000000000..1f5092540 --- /dev/null +++ b/vm/src/stdlib/pwd.rs @@ -0,0 +1,85 @@ +use pwd::Passwd; + +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +impl PyValue for Passwd { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("pwd", "struct_passwd") + } +} + +type PasswdRef = PyRef; + +impl PasswdRef { + fn pw_name(self, _vm: &VirtualMachine) -> String { + self.name.clone() + } + + fn pw_passwd(self, _vm: &VirtualMachine) -> Option { + self.passwd.clone() + } + + fn pw_uid(self, _vm: &VirtualMachine) -> u32 { + self.uid + } + + fn pw_gid(self, _vm: &VirtualMachine) -> u32 { + self.gid + } + + fn pw_gecos(self, _vm: &VirtualMachine) -> Option { + self.gecos.clone() + } + + fn pw_dir(self, _vm: &VirtualMachine) -> String { + self.dir.clone() + } + + fn pw_shell(self, _vm: &VirtualMachine) -> String { + self.shell.clone() + } +} + +fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + match Passwd::from_name(&name.value) { + Ok(Some(passwd)) => Ok(passwd), + _ => { + let name_repr = vm.to_repr(name.as_object())?; + let message = format!("getpwnam(): name not found: {}", name_repr); + Err(vm.new_key_error(message)) + } + } +} + +fn pwd_getpwuid(uid: u32, vm: &VirtualMachine) -> PyResult { + match Passwd::from_uid(uid) { + Some(passwd) => Ok(passwd), + _ => { + let message = format!("getpwuid(): uid not found: {}", uid); + Err(vm.new_key_error(message)) + } + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let passwd_type = py_class!(ctx, "struct_passwd", ctx.object(), { + "pw_name" => ctx.new_property(PasswdRef::pw_name), + "pw_passwd" => ctx.new_property(PasswdRef::pw_passwd), + "pw_uid" => ctx.new_property(PasswdRef::pw_uid), + "pw_gid" => ctx.new_property(PasswdRef::pw_gid), + "pw_gecos" => ctx.new_property(PasswdRef::pw_gecos), + "pw_dir" => ctx.new_property(PasswdRef::pw_dir), + "pw_shell" => ctx.new_property(PasswdRef::pw_shell), + }); + + py_module!(vm, "pwd", { + "struct_passwd" => passwd_type, + "getpwnam" => ctx.new_rustfunc(pwd_getpwnam), + "getpwuid" => ctx.new_rustfunc(pwd_getpwuid), + }) +} From 347d1fd2405c2f8f7f7de04d6c7bed6d3685d850 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Sun, 19 May 2019 13:16:37 +0900 Subject: [PATCH 594/658] Enable pwd module only on Unix --- vm/Cargo.toml | 4 +++- vm/src/stdlib/mod.rs | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index cdd7bbb30..0d80923fd 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -19,7 +19,6 @@ serde = "1.0.66" serde_derive = "1.0.66" serde_json = "1.0.26" byteorder = "1.2.6" -pwd = "1" regex = "1" rustc_version_runtime = "0.1.*" statrs = "0.10.0" @@ -35,3 +34,6 @@ hexf = "0.1.0" [dependencies.unicode-casing] git = "https://github.com/OddCoincidence/unicode-casing" rev = "90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" + +[target.'cfg(unix)'.dependencies] +pwd = "1" diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 9abc8b26c..2b17586c6 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -6,7 +6,6 @@ pub(crate) mod json; mod keyword; mod math; mod platform; -mod pwd; mod pystruct; mod random; mod re; @@ -24,6 +23,8 @@ use crate::vm::VirtualMachine; pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; +#[cfg(unix)] +mod pwd; use crate::pyobject::PyObjectRef; @@ -42,7 +43,6 @@ pub fn get_module_inits() -> HashMap { modules.insert("keyword".to_string(), Box::new(keyword::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); modules.insert("platform".to_string(), Box::new(platform::make_module)); - modules.insert("pwd".to_string(), Box::new(pwd::make_module)); modules.insert("re".to_string(), Box::new(re::make_module)); modules.insert("random".to_string(), Box::new(random::make_module)); modules.insert("string".to_string(), Box::new(string::make_module)); @@ -60,5 +60,11 @@ pub fn get_module_inits() -> HashMap { modules.insert("socket".to_string(), Box::new(socket::make_module)); } + // Unix-only + #[cfg(unix)] + { + modules.insert("pwd".to_string(), Box::new(pwd::make_module)); + } + modules } From 1e7fa52e0ffce47db5a463e1704e91d5725f120f Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Tue, 14 May 2019 19:56:54 +0200 Subject: [PATCH 595/658] Fix no-arg dir(), convert it to new args style --- tests/snippets/builtin_dir.py | 12 ++++++++++++ vm/src/builtins.rs | 17 ++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index b5fbf8d0b..3e808597c 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -1,3 +1,5 @@ +assert isinstance(dir(), list) +assert '__builtins__' in dir() class A: def test(): @@ -21,6 +23,16 @@ assert ['h', 'q'] == dir(B()) # This calls type.__dir__ so isn't changed (but inheritance works)! assert 'test' in dir(A) +# eval() takes any mapping-like type, so dir() must support them +# TODO: eval() should take any mapping as locals, not just dict-derived types +class A(dict): + def __getitem__(self, x): + return dir + def keys(self): + yield 6 + yield 5 +assert eval("dir()", {}, A()) == [5, 6] + import socket assert "AF_INET" in dir(socket) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 3250bef80..7206fe579 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -111,15 +111,13 @@ fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> vm.del_attr(&obj, attr.into_object()) } -fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - Ok(vm.get_locals().into_object()) - } else { - let obj = args.args.into_iter().next().unwrap(); - let seq = vm.call_method(&obj, "__dir__", vec![])?; - let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; - Ok(sorted) - } +fn builtin_dir(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { + let seq = match obj { + OptionalArg::Present(obj) => vm.call_method(&obj, "__dir__", vec![])?, + OptionalArg::Missing => vm.call_method(&vm.get_locals().into_object(), "keys", vec![])?, + }; + let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; + Ok(sorted) } fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -136,6 +134,7 @@ fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { /// Implements `eval`. /// See also: https://docs.python.org/3/library/functions.html#eval fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: support any mapping for `locals` arg_check!( vm, args, From 8f8194ae1391089fe66722a186d66708b398aa65 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 19 May 2019 14:05:09 -0500 Subject: [PATCH 596/658] Fix os.rs --- vm/src/stdlib/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 45e862049..f5a6ced96 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -14,7 +14,7 @@ use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objiter; use crate::obj::objset::PySet; use crate::obj::objstr::{self, PyString, PyStringRef}; -use crate::obj::objtype::PyClassRef; +use crate::obj::objtype::{self, PyClassRef}; use crate::pyobject::{ ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, TypeProtocol, }; From 9bf386ea876227f434495a470fc70d4ef3d5a99d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 18 May 2019 23:31:47 +0300 Subject: [PATCH 597/658] Add `itertools.chain` --- tests/snippets/stdlib_itertools.py | 18 +++++++++ vm/src/stdlib/itertools.rs | 63 +++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index 410f39732..c8921cb91 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -3,6 +3,24 @@ import itertools from testutils import assertRaises +# itertools.chain tests +chain = itertools.chain + +# empty +assert list(chain()) == [] +assert list(chain([], "", b"", ())) == [] + +assert list(chain([1, 2, 3, 4])) == [1, 2, 3, 4] +assert list(chain("ab", "cd", (), 'e')) == ['a', 'b', 'c', 'd', 'e'] +with assertRaises(TypeError): + list(chain(1)) + +x = chain("ab", 1) +assert next(x) == 'a' +assert next(x) == 'b' +with assertRaises(TypeError): + next(x) + # itertools.count tests # default arguments diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 4e6addd2e..7380a7677 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -4,14 +4,72 @@ use std::ops::{AddAssign, SubAssign}; use num_bigint::BigInt; -use crate::function::OptionalArg; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objint::{PyInt, PyIntRef}; use crate::obj::objiter::{call_next, get_iter, new_stop_iteration}; +use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass(name = "chain")] +#[derive(Debug)] +struct PyItertoolsChain { + iterables: Vec, + cur: RefCell<(usize, Option)>, +} + +impl PyValue for PyItertoolsChain { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "chain") + } +} + +#[pyimpl] +impl PyItertoolsChain { + #[pymethod(name = "__new__")] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyItertoolsChain { + iterables: args.args, + cur: RefCell::new((0, None)), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let (ref mut cur_idx, ref mut cur_iter) = *self.cur.borrow_mut(); + while *cur_idx < self.iterables.len() { + if cur_iter.is_none() { + *cur_iter = Some(get_iter(vm, &self.iterables[*cur_idx])?); + } + + // can't be directly inside the 'match' clause, otherwise the borrows collide. + let obj = call_next(vm, cur_iter.as_ref().unwrap()); + match obj { + Ok(ok) => return Ok(ok), + Err(err) => { + if objtype::isinstance(&err, &vm.ctx.exceptions.stop_iteration) { + *cur_idx += 1; + *cur_iter = None; + } else { + return Err(err); + } + } + } + } + + Err(new_stop_iteration(vm)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + #[pyclass] #[derive(Debug)] struct PyItertoolsCount { @@ -229,6 +287,8 @@ impl PyItertoolsTakewhile { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let chain = PyItertoolsChain::make_class(ctx); + let count = ctx.new_class("count", ctx.object()); PyItertoolsCount::extend_class(ctx, &count); @@ -241,6 +301,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { PyItertoolsTakewhile::extend_class(ctx, &takewhile); py_module!(vm, "itertools", { + "chain" => chain, "count" => count, "repeat" => repeat, "starmap" => starmap, From 9ad527c800a7ed2c4c94a0b9eba754c4af429f37 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 19 May 2019 22:40:10 +0300 Subject: [PATCH 598/658] Remove tuple handling from `get_mut_elements` This function is unused anyway; But even was it used, it makes no sense to return mutable references to tuple items, when the tuple holding them is immutable. --- vm/src/obj/objsequence.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index af2034370..9b38ccbe8 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -368,9 +368,6 @@ pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut() { return list.elements.borrow_mut(); } - if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow_mut(); - } panic!("Cannot extract elements from non-sequence"); } From 802f07980e3585fd403b1e0ee5ad0b6a3c03968a Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 19 May 2019 22:42:05 +0300 Subject: [PATCH 599/658] Remove tuple handling from `get_elements_cell` Like my HEAD, this shouldn't be allowed for a tuple (and the only call-site of this function doesn't run on tuples anyway). --- vm/src/obj/objsequence.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 9b38ccbe8..cee66752c 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -348,9 +348,6 @@ pub fn get_elements_cell<'a>(obj: &'a PyObjectRef) -> &'a RefCell() { return &list.elements; } - if let Some(tuple) = obj.payload::() { - return &tuple.elements; - } panic!("Cannot extract elements from non-sequence"); } From 84684e5f7deeb85607f0cc803887e0fb628d0493 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 19 May 2019 22:51:05 +0300 Subject: [PATCH 600/658] Don't use `RefCell` in tuple object --- vm/src/exceptions.rs | 4 +-- vm/src/function.rs | 2 +- vm/src/import.rs | 2 +- vm/src/obj/objbyteinner.rs | 2 +- vm/src/obj/objlist.rs | 18 +++++----- vm/src/obj/objsequence.rs | 8 +++-- vm/src/obj/objtuple.rs | 68 +++++++++++++++----------------------- vm/src/stdlib/json.rs | 9 ++--- vm/src/stdlib/socket.rs | 6 ++-- vm/src/vm.rs | 12 +++---- wasm/lib/src/convert.rs | 4 +-- 11 files changed, 63 insertions(+), 72 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index a8b753e1a..e02cd2894 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -44,11 +44,11 @@ pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let mut elements = objsequence::get_elements(&tb).to_vec(); + let mut elements = objsequence::get_elements_list(&tb).to_vec(); elements.reverse(); for element in elements.iter() { if objtype::isinstance(&element, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&element); + let element = objsequence::get_elements_tuple(&element); let filename = if let Ok(x) = vm.to_str(&element[0]) { x.value.clone() } else { diff --git a/vm/src/function.rs b/vm/src/function.rs index 170c395f6..e7dbe59e4 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -538,7 +538,7 @@ pub fn single_or_tuple_any) -> PyResult>( match_class!(obj, obj @ T => predicate(obj), tuple @ PyTuple => { - for obj in tuple.elements.borrow().iter() { + for obj in tuple.elements.iter() { let inner_val = PyRef::::try_from_object(vm, obj.clone())?; if predicate(inner_val)? { return Ok(true); diff --git a/vm/src/import.rs b/vm/src/import.rs index 8445e65f9..16385e3d3 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -67,7 +67,7 @@ pub fn import_file( fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result { let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap(); - let mut paths: Vec = objsequence::get_elements(&sys_path) + let mut paths: Vec = objsequence::get_elements_list(&sys_path) .iter() .map(|item| PathBuf::from(objstr::get_value(item))) .collect(); diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4e51c4170..c07470be2 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -686,7 +686,7 @@ impl PyByteInner { Either::A(byte) => byte.elements, Either::B(tuple) => { let mut flatten = vec![]; - for v in objsequence::get_elements(tuple.as_object()).to_vec() { + for v in objsequence::get_elements_tuple(tuple.as_object()).to_vec() { flatten.extend(PyByteInner::try_from_object(vm, v)?.elements) } flatten diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 88842f9b7..77c1b4c9f 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -17,8 +17,8 @@ use super::objbool; //use super::objint; use super::objiter; use super::objsequence::{ - get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - SequenceIndex, + get_elements_cell, get_elements_list, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, + seq_mul, SequenceIndex, }; use super::objslice::PySliceRef; use super::objtype; @@ -129,7 +129,7 @@ impl PyListRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let e1 = self.elements.borrow(); - let e2 = get_elements(&other); + let e2 = get_elements_list(&other); let elements = e1.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_list(elements)) } else { @@ -141,7 +141,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { self.elements .borrow_mut() - .extend_from_slice(&get_elements(&other)); + .extend_from_slice(&get_elements_list(&other)); Ok(self.into_object()) } else { Ok(vm.ctx.not_implemented()) @@ -490,7 +490,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_equal(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -501,7 +501,7 @@ impl PyListRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_lt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -512,7 +512,7 @@ impl PyListRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_gt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -523,7 +523,7 @@ impl PyListRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_ge(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -534,7 +534,7 @@ impl PyListRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_le(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index cee66752c..d4226d51c 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -351,12 +351,16 @@ pub fn get_elements_cell<'a>(obj: &'a PyObjectRef) -> &'a RefCell(obj: &'a PyObjectRef) -> impl Deref> + 'a { +pub fn get_elements_list<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { if let Some(list) = obj.payload::() { return list.elements.borrow(); } + panic!("Cannot extract elements from non-sequence"); +} + +pub fn get_elements_tuple<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow(); + return &tuple.elements; } panic!("Cannot extract elements from non-sequence"); } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 795e17d62..d7faed04a 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -1,4 +1,4 @@ -use std::cell::{Cell, RefCell}; +use std::cell::Cell; use std::fmt; use crate::function::OptionalArg; @@ -9,14 +9,13 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objiter; use super::objsequence::{ - get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, + get_elements_tuple, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; use super::objtype::{self, PyClassRef}; pub struct PyTuple { // TODO: shouldn't be public - // TODO: tuples are immutable, remove this RefCell - pub elements: RefCell>, + pub elements: Vec, } impl fmt::Debug for PyTuple { @@ -28,9 +27,7 @@ impl fmt::Debug for PyTuple { impl From> for PyTuple { fn from(elements: Vec) -> Self { - PyTuple { - elements: RefCell::new(elements), - } + PyTuple { elements: elements } } } @@ -45,9 +42,8 @@ pub type PyTupleRef = PyRef; impl PyTupleRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_lt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_lt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -56,9 +52,8 @@ impl PyTupleRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_gt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_gt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -67,9 +62,8 @@ impl PyTupleRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_ge(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_ge(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -78,9 +72,8 @@ impl PyTupleRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_le(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_le(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -89,9 +82,8 @@ impl PyTupleRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let e1 = self.elements.borrow(); - let e2 = get_elements(&other); - let elements = e1.iter().chain(e2.iter()).cloned().collect(); + let e2 = get_elements_tuple(&other); + let elements = self.elements.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_tuple(elements)) } else { Ok(vm.ctx.not_implemented()) @@ -99,12 +91,12 @@ impl PyTupleRef { } fn bool(self, _vm: &VirtualMachine) -> bool { - !self.elements.borrow().is_empty() + !self.elements.is_empty() } fn count(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut count: usize = 0; - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { count += 1; } else { @@ -119,9 +111,8 @@ impl PyTupleRef { fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = &self.elements.borrow(); - let other = get_elements(&other); - let res = seq_equal(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_equal(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -129,7 +120,7 @@ impl PyTupleRef { } fn hash(self, vm: &VirtualMachine) -> PyResult { - pyhash::hash_iter(self.elements.borrow().iter(), vm) + pyhash::hash_iter(self.elements.iter(), vm) } fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { @@ -140,13 +131,13 @@ impl PyTupleRef { } fn len(self, _vm: &VirtualMachine) -> usize { - self.elements.borrow().len() + self.elements.len() } fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; - for elem in self.elements.borrow().iter() { + for elem in self.elements.iter() { let s = vm.to_repr(elem)?; str_parts.push(s.value.clone()); } @@ -163,21 +154,16 @@ impl PyTupleRef { } fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements.borrow(), counter); + let new_elements = seq_mul(&self.elements, counter); vm.ctx.new_tuple(new_elements) } fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - get_item( - vm, - self.as_object(), - &self.elements.borrow(), - needle.clone(), - ) + get_item(vm, self.as_object(), &self.elements, needle.clone()) } fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for (index, element) in self.elements.borrow().iter().enumerate() { + for (index, element) in self.elements.iter().enumerate() { if element.is(&needle) { return Ok(index); } @@ -190,7 +176,7 @@ impl PyTupleRef { } fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { return Ok(true); } @@ -234,8 +220,8 @@ impl PyValue for PyTupleIterator { impl PyTupleIterator { #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { - if self.position.get() < self.tuple.elements.borrow().len() { - let ret = self.tuple.elements.borrow()[self.position.get()].clone(); + if self.position.get() < self.tuple.elements.len() { + let ret = self.tuple.elements[self.position.get()].clone(); self.position.set(self.position.get() + 1); Ok(ret) } else { diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index d57dd14a9..4b0ac1ca9 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -57,10 +57,11 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serializer.serialize_i64(v.to_i64().unwrap()) // Although this may seem nice, it does not give the right result: // v.serialize(serializer) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) - || objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) - { - let elements = objsequence::get_elements(self.pyobject); + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) { + let elements = objsequence::get_elements_list(self.pyobject); + serialize_seq_elements(serializer, &elements) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) { + let elements = objsequence::get_elements_tuple(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index edab574b3..4f7cab5a1 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -349,14 +349,14 @@ impl Address { impl TryFromObject for Address { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let tuple = PyTupleRef::try_from_object(vm, obj)?; - if tuple.elements.borrow().len() != 2 { + if tuple.elements.len() != 2 { Err(vm.new_type_error("Address tuple should have only 2 values".to_string())) } else { Ok(Address { - host: PyStringRef::try_from_object(vm, tuple.elements.borrow()[0].clone())? + host: PyStringRef::try_from_object(vm, tuple.elements[0].clone())? .value .to_string(), - port: PyIntRef::try_from_object(vm, tuple.elements.borrow()[1].clone())? + port: PyIntRef::try_from_object(vm, tuple.elements[1].clone())? .as_bigint() .to_usize() .unwrap(), diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 6cd1165db..b2a46930f 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -527,7 +527,7 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.borrow().len()); + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception @@ -547,7 +547,7 @@ impl VirtualMachine { ))); } if let Some(defaults) = defaults { - let defaults = defaults.elements.borrow(); + let defaults = &defaults.elements; // We have sufficient defaults, so iterate over the corresponding names and use // the default if we don't already have a value for (default_index, i) in (required_args..nexpected_args).enumerate() { @@ -580,10 +580,10 @@ impl VirtualMachine { pub fn extract_elements(&self, value: &PyObjectRef) -> PyResult> { // Extract elements from item, if possible: - let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) - || objtype::isinstance(value, &self.ctx.list_type()) - { - objsequence::get_elements(value).to_vec() + let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) { + objsequence::get_elements_tuple(value).to_vec() + } else if objtype::isinstance(value, &self.ctx.list_type()) { + objsequence::get_elements_list(value).to_vec() } else { let iter = objiter::get_iter(self, value)?; objiter::get_all(self, &iter)? diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 6f40fb182..2f3b72076 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -40,10 +40,10 @@ pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { }); if let Ok(tb) = vm.get_attribute(py_err.clone(), "__traceback__") { if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let elements = objsequence::get_elements(&tb).to_vec(); + let elements = objsequence::get_elements_list(&tb).to_vec(); if let Some(top) = elements.get(0) { if objtype::isinstance(&top, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&top); + let element = objsequence::get_elements_list(&top); if let Some(lineno) = objint::to_int(vm, &element[1], 10) .ok() From 433434a1bbb242c90ebfadcba3010c77af160710 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Tue, 21 May 2019 23:01:04 +0900 Subject: [PATCH 601/658] Fix rsplit --- tests/snippets/strings.py | 5 +++-- vm/src/obj/objstr.rs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 521b56353..2abd35da7 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -40,7 +40,6 @@ assert -1 * "x" == "" a = 'Hallo' assert a.lower() == 'hallo' assert a.upper() == 'HALLO' -assert a.split('al') == ['H', 'lo'] assert a.startswith('H') assert a.startswith(('H', 1)) assert a.startswith(('A', 'H')) @@ -59,7 +58,9 @@ assert not a.isnumeric() assert a.istitle() assert a.isalpha() - +s = '1 2 3' +assert s.split(' ', 1) == ['1', '2 3'] +assert s.rsplit(' ', 1) == ['1 2', '3'] b = ' hallo ' assert b.strip() == 'hallo' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index cf42a1a5c..44542a970 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -301,10 +301,13 @@ impl PyString { let num_splits = num .into_option() .unwrap_or_else(|| value.split(pattern).count()); - let elements = value + let mut elements: Vec<_> = value .rsplitn(num_splits + 1, pattern) .map(|o| vm.ctx.new_str(o.to_string())) .collect(); + // Unlike Python rsplit, Rust rsplitn returns an iterator that + // starts from the end of the string. + elements.reverse(); vm.ctx.new_list(elements) } From 0098e8485a2b8c7a261d915892d6ab07ea48873f Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Wed, 22 May 2019 03:17:25 +0300 Subject: [PATCH 602/658] move PyFuncArgs.kwargs to IndexMap --- Cargo.lock | 7 +++++ tests/snippets/function_args.py | 14 +++++++++ vm/Cargo.toml | 1 + vm/src/frame.rs | 5 ++-- vm/src/function.rs | 50 +++++++++++++++------------------ vm/src/pyobject.rs | 3 +- wasm/lib/src/convert.rs | 2 +- 7 files changed, 51 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66b1f01af..272786ef3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,6 +356,11 @@ dependencies = [ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indexmap" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "itertools" version = "0.8.0" @@ -882,6 +887,7 @@ dependencies = [ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1453,6 +1459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "22eadcfadba76a730b2764eaa577d045f35e0ef5174b9c5b46adf1ee42b85e12" "checksum hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79296f72d53a89096cbc9a88c9547ee8dfe793388674620e2207593d370550ac" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "d4bda84b98977341bb6ba1086ecbd9eab8bc929de0f2923c118baf76c21dd0c8" diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index b9c5a1626..86c92617a 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -81,3 +81,17 @@ def fubar(x, y, obj=None): rest = [4, 5] fubar(obj=6, *rest) + + +# https://www.python.org/dev/peps/pep-0468/ +def func(**kwargs): + return list(kwargs.items()) + +empty_kwargs = func() +assert empty_kwargs == [] + +kwargs = func(a=1, b=2) +assert kwargs == [('a', 1), ('b', 2)] + +kwargs = func(a=1, b=2, c=3) +assert kwargs == [('a', 1), ('b', 2), ('c', 3)] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 0d80923fd..5ee8e81c4 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -29,6 +29,7 @@ lazy_static = "^1.0.1" lexical = "2.0.0" itertools = "^0.8.0" hexf = "0.1.0" +indexmap = "1.0.2" # TODO: release and publish to crates.io [dependencies.unicode-casing] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 5955660b6..c5e3b53ca 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -23,6 +23,7 @@ use crate::pyobject::{ TypeProtocol, }; use crate::vm::VirtualMachine; +use indexmap::IndexMap; use itertools::Itertools; /* @@ -674,7 +675,7 @@ impl Frame { let args: Vec = self.pop_multiple(*count); PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } bytecode::CallType::Keyword(count) => { @@ -697,7 +698,7 @@ impl Frame { .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { - vec![] + IndexMap::new() }; let args = self.pop_value(); let args = vm.extract_elements(&args)?; diff --git a/vm/src/function.rs b/vm/src/function.rs index e7dbe59e4..a02c85bb0 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; use std::mem; use std::ops::RangeInclusive; +use indexmap::IndexMap; + use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{isinstance, PyClassRef}; use crate::pyobject::{ @@ -17,7 +19,8 @@ use self::OptionalArg::*; #[derive(Debug, Default, Clone)] pub struct PyFuncArgs { pub args: Vec, - pub kwargs: Vec<(String, PyObjectRef)>, + // sorted map, according to https://www.python.org/dev/peps/pep-0468/ + pub kwargs: IndexMap, } /// Conversion from vector of python objects to function arguments. @@ -25,7 +28,7 @@ impl From> for PyFuncArgs { fn from(args: Vec) -> Self { PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } } @@ -34,7 +37,7 @@ impl From for PyFuncArgs { fn from(arg: PyObjectRef) -> Self { PyFuncArgs { args: vec![arg], - kwargs: vec![], + kwargs: IndexMap::new(), } } } @@ -58,9 +61,12 @@ impl FromArgs for PyFuncArgs { impl PyFuncArgs { pub fn new(mut args: Vec, kwarg_names: Vec) -> PyFuncArgs { - let mut kwargs = vec![]; - for name in kwarg_names.iter().rev() { - kwargs.push((name.clone(), args.pop().unwrap())); + // last `kwarg_names.len()` elements of args in order of appearance in the call signature + let kwarg_values = args.drain((args.len() - kwarg_names.len())..); + + let mut kwargs = IndexMap::new(); + for (name, value) in kwarg_names.iter().zip(kwarg_values) { + kwargs.insert(name.clone(), value); } PyFuncArgs { args, kwargs } } @@ -79,19 +85,15 @@ impl PyFuncArgs { } pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return arg_value.clone(); - } + if let Some(kwarg_value) = self.kwargs.get(key) { + return kwarg_value.clone(); } default.clone() } pub fn get_optional_kwarg(&self, key: &str) -> Option { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return Some(arg_value.clone()); - } + if let Some(kwarg_value) = self.kwargs.get(key) { + return Some(kwarg_value.clone()); } None } @@ -132,19 +134,10 @@ impl PyFuncArgs { } pub fn take_keyword(&mut self, name: &str) -> Option { - // TODO: change kwarg representation so this scan isn't necessary - if let Some(index) = self - .kwargs - .iter() - .position(|(arg_name, _)| arg_name == name) - { - Some(self.kwargs.remove(index).1) - } else { - None - } + self.kwargs.remove(name) } - pub fn remaining_keyword<'a>(&'a mut self) -> impl Iterator + 'a { + pub fn remaining_keywords<'a>(&'a mut self) -> impl Iterator + 'a { self.kwargs.drain(..) } @@ -192,7 +185,10 @@ impl PyFuncArgs { given_args, ))) } else if !self.kwargs.is_empty() { - Err(vm.new_type_error(format!("Unexpected keyword argument {}", self.kwargs[0].0))) + Err(vm.new_type_error(format!( + "Unexpected keyword argument {}", + self.kwargs.keys().next().unwrap() + ))) } else { Ok(bound) } @@ -264,7 +260,7 @@ where { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { let mut kwargs = HashMap::new(); - for (name, value) in args.remaining_keyword() { + for (name, value) in args.remaining_keywords() { kwargs.insert(name, T::try_from_object(vm, value)?); } Ok(KwArgs(kwargs)) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index afaad09b6..78c78e333 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -57,6 +57,7 @@ use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; +use indexmap::IndexMap; /* Python objects and references. @@ -1088,7 +1089,7 @@ impl PyIterable { self.method.clone(), PyFuncArgs { args: vec![], - kwargs: vec![], + kwargs: IndexMap::new(), }, )?; diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 2f3b72076..a65141c57 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -93,7 +93,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { let (key, val) = pair?; py_func_args .kwargs - .push((js_sys::JsString::from(key).into(), js_to_py(vm, val))); + .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); } } let result = vm.invoke(py_obj.clone(), py_func_args); From b5352da2c04285aa03b936016e983f6074ebf68b Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Wed, 22 May 2019 03:30:30 +0300 Subject: [PATCH 603/658] fix formatting --- vm/src/function.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index a02c85bb0..b9a7e09dc 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -137,7 +137,9 @@ impl PyFuncArgs { self.kwargs.remove(name) } - pub fn remaining_keywords<'a>(&'a mut self) -> impl Iterator + 'a { + pub fn remaining_keywords<'a>( + &'a mut self, + ) -> impl Iterator + 'a { self.kwargs.drain(..) } From 737ec52365ee0e3bca18ee7826f9ce5043045c44 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Wed, 22 May 2019 12:47:28 +0300 Subject: [PATCH 604/658] simplify PyFuncArgs get_kwarg/get_optional_kwarg with cloned() --- vm/src/function.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index b9a7e09dc..47fc051d5 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -85,17 +85,14 @@ impl PyFuncArgs { } pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { - if let Some(kwarg_value) = self.kwargs.get(key) { - return kwarg_value.clone(); - } - default.clone() + self.kwargs + .get(key) + .cloned() + .unwrap_or_else(|| default.clone()) } pub fn get_optional_kwarg(&self, key: &str) -> Option { - if let Some(kwarg_value) = self.kwargs.get(key) { - return Some(kwarg_value.clone()); - } - None + self.kwargs.get(key).cloned() } pub fn get_optional_kwarg_with_type( From fdd569cb8b397e18edcef4f81c2f6fa1adf6f744 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 23 May 2019 16:08:12 +0800 Subject: [PATCH 605/658] add crc32 --- vm/src/stdlib/binascii.rs | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 97fb76f8e..f0c6cc8ca 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -1,7 +1,9 @@ use crate::function::PyFuncArgs; use crate::obj::objbytes; +use crate::obj::objint; use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use num_traits::ToPrimitive; fn hex_nibble(n: u8) -> u8 { match n { @@ -56,6 +58,63 @@ fn binascii_unhexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(unhex)) } +static CRC32_TABLE: [u32; 256] = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +]; +fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(data, Some(vm.ctx.bytes_type()))], + optional = [(value, None)] + ); + + let bytes = objbytes::get_value(data); + let mut crc = match value { + None => 0u32, + Some(value) => objint::get_value(&value).to_u32().unwrap(), + }; + + crc = !crc; + for b in bytes.iter() { + let b = *b as u32; + crc = CRC32_TABLE[((crc ^ b) & 0xff) as usize] ^ (crc >> 8); + } + crc = crc ^ 0xffffffff; + Ok(vm.ctx.new_int(crc)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -64,5 +123,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "b2a_hex" => ctx.new_rustfunc(binascii_hexlify), "unhexlify" => ctx.new_rustfunc(binascii_unhexlify), "a2b_hex" => ctx.new_rustfunc(binascii_unhexlify), + "crc32" => ctx.new_rustfunc(binascii_crc32), }) } From 21bffc8ffa5b5b6586e7e7e37451af1e94575c2c Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 23 May 2019 16:11:17 +0800 Subject: [PATCH 606/658] add crc32 test --- tests/snippets/stdlib_binascii.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index e73e162bd..6b116fb9f 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -33,3 +33,7 @@ with assertRaises(ValueError): with assertRaises(ValueError): uh(b"nn") # Non-hexadecimal digit found + +assert binascii.crc32(b"hello world") == 222957957 +assert binascii.crc32(b"hello world", 555555) == 1216827162 +assert binascii.crc32(b"goodbye interesting world",777777) == 1885538403 \ No newline at end of file From 813c6112e69f33f36dc60def6f6996d4054313c3 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Thu, 23 May 2019 16:30:19 +0800 Subject: [PATCH 607/658] add crc32 test --- tests/snippets/stdlib_binascii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index 6b116fb9f..e56b3e47d 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -36,4 +36,4 @@ with assertRaises(ValueError): assert binascii.crc32(b"hello world") == 222957957 assert binascii.crc32(b"hello world", 555555) == 1216827162 -assert binascii.crc32(b"goodbye interesting world",777777) == 1885538403 \ No newline at end of file +assert binascii.crc32(b"goodbye interesting world",777777) == 1885538403 From 8afb6a201c42088067f92f8e23986a78d43131d5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 24 May 2019 10:23:30 +0300 Subject: [PATCH 608/658] RawIOBase inherits IOBase --- tests/snippets/stdlib_io.py | 6 ++++++ vm/src/stdlib/io.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_io.py b/tests/snippets/stdlib_io.py index 503759df7..e6da760d2 100644 --- a/tests/snippets/stdlib_io.py +++ b/tests/snippets/stdlib_io.py @@ -8,3 +8,9 @@ result = bb.read() assert len(result) <= 8*1024 assert len(result) >= 0 assert isinstance(result, bytes) + +with FileIO('README.md') as fio: + res = fio.read() + assert len(result) <= 8*1024 + assert len(result) >= 0 + assert isinstance(result, bytes) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index e9b49ec44..8b203cc78 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -377,7 +377,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); // IOBase Subclasses - let raw_io_base = py_class!(ctx, "RawIOBase", ctx.object(), {}); + let raw_io_base = py_class!(ctx, "RawIOBase", io_base.clone(), {}); let buffered_io_base = py_class!(ctx, "BufferedIOBase", io_base.clone(), { "__init__" => ctx.new_rustfunc(buffered_io_base_init) From fedbd29051702d1f984f7c4dc335b79ecef13c25 Mon Sep 17 00:00:00 2001 From: ZapAnton Date: Fri, 24 May 2019 13:01:15 +0300 Subject: [PATCH 609/658] objstr: Refactored the 'mul' method - Used `str::repeat` instead of manually building the result string - Rewritten the int type check to return error at the beginning of the method - Replaced the `unwrap` calls with the single `unwrap_or` --- vm/src/obj/objstr.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 44542a970..80b37d29a 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -183,22 +183,16 @@ impl PyString { #[pymethod(name = "__mul__")] fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &self.value; - let multiplier = objint::get_value(&val).to_i32().unwrap(); - let capacity = if multiplier > 0 { - multiplier.to_usize().unwrap() * value.len() - } else { - 0 - }; - let mut result = String::with_capacity(capacity); - for _x in 0..multiplier { - result.push_str(value.as_str()); - } - Ok(result) - } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) + if !objtype::isinstance(&val, &vm.ctx.int_type()) { + return Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))); } + let value = &self.value; + let multiplier = objint::get_value(&val) + .to_i32() + .map(|multiplier| if multiplier < 0 { 0 } else { multiplier }) + .and_then(|multiplier| multiplier.to_usize()) + .unwrap_or(0); + Ok(value.repeat(multiplier)) } #[pymethod(name = "__rmul__")] From e38c54985e3fd40ff0c22263f53079b71cd9361a Mon Sep 17 00:00:00 2001 From: ZapAnton Date: Fri, 24 May 2019 20:37:47 +0300 Subject: [PATCH 610/658] objstr: Replaced the if-else construct with the max function --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 80b37d29a..3082a0c6e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -189,7 +189,7 @@ impl PyString { let value = &self.value; let multiplier = objint::get_value(&val) .to_i32() - .map(|multiplier| if multiplier < 0 { 0 } else { multiplier }) + .map(|multiplier| multiplier.max(0)) .and_then(|multiplier| multiplier.to_usize()) .unwrap_or(0); Ok(value.repeat(multiplier)) From a38f205f0e52ef607202c53333f08b05bb468200 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 25 May 2019 11:13:25 +0300 Subject: [PATCH 611/658] Add support for frozen modules --- vm/src/frozen.rs | 11 +++++++++++ vm/src/import.rs | 2 ++ vm/src/lib.rs | 1 + vm/src/vm.rs | 4 ++++ 4 files changed, 18 insertions(+) create mode 100644 vm/src/frozen.rs diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs new file mode 100644 index 000000000..cc6eb85fe --- /dev/null +++ b/vm/src/frozen.rs @@ -0,0 +1,11 @@ +use std::collections::hash_map::HashMap; + +const HELLO: &str = "initialized = True +print(\"Hello world!\") +"; + +pub fn get_module_inits() -> HashMap { + let mut modules = HashMap::new(); + modules.insert("__hello__".to_string(), HELLO); + modules +} diff --git a/vm/src/import.rs b/vm/src/import.rs index 16385e3d3..0e8b7ba53 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -18,6 +18,8 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s // First, see if we already loaded the module: if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { Ok(module) + } else if let Some(frozen) = vm.frozen.borrow().get(module_name) { + import_file(vm, module_name, "frozen".to_string(), frozen.to_string()) } else if let Some(make_module_func) = vm.stdlib_inits.borrow().get(module_name) { let module = make_module_func(vm); sys_modules.set_item(module_name, module.clone(), vm)?; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index caa63d145..24f28709e 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -45,6 +45,7 @@ pub mod eval; mod exceptions; pub mod format; pub mod frame; +mod frozen; pub mod function; pub mod import; pub mod obj; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index b2a46930f..17d1caffb 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -16,6 +16,7 @@ use crate::builtins; use crate::bytecode; use crate::error::CompileError; use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; +use crate::frozen; use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; @@ -53,6 +54,7 @@ pub struct VirtualMachine { pub frames: RefCell>, pub wasm_id: Option, pub exceptions: RefCell>, + pub frozen: RefCell>, } impl VirtualMachine { @@ -65,6 +67,7 @@ impl VirtualMachine { let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); + let frozen = RefCell::new(frozen::get_module_inits()); let vm = VirtualMachine { builtins: builtins.clone(), sys_module: sysmod.clone(), @@ -73,6 +76,7 @@ impl VirtualMachine { frames: RefCell::new(vec![]), wasm_id: None, exceptions: RefCell::new(vec![]), + frozen, }; builtins::make_module(&vm, builtins.clone()); From f33d15ab6af5c1ba5c7008c925acbe3f8cb5af96 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 25 May 2019 11:15:17 +0300 Subject: [PATCH 612/658] Test frozen modules import --- tests/snippets/frozen.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/snippets/frozen.py diff --git a/tests/snippets/frozen.py b/tests/snippets/frozen.py new file mode 100644 index 000000000..d03658c19 --- /dev/null +++ b/tests/snippets/frozen.py @@ -0,0 +1,2 @@ +import __hello__ +assert __hello__.initialized == True From 2c74d7cc1245f0ea66113b5ddf8005c6d9578598 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 25 May 2019 15:02:03 +0300 Subject: [PATCH 613/658] Add _imp --- vm/src/stdlib/imp.rs | 15 +++++++++++++++ vm/src/stdlib/mod.rs | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 vm/src/stdlib/imp.rs diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs new file mode 100644 index 000000000..1498f7435 --- /dev/null +++ b/vm/src/stdlib/imp.rs @@ -0,0 +1,15 @@ +use crate::pyobject::{PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(vec![])) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_imp", { + "extension_suffixes" => ctx.new_rustfunc(imp_extension_suffixes) + }); + + module +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 2b17586c6..e475ba057 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,6 +1,7 @@ mod ast; mod binascii; mod dis; +mod imp; mod itertools; pub(crate) mod json; mod keyword; @@ -51,6 +52,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); modules.insert("_weakref".to_string(), Box::new(weakref::make_module)); + modules.insert("_imp".to_string(), Box::new(imp::make_module)); // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] From 8197b17c1462ae14b3892476215d9004da73fc98 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 25 May 2019 15:07:51 +0300 Subject: [PATCH 614/658] Add empty _imp locks methods --- vm/src/stdlib/imp.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 1498f7435..a60f89751 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -5,10 +5,28 @@ fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_list(vec![])) } +fn imp_acquire_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_release_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { - "extension_suffixes" => ctx.new_rustfunc(imp_extension_suffixes) + "extension_suffixes" => ctx.new_rustfunc(imp_extension_suffixes), + "acquire_lock" => ctx.new_rustfunc(imp_acquire_lock), + "release_lock" => ctx.new_rustfunc(imp_release_lock), + "lock_held" => ctx.new_rustfunc(imp_lock_held), }); module From 7a467781e2e0dc12a10b97d9e8fba51e1ca0256d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 25 May 2019 15:33:57 +0300 Subject: [PATCH 615/658] Add _imp.is_builtin --- tests/snippets/imp.py | 5 +++++ vm/src/stdlib/imp.rs | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/snippets/imp.py diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py new file mode 100644 index 000000000..45332a9b9 --- /dev/null +++ b/tests/snippets/imp.py @@ -0,0 +1,5 @@ +import _imp + +assert _imp.is_builtin("time") == True +assert _imp.is_builtin("os") == False +assert _imp.is_builtin("not existing module") == False diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index a60f89751..4c6c2e008 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,3 +1,4 @@ +use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; @@ -20,6 +21,14 @@ fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { Ok(()) } +fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { + if let Some(_) = vm.stdlib_inits.borrow().get(name.as_str()) { + true + } else { + false + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -27,6 +36,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "acquire_lock" => ctx.new_rustfunc(imp_acquire_lock), "release_lock" => ctx.new_rustfunc(imp_release_lock), "lock_held" => ctx.new_rustfunc(imp_lock_held), + "is_builtin" => ctx.new_rustfunc(imp_is_builtin), }); module From a2e64c0425f9d55310d4e69ab46bea88a5896dd8 Mon Sep 17 00:00:00 2001 From: ZapAnton Date: Sat, 25 May 2019 16:33:47 +0300 Subject: [PATCH 616/658] objstr: Replaced the unwrap_or with the OverflowError --- tests/snippets/strings.py | 2 ++ vm/src/obj/objstr.rs | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 2abd35da7..786622e9f 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -37,6 +37,8 @@ assert 3 * "xy" == "xyxyxy" assert 0 * "x" == "" assert -1 * "x" == "" +assert_raises(OverflowError, lambda: 'xy' * 234234234234234234234234234234) + a = 'Hallo' assert a.lower() == 'hallo' assert a.upper() == 'HALLO' diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 3082a0c6e..6545c9985 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -186,13 +186,14 @@ impl PyString { if !objtype::isinstance(&val, &vm.ctx.int_type()) { return Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))); } - let value = &self.value; - let multiplier = objint::get_value(&val) - .to_i32() + objint::get_value(&val) + .to_isize() .map(|multiplier| multiplier.max(0)) .and_then(|multiplier| multiplier.to_usize()) - .unwrap_or(0); - Ok(value.repeat(multiplier)) + .map(|multiplier| self.value.repeat(multiplier)) + .ok_or_else(|| { + vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_string()) + }) } #[pymethod(name = "__rmul__")] From 017377061d4acc322cd3adf8ed208e7bb7a683ec Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 25 May 2019 15:52:44 -0500 Subject: [PATCH 617/658] Check RUSTPYTHONPATH as well as PYTHONPATH env variables --- vm/src/sysmodule.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index aefdb117b..bc24baf1b 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -54,18 +54,23 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR "cache_tag" => ctx.new_str("rustpython-01".to_string()), }); - let path_list = match env::var_os("PYTHONPATH") { - Some(paths) => env::split_paths(&paths) - .map(|path| { - ctx.new_str( - path.to_str() - .expect("PYTHONPATH isn't valid unicode") - .to_string(), - ) - }) - .collect(), - None => vec![], - }; + fn get_paths(env_name: &str) -> impl Iterator { + match env::var_os(env_name) { + Some(paths) => env::split_paths(&paths), + None => env::split_paths(""), + } + } + + let path_list = get_paths("RUSTPYTHONPATH") + .chain(get_paths("PYTHONPATH")) + .map(|path| { + ctx.new_str( + path.to_str() + .expect("PYTHONPATH isn't valid unicode") + .to_string(), + ) + }) + .collect(); let path = ctx.new_list(path_list); let platform = if cfg!(target_os = "linux") { From 770f54791eea3484a46340dc7749fccd3e07f446 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 25 May 2019 17:28:25 -0500 Subject: [PATCH 618/658] Fix ownership errors --- vm/src/sysmodule.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index bc24baf1b..dd52a189a 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -54,15 +54,19 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR "cache_tag" => ctx.new_str("rustpython-01".to_string()), }); - fn get_paths(env_name: &str) -> impl Iterator { - match env::var_os(env_name) { - Some(paths) => env::split_paths(&paths), + fn get_paths<'a>( + paths: &'a Option, + ) -> impl Iterator + 'a { + match paths { + Some(paths) => env::split_paths(paths), None => env::split_paths(""), } } - let path_list = get_paths("RUSTPYTHONPATH") - .chain(get_paths("PYTHONPATH")) + let rustpy_path = env::var_os("RUSTPYTHONPATH"); + let py_path = env::var_os("PYTHONPATH"); + let path_list = get_paths(&rustpy_path) + .chain(get_paths(&py_path)) .map(|path| { ctx.new_str( path.to_str() From 042a673ea23350ccdbc4cd10135bad0b33d32499 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 26 May 2019 21:16:40 +0300 Subject: [PATCH 619/658] Use contains_key --- vm/src/stdlib/imp.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 4c6c2e008..f5d889852 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -22,11 +22,7 @@ fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { } fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { - if let Some(_) = vm.stdlib_inits.borrow().get(name.as_str()) { - true - } else { - false - } + vm.stdlib_inits.borrow().contains_key(name.as_str()) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From b2f17a18ea4467f25f93529bdc076bfb450d90d3 Mon Sep 17 00:00:00 2001 From: Daniel Shaulov Date: Mon, 27 May 2019 22:09:47 +0300 Subject: [PATCH 620/658] Add support for marshal with loads and dumps --- Cargo.lock | 19 ++++++++++++++++++- parser/Cargo.toml | 1 + parser/src/ast.rs | 4 +++- parser/src/lexer.rs | 3 ++- tests/snippets/stdlib_marshal.py | 7 +++++++ vm/Cargo.toml | 8 ++++---- vm/src/bytecode.rs | 20 +++++++++++--------- vm/src/pyobject.rs | 2 +- vm/src/stdlib/marshal.rs | 24 ++++++++++++++++++++++++ vm/src/stdlib/mod.rs | 2 ++ 10 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 tests/snippets/stdlib_marshal.py create mode 100644 vm/src/stdlib/marshal.rs diff --git a/Cargo.lock b/Cargo.lock index 272786ef3..44ad4ceed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,6 +78,16 @@ dependencies = [ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bincode" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bit-set" version = "0.5.1" @@ -509,6 +519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -517,6 +528,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -875,6 +887,7 @@ dependencies = [ "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -883,6 +896,7 @@ dependencies = [ name = "rustpython_vm" version = "0.1.0" dependencies = [ + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -904,7 +918,6 @@ dependencies = [ "rustpython_derive 0.1.0", "rustpython_parser 0.0.1", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", @@ -970,6 +983,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "serde" version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" @@ -1423,6 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e3c4e36bd..6cc7805c6 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -16,3 +16,4 @@ num-bigint = "0.2" num-traits = "0.2" unicode-xid = "0.1.0" unic-emoji-char = "0.9.0" +serde = { version = "1.0.66", features = ["derive"] } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index ec0677fe8..cf7676a3b 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -4,6 +4,8 @@ pub use super::lexer::Location; use num_bigint::BigInt; +use serde::{Serialize, Deserialize}; + /* #[derive(Debug)] @@ -381,7 +383,7 @@ pub enum Number { } /// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum ConversionFlag { /// Converts by calling `str()`. Str, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index f95563eac..269864daf 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -12,6 +12,7 @@ use std::collections::HashMap; use std::str::FromStr; use unic_emoji_char::is_emoji_presentation; use unicode_xid::UnicodeXID; +use serde::{Serialize, Deserialize}; #[derive(Clone, Copy, PartialEq, Debug)] struct IndentationLevel { @@ -71,7 +72,7 @@ pub enum LexicalErrorType { OtherError(String), } -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Location { row: usize, column: usize, diff --git a/tests/snippets/stdlib_marshal.py b/tests/snippets/stdlib_marshal.py new file mode 100644 index 000000000..118bb1bdd --- /dev/null +++ b/tests/snippets/stdlib_marshal.py @@ -0,0 +1,7 @@ +import marshal +orig = compile("1 + 1", "", 'eval') + +dumped = marshal.dumps(orig) +loaded = marshal.loads(dumped) + +assert eval(loaded) == eval(orig) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5ee8e81c4..4f6e4b1b2 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,8 +6,8 @@ edition = "2018" [dependencies] bitflags = "1.0.4" -num-complex = "0.2" -num-bigint = "0.2.1" +num-complex = { version = "0.2", features = ["serde"] } +num-bigint = { version = "0.2.1", features = ["serde"] } num-traits = "0.2" num-integer = "0.1.39" num-rational = "0.2.1" @@ -15,8 +15,7 @@ rand = "0.5" log = "0.3" rustpython_derive = {path = "../derive"} rustpython_parser = {path = "../parser"} -serde = "1.0.66" -serde_derive = "1.0.66" +serde = { version = "1.0.66", features = ["derive"] } serde_json = "1.0.26" byteorder = "1.2.6" regex = "1" @@ -30,6 +29,7 @@ lexical = "2.0.0" itertools = "^0.8.0" hexf = "0.1.0" indexmap = "1.0.2" +bincode = "1.1.4" # TODO: release and publish to crates.io [dependencies.unicode-casing] diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 09bb7e70a..5291cd0c3 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -10,10 +10,11 @@ use num_complex::Complex64; use rustpython_parser::ast; use std::collections::{HashMap, HashSet}; use std::fmt; +use serde::{Serialize, Deserialize}; /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct CodeObject { pub instructions: Vec, pub label_map: HashMap, @@ -29,6 +30,7 @@ pub struct CodeObject { } bitflags! { + #[derive(Serialize, Deserialize)] pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; const HAS_KW_ONLY_DEFAULTS = 0x02; @@ -38,7 +40,7 @@ bitflags! { pub type Label = usize; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum NameScope { Local, NonLocal, @@ -46,7 +48,7 @@ pub enum NameScope { } /// A Single bytecode instruction. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Instruction { Import { name: String, @@ -187,14 +189,14 @@ pub enum Instruction { use self::Instruction::*; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum CallType { Positional(usize), Keyword(usize), Ex(bool), } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Constant { Integer { value: BigInt }, Float { value: f64 }, @@ -208,7 +210,7 @@ pub enum Constant { Ellipsis, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum ComparisonOperator { Greater, GreaterOrEqual, @@ -222,7 +224,7 @@ pub enum ComparisonOperator { IsNot, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum BinaryOperator { Power, Multiply, @@ -240,7 +242,7 @@ pub enum BinaryOperator { Or, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum UnaryOperator { Not, Invert, @@ -248,7 +250,7 @@ pub enum UnaryOperator { Plus, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Varargs { None, Unnamed, diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 78c78e333..d88991b9e 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -294,7 +294,7 @@ impl PyContext { let zip_type = create_type("zip", &type_type, &object_type); let bool_type = create_type("bool", &type_type, &int_type); let memoryview_type = create_type("memoryview", &type_type, &object_type); - let code_type = create_type("code", &type_type, &int_type); + let code_type = create_type("code", &type_type, &object_type); let range_type = create_type("range", &type_type, &object_type); let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs new file mode 100644 index 000000000..047559881 --- /dev/null +++ b/vm/src/stdlib/marshal.rs @@ -0,0 +1,24 @@ +use crate::obj::objcode::{PyCodeRef, PyCode}; +use crate::pyobject::{PyObjectRef, PyResult, IntoPyObject}; +use crate::vm::VirtualMachine; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::bytecode; + +fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult { + PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm) +} + +fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let code = bincode::deserialize::(&code_bytes).unwrap(); + let pycode = PyCode { code }; + pycode.into_pyobject(vm) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "marshal", { + "loads" => ctx.new_rustfunc(marshal_loads), + "dumps" => ctx.new_rustfunc(marshal_dumps) + }) +} \ No newline at end of file diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index e475ba057..ef2d7f3a5 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -5,6 +5,7 @@ mod imp; mod itertools; pub(crate) mod json; mod keyword; +mod marshal; mod math; mod platform; mod pystruct; @@ -42,6 +43,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("itertools".to_string(), Box::new(itertools::make_module)); modules.insert("json".to_string(), Box::new(json::make_module)); modules.insert("keyword".to_string(), Box::new(keyword::make_module)); + modules.insert("marshal".to_string(), Box::new(marshal::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); modules.insert("platform".to_string(), Box::new(platform::make_module)); modules.insert("re".to_string(), Box::new(re::make_module)); From bd89c4e32b28b6b907e9ee72ae437137b025479a Mon Sep 17 00:00:00 2001 From: Daniel Shaulov Date: Mon, 27 May 2019 22:19:07 +0300 Subject: [PATCH 621/658] Reformat with rustfmt --- parser/src/ast.rs | 2 +- parser/src/lexer.rs | 2 +- vm/src/bytecode.rs | 2 +- vm/src/stdlib/marshal.rs | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index cf7676a3b..1b985db2d 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -4,7 +4,7 @@ pub use super::lexer::Location; use num_bigint::BigInt; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; /* #[derive(Debug)] diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 269864daf..1fe74e3e1 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -7,12 +7,12 @@ extern crate unicode_xid; pub use super::token::Tok; use num_bigint::BigInt; use num_traits::Num; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; use unic_emoji_char::is_emoji_presentation; use unicode_xid::UnicodeXID; -use serde::{Serialize, Deserialize}; #[derive(Clone, Copy, PartialEq, Debug)] struct IndentationLevel { diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 5291cd0c3..fb8cab98b 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -8,9 +8,9 @@ use num_bigint::BigInt; use num_complex::Complex64; use rustpython_parser::ast; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt; -use serde::{Serialize, Deserialize}; /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 047559881..e7afc86d2 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -1,8 +1,8 @@ -use crate::obj::objcode::{PyCodeRef, PyCode}; -use crate::pyobject::{PyObjectRef, PyResult, IntoPyObject}; -use crate::vm::VirtualMachine; -use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::bytecode; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::obj::objcode::{PyCode, PyCodeRef}; +use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult { PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm) @@ -21,4 +21,4 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "loads" => ctx.new_rustfunc(marshal_loads), "dumps" => ctx.new_rustfunc(marshal_dumps) }) -} \ No newline at end of file +} From 7be801db2f9d8b671387aa400d3e30e2f6f972e2 Mon Sep 17 00:00:00 2001 From: rbrtberglund Date: Sun, 26 May 2019 00:42:01 +0200 Subject: [PATCH 622/658] implemented __setitem__ for bytearray --- tests/snippets/bytearray.py | 15 +++++++++ vm/src/obj/objbytearray.rs | 10 ++++++ vm/src/obj/objbyteinner.rs | 63 +++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index dff0852c4..d3930c9cc 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -691,3 +691,18 @@ assert a == bytearray(b'abdabcd') a = bytearray(b'hello, world') a.reverse() assert a == bytearray(b'dlrow ,olleh') + +# __setitem__ +a = bytearray(b'test') +a[0] = 1 +assert a == bytearray(b'\x01est') +with assertRaises(TypeError): + a[0] = b'a' +with assertRaises(TypeError): + a[0] = memoryview(b'a') +a[:2] = [0, 9] +assert a == bytearray(b'\x00\x09st') +a[1:3] = b'test' +assert a == bytearray(b'\x00testt') +a[:6] = memoryview(b'test') +assert a == bytearray(b'test') \ No newline at end of file diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index e5078bdbb..2c72a3f46 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -166,6 +166,16 @@ impl PyByteArrayRef { self.inner.borrow().getitem(needle, vm) } + #[pymethod(name = "__setitem__")] + fn setitem( + self, + needle: Either, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.borrow_mut().setitem(needle, value, vm) + } + #[pymethod(name = "isalnum")] fn isalnum(self, vm: &VirtualMachine) -> PyResult { self.inner.borrow().isalnum(vm) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index c07470be2..8dca0d33a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -426,6 +426,69 @@ impl PyByteInner { } } + pub fn setitem( + &mut self, + needle: Either, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match needle { + Either::A(int) => { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + let result = match_class!(object, + i @ PyInt => { + if let Some(value) = i.as_bigint().to_u8() { + Ok(value) + }else{ + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } + }, + _ => {Err(vm.new_type_error("an integer is required".to_string()))} + ); + match result { + Ok(value) => { + self.elements[idx] = value; + Ok(vm.new_int(value)) + } + Err(error) => Err(error), + } + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + Either::B(slice) => { + let sec = match PyIterable::try_from_object(vm, object.clone()) { + Ok(sec) => { + let items: Result, _> = sec.iter(vm)?.collect(); + Ok(items? + .into_iter() + .map(|obj| u8::try_from_object(vm, obj)) + .collect::>>()?) + } + _ => match_class!(object, + i @ PyMemoryView => { + Ok(i.get_obj_value().unwrap()) + }, + _ => Err(vm.new_index_error( + "can assign only bytes, buffers, or iterables of ints in range(0, 256)" + .to_string()))), + }; + match sec { + Ok(items) => { + let range = self + .elements + .get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); + self.elements.splice(range, items); + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) + } + Err(error) => Err(error), + } + } + } + } + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { Ok(vm.new_bool( !self.elements.is_empty() From 9b4b2d7fa6b9196f558ef063da9c01db13133c04 Mon Sep 17 00:00:00 2001 From: rbrtberglund Date: Mon, 27 May 2019 22:48:25 +0200 Subject: [PATCH 623/658] fixed SyntaxError: Got unexpected EOF in bytearray.py --- tests/snippets/bytearray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index d3930c9cc..f69edeac1 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -705,4 +705,4 @@ assert a == bytearray(b'\x00\x09st') a[1:3] = b'test' assert a == bytearray(b'\x00testt') a[:6] = memoryview(b'test') -assert a == bytearray(b'test') \ No newline at end of file +assert a == bytearray(b'test') From eed9c44a74319b54bb1f61096cc61ef937b48e8a Mon Sep 17 00:00:00 2001 From: rbrtberglund Date: Wed, 29 May 2019 00:02:58 +0200 Subject: [PATCH 624/658] improved PyByteInner.setitem --- vm/src/obj/objbyteinner.rs | 107 +++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8dca0d33a..e420fa038 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -426,6 +426,58 @@ impl PyByteInner { } } + fn setindex(&mut self, int: PyIntRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + let result = match_class!(object, + i @ PyInt => { + if let Some(value) = i.as_bigint().to_u8() { + Ok(value) + }else{ + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } + }, + _ => {Err(vm.new_type_error("an integer is required".to_string()))} + ); + let value = result?; + self.elements[idx] = value; + Ok(vm.new_int(value)) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + fn setslice( + &mut self, + slice: PySliceRef, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let sec = match PyIterable::try_from_object(vm, object.clone()) { + Ok(sec) => { + let items: Result, _> = sec.iter(vm)?.collect(); + Ok(items? + .into_iter() + .map(|obj| u8::try_from_object(vm, obj)) + .collect::>>()?) + } + _ => match_class!(object, + i @ PyMemoryView => { + Ok(i.get_obj_value().unwrap()) + }, + _ => Err(vm.new_index_error( + "can assign only bytes, buffers, or iterables of ints in range(0, 256)" + .to_string()))), + }; + let items = sec?; + let range = self + .elements + .get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); + self.elements.splice(range, items); + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) + } + pub fn setitem( &mut self, needle: Either, @@ -433,59 +485,8 @@ impl PyByteInner { vm: &VirtualMachine, ) -> PyResult { match needle { - Either::A(int) => { - if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { - let result = match_class!(object, - i @ PyInt => { - if let Some(value) = i.as_bigint().to_u8() { - Ok(value) - }else{ - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) - } - }, - _ => {Err(vm.new_type_error("an integer is required".to_string()))} - ); - match result { - Ok(value) => { - self.elements[idx] = value; - Ok(vm.new_int(value)) - } - Err(error) => Err(error), - } - } else { - Err(vm.new_index_error("index out of range".to_string())) - } - } - Either::B(slice) => { - let sec = match PyIterable::try_from_object(vm, object.clone()) { - Ok(sec) => { - let items: Result, _> = sec.iter(vm)?.collect(); - Ok(items? - .into_iter() - .map(|obj| u8::try_from_object(vm, obj)) - .collect::>>()?) - } - _ => match_class!(object, - i @ PyMemoryView => { - Ok(i.get_obj_value().unwrap()) - }, - _ => Err(vm.new_index_error( - "can assign only bytes, buffers, or iterables of ints in range(0, 256)" - .to_string()))), - }; - match sec { - Ok(items) => { - let range = self - .elements - .get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); - self.elements.splice(range, items); - Ok(vm - .ctx - .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) - } - Err(error) => Err(error), - } - } + Either::A(int) => self.setindex(int, object, vm), + Either::B(slice) => self.setslice(slice, object, vm), } } From 3675ce859a8fc2fe914507368de8c560b395386c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 11 May 2019 15:13:17 +0900 Subject: [PATCH 625/658] Add sys.getfilesystemencoding, sys.getfilesystemencodeerrors --- tests/snippets/sysmod.py | 3 +++ vm/src/sysmodule.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index 409d961ed..b8b9bed9f 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -10,3 +10,6 @@ assert 'sys' in sys.builtin_module_names assert isinstance(sys.implementation.name, str) assert isinstance(sys.implementation.cache_tag, str) + +assert sys.getfilesystemencoding() == 'utf-8' +assert sys.getfilesystemencodeerrors().startswith('surrogate') diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index aefdb117b..ec0f3d433 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -40,6 +40,21 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } +fn sys_getfilesystemencoding(_vm: &VirtualMachine) -> String { + // TODO: implmement non-utf-8 mode. + "utf-8".to_string() +} + +#[cfg(not(windows))] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogateescape".to_string() +} + +#[cfg(windows)] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogatepass".to_string() +} + // TODO implement string interning, this will be key for performance fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { value @@ -161,6 +176,8 @@ settrace() -- set the global debug tracing function "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), "implementation" => implementation, + "getfilesystemencoding" => ctx.new_rustfunc(sys_getfilesystemencoding), + "getfilesystemencodeerrors" => ctx.new_rustfunc(sys_getfilesystemencodeerrors), "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, From ad357d08af064ff6611760467a9450321f50c6fe Mon Sep 17 00:00:00 2001 From: jgirardet Date: Tue, 7 May 2019 01:42:16 +0900 Subject: [PATCH 626/658] normalize_encoding --- vm/src/obj/objbyteinner.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index e420fa038..ee7ad3d3a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -74,6 +74,25 @@ pub struct ByteInnerNewOptions { encoding: OptionalArg, } +//same algorithm as cpython +pub fn normalize_encoding(encoding: &str) -> String { + let mut res = String::new(); + let mut punct = false; + + for c in encoding.chars() { + if c.is_alphanumeric() || c == '.' { + if punct && !res.is_empty() { + res.push('_') + } + res.push(c.to_ascii_lowercase()); + punct = false; + } else { + punct = true; + } + } + res +} + impl ByteInnerNewOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) From 7f2560c9e1b89d63559870048b6469711400a336 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 May 2019 13:48:36 +0900 Subject: [PATCH 627/658] Add str.encode for utf-8 --- tests/snippets/strings.py | 10 ++++++++++ vm/src/function.rs | 11 +++++++++++ vm/src/obj/objbyteinner.rs | 26 +++++++++++++++----------- vm/src/obj/objstr.rs | 26 ++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 786622e9f..aaeaed5f3 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -206,3 +206,13 @@ def try_mutate_str(): word[0] = 'x' assert_raises(TypeError, try_mutate_str) + +ss = ['Hello', '안녕', '👋'] +bs = [b'Hello', b'\xec\x95\x88\xeb\x85\x95', b'\xf0\x9f\x91\x8b'] + +for s, b in zip(ss, bs): + assert s.encode() == b + +for s, b, e in zip(ss, bs, ['u8', 'U8', 'utf-8', 'UTF-8', 'utf_8']): + assert s.encode(e) == b + # assert s.encode(encoding=e) == b diff --git a/vm/src/function.rs b/vm/src/function.rs index 47fc051d5..22802ac1f 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -374,6 +374,17 @@ impl OptionalArg { Missing => f(), } } + + pub fn map_or_else(self, default: D, f: F) -> U + where + D: FnOnce() -> U, + F: FnOnce(T) -> U, + { + match self { + Present(value) => f(value), + Missing => default(), + } + } } impl FromArgs for OptionalArg diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index ee7ad3d3a..85bedd301 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -93,6 +93,18 @@ pub fn normalize_encoding(encoding: &str) -> String { res } +pub fn encode_to_vec(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult> { + let encoding = normalize_encoding(encoding); + if encoding == "utf_8" || encoding == "u8" { + Ok(value.as_bytes().to_vec()) + } else { + // TODO: different encoding + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding)), //should be lookup error + ); + } +} + impl ByteInnerNewOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) @@ -100,17 +112,9 @@ impl ByteInnerNewOptions { if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { let encoding = enc.as_str(); - if encoding.to_lowercase() == "utf8" || encoding.to_lowercase() == "utf-8" - // TODO: different encoding - { - return Ok(PyByteInner { - elements: input.value.as_bytes().to_vec(), - }); - } else { - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding)), //should be lookup error - ); - } + return Ok(PyByteInner { + elements: encode_to_vec(&input.value, &encoding, vm)?, + }); } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 6545c9985..827b109f0 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -19,6 +19,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; +use super::objbyteinner; use super::objdict::PyDict; use super::objint::{self, PyInt}; use super::objnone::PyNone; @@ -957,6 +958,31 @@ impl PyString { } } } + + #[pymethod] + fn encode( + &self, + encoding: OptionalArg, + _errors: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let encoding = encoding.map_or_else( + || Ok("utf-8".to_string()), + |v| { + if objtype::isinstance(&v, &vm.ctx.str_type()) { + Ok(get_value(&v)) + } else { + Err(vm.new_type_error(format!( + "encode() argument 1 must be str, not {}", + v.class().name + ))) + } + }, + )?; + + let encoded = objbyteinner::encode_to_vec(&self.value, &encoding, vm)?; + Ok(vm.ctx.new_bytes(encoded)) + } } impl PyValue for PyString { From 59476c65bb19a151dd0e5e795e8119f5aadeef64 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 7 May 2019 02:05:46 +0900 Subject: [PATCH 628/658] PyBytes::from_string --- vm/src/obj/objbyteinner.rs | 32 ++++++++++++++++---------------- vm/src/obj/objbytes.rs | 7 +++++++ vm/src/obj/objstr.rs | 6 +++--- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 85bedd301..32a58255f 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -93,28 +93,14 @@ pub fn normalize_encoding(encoding: &str) -> String { res } -pub fn encode_to_vec(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult> { - let encoding = normalize_encoding(encoding); - if encoding == "utf_8" || encoding == "u8" { - Ok(value.as_bytes().to_vec()) - } else { - // TODO: different encoding - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding)), //should be lookup error - ); - } -} - impl ByteInnerNewOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = self.encoding { if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { - let encoding = enc.as_str(); - return Ok(PyByteInner { - elements: encode_to_vec(&input.value, &encoding, vm)?, - }); + let inner = PyByteInner::from_string(&input.value, enc.as_str(), vm)?; + return Ok(inner); } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } @@ -334,6 +320,20 @@ impl ByteInnerSplitlinesOptions { } impl PyByteInner { + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + let normalized = normalize_encoding(encoding); + if normalized == "utf_8" || normalized == "utf8" || normalized == "u8" { + Ok(PyByteInner { + elements: value.as_bytes().to_vec(), + }) + } else { + // TODO: different encoding + Err( + vm.new_value_error(format!("unknown encoding: {}", encoding)), // should be lookup error + ) + } + } + pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); for i in self.elements.iter() { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index dd2e1518c..33016e511 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -45,6 +45,13 @@ impl PyBytes { inner: PyByteInner { elements }, } } + + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + Ok(PyBytes { + inner: PyByteInner::from_string(value, encoding, vm)?, + }) + } + pub fn get_value(&self) -> &[u8] { &self.inner.elements } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 827b109f0..b0d19e9c3 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -19,7 +19,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objbyteinner; +use super::objbytes::PyBytes; use super::objdict::PyDict; use super::objint::{self, PyInt}; use super::objnone::PyNone; @@ -980,8 +980,8 @@ impl PyString { }, )?; - let encoded = objbyteinner::encode_to_vec(&self.value, &encoding, vm)?; - Ok(vm.ctx.new_bytes(encoded)) + let encoded = PyBytes::from_string(&self.value, &encoding, vm)?; + Ok(encoded.into_pyobject(vm)?) } } From 89729c3155791fceb562d1cf592e6f1d6f34f6c5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 14 May 2019 04:22:13 +0900 Subject: [PATCH 629/658] PyClassImpl for PySet, PyFrozenSet --- vm/src/obj/objset.rs | 307 ++++++++++++++++++++++--------------------- 1 file changed, 159 insertions(+), 148 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index e404fa924..d59bf388d 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,8 +8,8 @@ use std::fmt; use crate::dictdatatype; use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, + PyClassImpl, PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -19,12 +19,22 @@ use super::objtype::PyClassRef; pub type SetContentType = dictdatatype::Dict<()>; +/// set() -> new empty set object +/// set(iterable) -> new set object +/// +/// Build an unordered collection of unique elements. +#[pyclass] #[derive(Default)] pub struct PySet { inner: RefCell, } pub type PySetRef = PyRef; +/// frozenset() -> empty frozenset object +/// frozenset(iterable) -> frozenset object +/// +/// Build an immutable unordered collection of unique elements. +#[pyclass] #[derive(Default)] pub struct PyFrozenSet { inner: PySetInner, @@ -293,7 +303,9 @@ impl PySetInner { } } -impl PySetRef { +#[pyimpl] +impl PySet { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, iterable: OptionalArg, @@ -305,21 +317,25 @@ impl PySetRef { .into_ref_with_type(vm, cls) } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.inner.borrow().len() } - fn copy(self, _vm: &VirtualMachine) -> PySet { + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> PySet { PySet { inner: RefCell::new(self.inner.borrow().copy()), } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow().contains(&needle, vm) } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), @@ -327,7 +343,8 @@ impl PySetRef { ) } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), @@ -335,7 +352,8 @@ impl PySetRef { ) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), @@ -343,7 +361,8 @@ impl PySetRef { ) } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), @@ -351,7 +370,8 @@ impl PySetRef { ) } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), @@ -359,7 +379,8 @@ impl PySetRef { ) } - fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { inner: RefCell::new(self.inner.borrow().union(other, vm)?), @@ -369,7 +390,8 @@ impl PySetRef { )) } - fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), @@ -379,7 +401,8 @@ impl PySetRef { )) } - fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { inner: RefCell::new(self.inner.borrow().difference(other, vm)?), @@ -389,7 +412,8 @@ impl PySetRef { )) } - fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), @@ -399,35 +423,42 @@ impl PySetRef { )) } - fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow().isdisjoint(other, vm) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } - fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } - fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } - fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } - fn repr(self, vm: &VirtualMachine) -> PyResult { - let inner = self.inner.borrow(); + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = zelf.inner.borrow(); let s = if inner.len() == 0 { "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { inner.repr(vm)? } else { "set(...)".to_string() @@ -435,78 +466,108 @@ impl PySetRef { Ok(vm.new_str(s)) } - pub fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().add(&item, vm)?; Ok(()) } - fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().remove(&item, vm) } - fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn discard(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().discard(&item, vm)?; Ok(()) } - fn clear(self, _vm: &VirtualMachine) { + #[pymethod] + fn clear(&self, _vm: &VirtualMachine) { self.inner.borrow_mut().clear() } - fn pop(self, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + #[pymethod(name = "__ior__")] + fn ior(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner.borrow_mut().update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) } - fn update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(vm.get_none()) } - fn intersection_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn intersection_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().intersection_update(iterable, vm)?; Ok(vm.get_none()) } - fn iand(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__iand__")] + fn iand(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .intersection_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) } - fn difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn isub(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__isub__")] + fn isub(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .difference_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) } - fn symmetric_difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn symmetric_difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner .borrow_mut() .symmetric_difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn ixor(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__ixor__")] + fn ixor(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .symmetric_difference_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) + } + + #[pymethod] + fn issubset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.le(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.ge(other, vm) + } + + #[pymethod(name = "__hash__")] + fn hash(&self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type".to_string())) } } -impl PyFrozenSetRef { +#[pyimpl] +impl PyFrozenSet { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, iterable: OptionalArg, @@ -518,21 +579,25 @@ impl PyFrozenSetRef { .into_ref_with_type(vm, cls) } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.inner.len() } - fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> PyFrozenSet { PyFrozenSet { inner: self.inner.copy(), } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.contains(&needle, vm) } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.eq(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), @@ -540,7 +605,8 @@ impl PyFrozenSetRef { ) } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.ge(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), @@ -548,7 +614,8 @@ impl PyFrozenSetRef { ) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.gt(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), @@ -556,7 +623,8 @@ impl PyFrozenSetRef { ) } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.le(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), @@ -564,7 +632,8 @@ impl PyFrozenSetRef { ) } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, set @ PySet => self.inner.lt(&set.inner.borrow(), vm), frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), @@ -572,7 +641,8 @@ impl PyFrozenSetRef { ) } - fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { inner: self.inner.union(other, vm)?, @@ -582,7 +652,8 @@ impl PyFrozenSetRef { )) } - fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { inner: self.inner.intersection(other, vm)?, @@ -592,7 +663,8 @@ impl PyFrozenSetRef { )) } - fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { inner: self.inner.difference(other, vm)?, @@ -602,7 +674,8 @@ impl PyFrozenSetRef { )) } - fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { inner: self.inner.symmetric_difference(other, vm)?, @@ -612,41 +685,58 @@ impl PyFrozenSetRef { )) } - fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.isdisjoint(other, vm) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } - fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } - fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } - fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } - fn repr(self, vm: &VirtualMachine) -> PyResult { - let inner = &self.inner; + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = &zelf.inner; let s = if inner.len() == 0 { "frozenset()".to_string() - } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { format!("frozenset({})", inner.repr(vm)?) } else { "frozenset(...)".to_string() }; Ok(vm.new_str(s)) } + + #[pymethod] + fn issubset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.le(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.ge(other, vm) + } } struct SetIterable { @@ -670,86 +760,7 @@ impl TryFromObject for SetIterable { } } -fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult<()> { - Err(vm.new_type_error("unhashable type".to_string())) -} - pub fn init(context: &PyContext) { - let set_type = &context.set_type; - - let set_doc = "set() -> new empty set object\n\ - set(iterable) -> new set object\n\n\ - Build an unordered collection of unique elements."; - - extend_class!(context, set_type, { - "__hash__" => context.new_rustfunc(set_hash), - "__contains__" => context.new_rustfunc(PySetRef::contains), - "__len__" => context.new_rustfunc(PySetRef::len), - "__new__" => context.new_rustfunc(PySetRef::new), - "__repr__" => context.new_rustfunc(PySetRef::repr), - "__eq__" => context.new_rustfunc(PySetRef::eq), - "__ge__" => context.new_rustfunc(PySetRef::ge), - "__gt__" => context.new_rustfunc(PySetRef::gt), - "__le__" => context.new_rustfunc(PySetRef::le), - "__lt__" => context.new_rustfunc(PySetRef::lt), - "issubset" => context.new_rustfunc(PySetRef::le), - "issuperset" => context.new_rustfunc(PySetRef::ge), - "union" => context.new_rustfunc(PySetRef::union), - "__or__" => context.new_rustfunc(PySetRef::or), - "intersection" => context.new_rustfunc(PySetRef::intersection), - "__and__" => context.new_rustfunc(PySetRef::and), - "difference" => context.new_rustfunc(PySetRef::difference), - "__sub__" => context.new_rustfunc(PySetRef::sub), - "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PySetRef::xor), - "__doc__" => context.new_str(set_doc.to_string()), - "add" => context.new_rustfunc(PySetRef::add), - "remove" => context.new_rustfunc(PySetRef::remove), - "discard" => context.new_rustfunc(PySetRef::discard), - "clear" => context.new_rustfunc(PySetRef::clear), - "copy" => context.new_rustfunc(PySetRef::copy), - "pop" => context.new_rustfunc(PySetRef::pop), - "update" => context.new_rustfunc(PySetRef::update), - "__ior__" => context.new_rustfunc(PySetRef::ior), - "intersection_update" => context.new_rustfunc(PySetRef::intersection_update), - "__iand__" => context.new_rustfunc(PySetRef::iand), - "difference_update" => context.new_rustfunc(PySetRef::difference_update), - "__isub__" => context.new_rustfunc(PySetRef::isub), - "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), - "__ixor__" => context.new_rustfunc(PySetRef::ixor), - "__iter__" => context.new_rustfunc(PySetRef::iter), - "isdisjoint" => context.new_rustfunc(PySetRef::isdisjoint), - }); - - let frozenset_type = &context.frozenset_type; - - let frozenset_doc = "frozenset() -> empty frozenset object\n\ - frozenset(iterable) -> frozenset object\n\n\ - Build an immutable unordered collection of unique elements."; - - extend_class!(context, frozenset_type, { - "__new__" => context.new_rustfunc(PyFrozenSetRef::new), - "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), - "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), - "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), - "__le__" => context.new_rustfunc(PyFrozenSetRef::le), - "__lt__" => context.new_rustfunc(PyFrozenSetRef::lt), - "issubset" => context.new_rustfunc(PyFrozenSetRef::le), - "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), - "union" => context.new_rustfunc(PyFrozenSetRef::union), - "__or__" => context.new_rustfunc(PyFrozenSetRef::or), - "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), - "__and__" => context.new_rustfunc(PyFrozenSetRef::and), - "difference" => context.new_rustfunc(PyFrozenSetRef::difference), - "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), - "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PyFrozenSetRef::xor), - "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), - "__len__" => context.new_rustfunc(PyFrozenSetRef::len), - "__doc__" => context.new_str(frozenset_doc.to_string()), - "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), - "copy" => context.new_rustfunc(PyFrozenSetRef::copy), - "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter), - "isdisjoint" => context.new_rustfunc(PyFrozenSetRef::isdisjoint), - }); + PySet::extend_class(context, &context.set_type); + PyFrozenSet::extend_class(context, &context.frozenset_type); } From 31c88721e6513c5bd9ef6c83e03b8a592027ce6c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 03:06:39 +0900 Subject: [PATCH 630/658] Refactor PySet with try_set_inner! --- vm/src/obj/objset.rs | 70 +++++++++++++------------------------------- 1 file changed, 20 insertions(+), 50 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index d59bf388d..4aeacc0c0 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -303,6 +303,16 @@ impl PySetInner { } } +macro_rules! try_set_inner { + ($vm:expr, $other:expr, $op:expr) => { + match_class!($other, + set @ PySet => $op(&*set.inner.borrow()), + frozen @ PyFrozenSet => $op(&frozen.inner), + other => Err($vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class()))), + ); + }; +} + #[pyimpl] impl PySet { #[pymethod(name = "__new__")] @@ -336,47 +346,27 @@ impl PySet { #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.borrow().eq(other, vm)) } #[pymethod(name = "__ge__")] fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.borrow().ge(other, vm)) } #[pymethod(name = "__gt__")] fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.borrow().gt(other, vm)) } #[pymethod(name = "__le__")] fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.borrow().le(other, vm)) } #[pymethod(name = "__lt__")] fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.borrow().lt(other, vm)) } #[pymethod] @@ -598,47 +588,27 @@ impl PyFrozenSet { #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.eq(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.eq(other, vm)) } #[pymethod(name = "__ge__")] fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.ge(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.ge(other, vm)) } #[pymethod(name = "__gt__")] fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.gt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.gt(other, vm)) } #[pymethod(name = "__le__")] fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.le(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.le(other, vm)) } #[pymethod(name = "__lt__")] fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.lt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + try_set_inner!(vm, other, |other| self.inner.lt(other, vm)) } #[pymethod] From 979e1253ae046b8e5756fe0cfd0914bf32ee10a8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 04:08:05 +0900 Subject: [PATCH 631/658] Fix set/frozenset comparison --- tests/snippets/set.py | 20 +++++++++ vm/src/obj/objset.rs | 98 +++++++++++++++++++++++-------------------- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 65a568f9d..6e7413e56 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -27,6 +27,26 @@ assert set([1,2]) < set([1,2,3]) assert not set([1,2]) < set([1,2]) assert not set([1,3]) < set([1,2]) +assert (set() == []) is False +assert set().__eq__([]) == NotImplemented +assert_raises(TypeError, lambda: set() < [], "'<' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() <= [], "'<=' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() > [], "'>' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() >= [], "'>=' not supported between instances of 'set' and 'list'") +assert set().issuperset([]) +assert set().issubset([]) +assert not set().issuperset([1, 2, 3]) +assert set().issubset([1, 2]) + +assert (set() == 3) is False +assert set().__eq__(3) == NotImplemented +assert_raises(TypeError, lambda: set() < 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() <= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() > 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() >= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issuperset(3), "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issubset(3), "'int' object is not iterable") + class Hashable(object): def __init__(self, obj): self.obj = obj diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 4aeacc0c0..ce97e861c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -73,16 +73,22 @@ struct PySetInner { } impl PySetInner { - fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn new(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut set = PySetInner::default(); - if let OptionalArg::Present(iterable) = iterable { - for item in iterable.iter(vm)? { - set.add(&item?, vm)?; - } + for item in iterable.iter(vm)? { + set.add(&item?, vm)?; } Ok(set) } + fn from_arg(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(iterable) = iterable { + Self::new(iterable, vm) + } else { + Ok(PySetInner::default()) + } + } + fn len(&self) -> usize { self.content.len() } @@ -97,6 +103,7 @@ impl PySetInner { self.content.contains(vm, needle) } + #[inline] fn _compare_inner( &self, other: &PySetInner, @@ -104,26 +111,13 @@ impl PySetInner { swap: bool, vm: &VirtualMachine, ) -> PyResult { - let get_zelf = |swap: bool| -> &PySetInner { - if swap { - other - } else { - self - } - }; - let get_other = |swap: bool| -> &PySetInner { - if swap { - self - } else { - other - } - }; + let (zelf, other) = if swap { (other, self) } else { (self, other) }; - if size_func(get_zelf(swap).len(), get_other(swap).len()) { + if size_func(zelf.len(), other.len()) { return Ok(vm.new_bool(false)); } - for key in get_other(swap).content.keys() { - if !get_zelf(swap).contains(&key, vm)? { + for key in other.content.keys() { + if !zelf.contains(&key, vm)? { return Ok(vm.new_bool(false)); } } @@ -213,6 +207,20 @@ impl PySetInner { Ok(new_inner) } + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in other.iter(vm)? { + if !self.contains(&item?, vm)? { + return Ok(false); + } + } + Ok(true) + } + + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let other_set = PySetInner::new(other, vm)?; + self.le(&other_set, vm) + } + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { for item in other.iter(vm)? { if self.contains(&item?, vm)? { @@ -308,7 +316,7 @@ macro_rules! try_set_inner { match_class!($other, set @ PySet => $op(&*set.inner.borrow()), frozen @ PyFrozenSet => $op(&frozen.inner), - other => Err($vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class()))), + _ => Ok($vm.ctx.not_implemented()), ); }; } @@ -322,7 +330,7 @@ impl PySet { vm: &VirtualMachine, ) -> PyResult { PySet { - inner: RefCell::new(PySetInner::new(iterable, vm)?), + inner: RefCell::new(PySetInner::from_arg(iterable, vm)?), } .into_ref_with_type(vm, cls) } @@ -413,6 +421,16 @@ impl PySet { )) } + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issuperset(other, vm) + } + #[pymethod] fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow().isdisjoint(other, vm) @@ -539,16 +557,6 @@ impl PySet { Ok(zelf.as_object().clone()) } - #[pymethod] - fn issubset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.le(other, vm) - } - - #[pymethod] - fn issuperset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.ge(other, vm) - } - #[pymethod(name = "__hash__")] fn hash(&self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) @@ -564,7 +572,7 @@ impl PyFrozenSet { vm: &VirtualMachine, ) -> PyResult { PyFrozenSet { - inner: PySetInner::new(iterable, vm)?, + inner: PySetInner::from_arg(iterable, vm)?, } .into_ref_with_type(vm, cls) } @@ -655,6 +663,16 @@ impl PyFrozenSet { )) } + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issuperset(other, vm) + } + #[pymethod] fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.isdisjoint(other, vm) @@ -697,16 +715,6 @@ impl PyFrozenSet { }; Ok(vm.new_str(s)) } - - #[pymethod] - fn issubset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.le(other, vm) - } - - #[pymethod] - fn issuperset(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.ge(other, vm) - } } struct SetIterable { From ba4ac902ebb0d14b208f6c2a0a6313c25c1a2b69 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 04:18:20 +0900 Subject: [PATCH 632/658] PySet/PyFrozenSet doesn't call PyObject::new --- vm/src/obj/objset.rs | 140 +++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 86 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index ce97e861c..444d4cd14 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,8 +8,8 @@ use std::fmt; use crate::dictdatatype; use crate::function::OptionalArg; use crate::pyobject::{ - PyClassImpl, PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -280,7 +280,7 @@ impl PySetInner { Ok(()) } - fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { let temp_inner = self.copy(); self.clear(); for item in iterable.iter(vm)? { @@ -289,25 +289,25 @@ impl PySetInner { self.add(&obj, vm)?; } } - Ok(vm.get_none()) + Ok(()) } - fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { for item in iterable.iter(vm)? { self.content.delete_if_exists(vm, &item?)?; } - Ok(vm.get_none()) + Ok(()) } fn symmetric_difference_update( &mut self, iterable: PyIterable, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult<()> { for item in iterable.iter(vm)? { self.content.delete_or_insert(vm, &item?, ())?; } - Ok(vm.get_none()) + Ok(()) } } @@ -329,7 +329,7 @@ impl PySet { iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PySet { + Self { inner: RefCell::new(PySetInner::from_arg(iterable, vm)?), } .into_ref_with_type(vm, cls) @@ -341,8 +341,8 @@ impl PySet { } #[pymethod] - fn copy(&self, _vm: &VirtualMachine) -> PySet { - PySet { + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { inner: RefCell::new(self.inner.borrow().copy()), } } @@ -378,47 +378,31 @@ impl PySet { } #[pymethod] - fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().union(other, vm)?), - }, - PySet::class(vm), - None, - )) + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().union(other, vm)?), + }) } #[pymethod] - fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), - }, - PySet::class(vm), - None, - )) + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), + }) } #[pymethod] - fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().difference(other, vm)?), - }, - PySet::class(vm), - None, - )) + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), + }) } #[pymethod] - fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), - }, - PySet::class(vm), - None, - )) + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), + }) } #[pymethod] @@ -437,22 +421,22 @@ impl PySet { } #[pymethod(name = "__or__")] - fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } #[pymethod(name = "__and__")] - fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } #[pymethod(name = "__sub__")] - fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } #[pymethod(name = "__xor__")] - fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } @@ -571,7 +555,7 @@ impl PyFrozenSet { iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PyFrozenSet { + Self { inner: PySetInner::from_arg(iterable, vm)?, } .into_ref_with_type(vm, cls) @@ -583,8 +567,8 @@ impl PyFrozenSet { } #[pymethod] - fn copy(&self, _vm: &VirtualMachine) -> PyFrozenSet { - PyFrozenSet { + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { inner: self.inner.copy(), } } @@ -620,47 +604,31 @@ impl PyFrozenSet { } #[pymethod] - fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.union(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.union(other, vm)?, + }) } #[pymethod] - fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.intersection(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.intersection(other, vm)?, + }) } #[pymethod] - fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.difference(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.difference(other, vm)?, + }) } #[pymethod] - fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.symmetric_difference(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.symmetric_difference(other, vm)?, + }) } #[pymethod] @@ -679,22 +647,22 @@ impl PyFrozenSet { } #[pymethod(name = "__or__")] - fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } #[pymethod(name = "__and__")] - fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } #[pymethod(name = "__sub__")] - fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } #[pymethod(name = "__xor__")] - fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } From c349f77207d37429c7bb86720be2df21b5f47e00 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 May 2019 04:33:01 +0900 Subject: [PATCH 633/658] Add {set,frozenset}.__r*__ --- tests/snippets/set.py | 6 ++++++ vm/src/obj/objset.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 6e7413e56..50b8f97d8 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -103,6 +103,12 @@ assert set([1,2,3]).isdisjoint(set([5,6])) == True assert set([1,2,3]).isdisjoint(set([2,5,6])) == False assert set([1,2,3]).isdisjoint([5,6]) == True +assert_raises(TypeError, lambda: set() & []) +assert_raises(TypeError, lambda: set() | []) +assert_raises(TypeError, lambda: set() ^ []) +assert_raises(TypeError, lambda: set() + []) +assert_raises(TypeError, lambda: set() - []) + assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 444d4cd14..b7f0276ce 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -425,21 +425,41 @@ impl PySet { self.union(other.iterable, vm) } + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + #[pymethod(name = "__and__")] fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + #[pymethod(name = "__sub__")] fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + #[pymethod(name = "__xor__")] fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + #[pymethod(name = "__iter__")] fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) @@ -651,21 +671,41 @@ impl PyFrozenSet { self.union(other.iterable, vm) } + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + #[pymethod(name = "__and__")] fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + #[pymethod(name = "__sub__")] fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + #[pymethod(name = "__xor__")] fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + #[pymethod(name = "__iter__")] fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) From 80b1f545e28b4e7966c8ef10b76d26138353a9d3 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 29 May 2019 23:25:10 +0300 Subject: [PATCH 634/658] Add `itertools.islice` --- tests/snippets/stdlib_itertools.py | 29 +++++++ vm/src/stdlib/itertools.rs | 134 ++++++++++++++++++++++++++++- 2 files changed, 162 insertions(+), 1 deletion(-) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index c8921cb91..e8285fac9 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -154,3 +154,32 @@ with assertRaises(StopIteration): next(t) with assertRaises(StopIteration): next(t) + + +# itertools.islice tests + +def assert_matches_seq(it, seq): + assert list(it) == list(seq) + +i = itertools.islice + +it = i([1, 2, 3, 4, 5], 3) +assert_matches_seq(it, [1, 2, 3]) + +it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2], None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, None) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2, 3], 1, None, None) +assert_matches_seq(it, [2, 3]) + +it = i([1, 2, 3], None, 2, None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, 3) +assert_matches_seq(it, [1]) diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 7380a7677..a8c333701 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -3,14 +3,16 @@ use std::cmp::Ordering; use std::ops::{AddAssign, SubAssign}; use num_bigint::BigInt; +use num_traits::ToPrimitive; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbool; +use crate::obj::objint; use crate::obj::objint::{PyInt, PyIntRef}; use crate::obj::objiter::{call_next, get_iter, new_stop_iteration}; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[pyclass(name = "chain")] @@ -284,6 +286,133 @@ impl PyItertoolsTakewhile { } } +#[pyclass(name = "islice")] +#[derive(Debug)] +struct PyItertoolsIslice { + iterable: PyObjectRef, + cur: RefCell, + next: RefCell, + stop: Option, + step: usize, +} + +impl PyValue for PyItertoolsIslice { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "islice") + } +} + +fn pyobject_to_opt_usize(obj: PyObjectRef, vm: &VirtualMachine) -> Option { + let is_int = objtype::isinstance(&obj, &vm.ctx.int_type()); + if is_int { + objint::get_value(&obj).to_usize() + } else { + None + } +} + +#[pyimpl] +impl PyItertoolsIslice { + #[pymethod(name = "__new__")] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let (iter, start, stop, step) = match args.args.len() { + 0 | 1 => { + return Err(vm.new_type_error(format!( + "islice expected at least 2 arguments, got {}", + args.args.len() + ))); + } + + 2 => { + let (iter, stop): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + + (iter, 0usize, stop, 1usize) + } + _ => { + let (iter, start, stop, step): ( + PyObjectRef, + PyObjectRef, + PyObjectRef, + PyObjectRef, + ) = args.bind(vm)?; + + let start = if !start.is(&vm.get_none()) { + pyobject_to_opt_usize(start, &vm).ok_or_else(|| { + vm.new_value_error( + "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize.".to_string(), + ) + })? + } else { + 0usize + }; + + let step = if !step.is(&vm.get_none()) { + pyobject_to_opt_usize(step, &vm).ok_or_else(|| { + vm.new_value_error( + "Step for islice() must be a positive integer or None.".to_string(), + ) + })? + } else { + 1usize + }; + + (iter, start, stop, step) + } + }; + + let stop = if !stop.is(&vm.get_none()) { + Some(pyobject_to_opt_usize(stop, &vm).ok_or_else(|| { + vm.new_value_error( + "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." + .to_string(), + ) + })?) + } else { + None + }; + + let iter = get_iter(vm, &iter)?; + + Ok(PyItertoolsIslice { + iterable: iter, + cur: RefCell::new(0), + next: RefCell::new(start), + stop: stop, + step: step, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + while *self.cur.borrow() < *self.next.borrow() { + call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + } + + if let Some(stop) = self.stop { + if *self.cur.borrow() >= stop { + return Err(new_stop_iteration(vm)); + } + } + + let obj = call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + + // TODO is this overflow check required? attempts to copy CPython. + let (next, ovf) = (*self.next.borrow()).overflowing_add(self.step); + *self.next.borrow_mut() = if ovf { self.stop.unwrap() } else { next }; + + Ok(obj) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -300,11 +429,14 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let takewhile = ctx.new_class("takewhile", ctx.object()); PyItertoolsTakewhile::extend_class(ctx, &takewhile); + let islice = PyItertoolsIslice::make_class(ctx); + py_module!(vm, "itertools", { "chain" => chain, "count" => count, "repeat" => repeat, "starmap" => starmap, "takewhile" => takewhile, + "islice" => islice, }) } From 7ec3cef48e888e269d01e956e5acf63fa72300e7 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 29 May 2019 23:29:00 +0300 Subject: [PATCH 635/658] Add `Lib/reprlib.py` from CPython Generated with: `git -C ../cpython show 1bf9cc509326bc42cd8cb1650eb9bf64550d817e:Lib/reprlib.py > Lib/reprlib.py` Required for `functools`. --- Lib/reprlib.py | 161 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 Lib/reprlib.py diff --git a/Lib/reprlib.py b/Lib/reprlib.py new file mode 100644 index 000000000..616b3439b --- /dev/null +++ b/Lib/reprlib.py @@ -0,0 +1,161 @@ +"""Redo the builtin repr() (representation) but with limits on most sizes.""" + +__all__ = ["Repr", "repr", "recursive_repr"] + +import builtins +from itertools import islice +from _thread import get_ident + +def recursive_repr(fillvalue='...'): + 'Decorator to make a repr function return fillvalue for a recursive call' + + def decorating_function(user_function): + repr_running = set() + + def wrapper(self): + key = id(self), get_ident() + if key in repr_running: + return fillvalue + repr_running.add(key) + try: + result = user_function(self) + finally: + repr_running.discard(key) + return result + + # Can't use functools.wraps() here because of bootstrap issues + wrapper.__module__ = getattr(user_function, '__module__') + wrapper.__doc__ = getattr(user_function, '__doc__') + wrapper.__name__ = getattr(user_function, '__name__') + wrapper.__qualname__ = getattr(user_function, '__qualname__') + wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + return wrapper + + return decorating_function + +class Repr: + + def __init__(self): + self.maxlevel = 6 + self.maxtuple = 6 + self.maxlist = 6 + self.maxarray = 5 + self.maxdict = 4 + self.maxset = 6 + self.maxfrozenset = 6 + self.maxdeque = 6 + self.maxstring = 30 + self.maxlong = 40 + self.maxother = 30 + + def repr(self, x): + return self.repr1(x, self.maxlevel) + + def repr1(self, x, level): + typename = type(x).__name__ + if ' ' in typename: + parts = typename.split() + typename = '_'.join(parts) + if hasattr(self, 'repr_' + typename): + return getattr(self, 'repr_' + typename)(x, level) + else: + return self.repr_instance(x, level) + + def _repr_iterable(self, x, level, left, right, maxiter, trail=''): + n = len(x) + if level <= 0 and n: + s = '...' + else: + newlevel = level - 1 + repr1 = self.repr1 + pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] + if n > maxiter: pieces.append('...') + s = ', '.join(pieces) + if n == 1 and trail: right = trail + right + return '%s%s%s' % (left, s, right) + + def repr_tuple(self, x, level): + return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') + + def repr_list(self, x, level): + return self._repr_iterable(x, level, '[', ']', self.maxlist) + + def repr_array(self, x, level): + if not x: + return "array('%s')" % x.typecode + header = "array('%s', [" % x.typecode + return self._repr_iterable(x, level, header, '])', self.maxarray) + + def repr_set(self, x, level): + if not x: + return 'set()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, '{', '}', self.maxset) + + def repr_frozenset(self, x, level): + if not x: + return 'frozenset()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, 'frozenset({', '})', + self.maxfrozenset) + + def repr_deque(self, x, level): + return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) + + def repr_dict(self, x, level): + n = len(x) + if n == 0: return '{}' + if level <= 0: return '{...}' + newlevel = level - 1 + repr1 = self.repr1 + pieces = [] + for key in islice(_possibly_sorted(x), self.maxdict): + keyrepr = repr1(key, newlevel) + valrepr = repr1(x[key], newlevel) + pieces.append('%s: %s' % (keyrepr, valrepr)) + if n > self.maxdict: pieces.append('...') + s = ', '.join(pieces) + return '{%s}' % (s,) + + def repr_str(self, x, level): + s = builtins.repr(x[:self.maxstring]) + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = builtins.repr(x[:i] + x[len(x)-j:]) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_int(self, x, level): + s = builtins.repr(x) # XXX Hope this isn't too slow... + if len(s) > self.maxlong: + i = max(0, (self.maxlong-3)//2) + j = max(0, self.maxlong-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_instance(self, x, level): + try: + s = builtins.repr(x) + # Bugs in x.__repr__() can cause arbitrary + # exceptions -- then make up something + except Exception: + return '<%s instance at %#x>' % (x.__class__.__name__, id(x)) + if len(s) > self.maxother: + i = max(0, (self.maxother-3)//2) + j = max(0, self.maxother-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + +def _possibly_sorted(x): + # Since not all sequences of items can be sorted and comparison + # functions may raise arbitrary exceptions, return an unsorted + # sequence in that case. + try: + return sorted(x) + except Exception: + return list(x) + +aRepr = Repr() +repr = aRepr.repr From 11892816a765d87ed7402aa1b35b5fcc5b3a63d3 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 29 May 2019 16:11:22 -0500 Subject: [PATCH 636/658] Don't check env variables on wasm --- vm/src/sysmodule.rs | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index dd52a189a..f2f123c71 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -54,27 +54,31 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR "cache_tag" => ctx.new_str("rustpython-01".to_string()), }); - fn get_paths<'a>( - paths: &'a Option, - ) -> impl Iterator + 'a { - match paths { - Some(paths) => env::split_paths(paths), - None => env::split_paths(""), + let path_list = if cfg!(target_arch = "wasm32") { + vec![] + } else { + fn get_paths<'a>( + paths: &'a Option, + ) -> impl Iterator + 'a { + match paths { + Some(paths) => env::split_paths(paths), + None => env::split_paths(""), + } } - } - let rustpy_path = env::var_os("RUSTPYTHONPATH"); - let py_path = env::var_os("PYTHONPATH"); - let path_list = get_paths(&rustpy_path) - .chain(get_paths(&py_path)) - .map(|path| { - ctx.new_str( - path.to_str() - .expect("PYTHONPATH isn't valid unicode") - .to_string(), - ) - }) - .collect(); + let rustpy_path = env::var_os("RUSTPYTHONPATH"); + let py_path = env::var_os("PYTHONPATH"); + get_paths(&rustpy_path) + .chain(get_paths(&py_path)) + .map(|path| { + ctx.new_str( + path.to_str() + .expect("PYTHONPATH isn't valid unicode") + .to_string(), + ) + }) + .collect() + }; let path = ctx.new_list(path_list); let platform = if cfg!(target_os = "linux") { From c78dc520ad929b472b06d35a2b62ac21e63ca426 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 29 May 2019 16:53:13 -0500 Subject: [PATCH 637/658] Don't clone PathBufs --- vm/src/sysmodule.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index f2f123c71..3c5a58f13 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -57,24 +57,20 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR let path_list = if cfg!(target_arch = "wasm32") { vec![] } else { - fn get_paths<'a>( - paths: &'a Option, - ) -> impl Iterator + 'a { - match paths { - Some(paths) => env::split_paths(paths), - None => env::split_paths(""), - } - } + let get_paths = |paths| match paths { + Some(paths) => env::split_paths(paths), + None => env::split_paths(""), + }; let rustpy_path = env::var_os("RUSTPYTHONPATH"); let py_path = env::var_os("PYTHONPATH"); - get_paths(&rustpy_path) - .chain(get_paths(&py_path)) + get_paths(rustpy_path.as_ref()) + .chain(get_paths(py_path.as_ref())) .map(|path| { ctx.new_str( - path.to_str() - .expect("PYTHONPATH isn't valid unicode") - .to_string(), + path.into_os_string() + .into_string() + .expect("PYTHONPATH isn't valid unicode"), ) }) .collect() From 1dc2d097135ed5d66a8595db8b97bb1bf65228aa Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 29 May 2019 22:46:12 -0500 Subject: [PATCH 638/658] Add __setattr__ to weakproxy --- vm/src/obj/objweakproxy.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs index 735670f4e..5d4f68d9d 100644 --- a/vm/src/obj/objweakproxy.rs +++ b/vm/src/obj/objweakproxy.rs @@ -1,9 +1,10 @@ use super::objweakref::PyWeak; use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass] #[derive(Debug)] pub struct PyWeakProxy { weak: PyWeak, @@ -17,21 +18,27 @@ impl PyValue for PyWeakProxy { pub type PyWeakProxyRef = PyRef; -impl PyWeakProxyRef { - // TODO callbacks +#[pyimpl] +impl PyWeakProxy { + // TODO: callbacks + #[pymethod(name = "__new__")] fn create( cls: PyClassRef, referent: PyObjectRef, - _callback: OptionalArg, + callback: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { + if callback.is_present() { + panic!("Passed a callback to weakproxy, but weakproxy does not yet support proxies."); + } PyWeakProxy { weak: PyWeak::downgrade(&referent), } .into_ref_with_type(vm, cls) } - fn getattr(self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__getattr__")] + fn getattr(&self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self.weak.upgrade() { Some(obj) => vm.get_attribute(obj, attr_name), None => Err(vm.new_exception( @@ -40,11 +47,19 @@ impl PyWeakProxyRef { )), } } + + #[pymethod(name = "__setattr__")] + fn setattr(&self, attr_name: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.set_attr(&obj, attr_name, value), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } } pub fn init(context: &PyContext) { - extend_class!(context, &context.weakproxy_type, { - "__new__" => context.new_rustfunc(PyWeakProxyRef::create), - "__getattr__" => context.new_rustfunc(PyWeakProxyRef::getattr), - }); + PyWeakProxy::extend_class(&context, &context.weakproxy_type); } From d2757a2680c323ebb74d4b583bb614834e9fc1ff Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 30 May 2019 21:22:30 +1200 Subject: [PATCH 639/658] Fix merge issue with PyTuple::fast_getitem --- vm/src/obj/objtuple.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 829a992de..03596fbd2 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -39,7 +39,7 @@ impl PyValue for PyTuple { impl PyTuple { pub fn fast_getitem(&self, idx: usize) -> PyObjectRef { - self.elements.borrow()[idx].clone() + self.elements[idx].clone() } } From 9e88b808dde3a51b828849f280dca9da1ff4624c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 30 May 2019 18:50:46 +0300 Subject: [PATCH 640/658] Add _imp.is_frozen --- tests/snippets/imp.py | 3 +++ vm/src/stdlib/imp.rs | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py index 45332a9b9..a9d827b63 100644 --- a/tests/snippets/imp.py +++ b/tests/snippets/imp.py @@ -3,3 +3,6 @@ import _imp assert _imp.is_builtin("time") == True assert _imp.is_builtin("os") == False assert _imp.is_builtin("not existing module") == False + +assert _imp.is_frozen("__hello__") == True +assert _imp.is_frozen("os") == False diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index f5d889852..5b249a135 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -25,6 +25,10 @@ fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { vm.stdlib_inits.borrow().contains_key(name.as_str()) } +fn imp_is_frozen(name: PyStringRef, vm: &VirtualMachine) -> bool { + vm.frozen.borrow().contains_key(name.as_str()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -33,6 +37,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "release_lock" => ctx.new_rustfunc(imp_release_lock), "lock_held" => ctx.new_rustfunc(imp_lock_held), "is_builtin" => ctx.new_rustfunc(imp_is_builtin), + "is_frozen" => ctx.new_rustfunc(imp_is_frozen), }); module From 39c63ef1395da891b824ac88dccdb92e4e0c0272 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Thu, 30 May 2019 23:03:58 -0500 Subject: [PATCH 641/658] Update Lib/operator.py and add Lib/heapq.py from cpython --- Lib/heapq.py | 601 ++++++++++++++++++++++++++++++++++++++++++++++++ Lib/operator.py | 455 +++++++++++++++++++++++++++++++++++- 2 files changed, 1053 insertions(+), 3 deletions(-) create mode 100644 Lib/heapq.py diff --git a/Lib/heapq.py b/Lib/heapq.py new file mode 100644 index 000000000..0e3555cf9 --- /dev/null +++ b/Lib/heapq.py @@ -0,0 +1,601 @@ +"""Heap queue algorithm (a.k.a. priority queue). + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +Usage: + +heap = [] # creates an empty heap +heappush(heap, item) # pushes a new item on the heap +item = heappop(heap) # pops the smallest item from the heap +item = heap[0] # smallest item on the heap without popping it +heapify(x) # transforms list into a heap, in-place, in linear time +item = heapreplace(heap, item) # pops and returns smallest item, and adds + # new item; the heap size is unchanged + +Our API differs from textbook heap algorithms as follows: + +- We use 0-based indexing. This makes the relationship between the + index for a node and the indexes for its children slightly less + obvious, but is more suitable since Python uses 0-based indexing. + +- Our heappop() method returns the smallest item, not the largest. + +These two make it possible to view the heap as a regular Python list +without surprises: heap[0] is the smallest item, and heap.sort() +maintains the heap invariant! +""" + +# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger + +__about__ = """Heap queues + +[explanation by François Pinard] + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +The strange invariant above is meant to be an efficient memory +representation for a tournament. The numbers below are `k', not a[k]: + + 0 + + 1 2 + + 3 4 5 6 + + 7 8 9 10 11 12 13 14 + + 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + +In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +a usual binary tournament we see in sports, each cell is the winner +over the two cells it tops, and we can trace the winner down the tree +to see all opponents s/he had. However, in many computer applications +of such tournaments, we do not need to trace the history of a winner. +To be more memory efficient, when a winner is promoted, we try to +replace it by something else at a lower level, and the rule becomes +that a cell and the two cells it tops contain three different items, +but the top cell "wins" over the two topped cells. + +If this heap invariant is protected at all time, index 0 is clearly +the overall winner. The simplest algorithmic way to remove it and +find the "next" winner is to move some loser (let's say cell 30 in the +diagram above) into the 0 position, and then percolate this new 0 down +the tree, exchanging values, until the invariant is re-established. +This is clearly logarithmic on the total number of items in the tree. +By iterating over all items, you get an O(n ln n) sort. + +A nice feature of this sort is that you can efficiently insert new +items while the sort is going on, provided that the inserted items are +not "better" than the last 0'th element you extracted. This is +especially useful in simulation contexts, where the tree holds all +incoming events, and the "win" condition means the smallest scheduled +time. When an event schedule other events for execution, they are +scheduled into the future, so they can easily go into the heap. So, a +heap is a good structure for implementing schedulers (this is what I +used for my MIDI sequencer :-). + +Various structures for implementing schedulers have been extensively +studied, and heaps are good for this, as they are reasonably speedy, +the speed is almost constant, and the worst case is not much different +than the average case. However, there are other representations which +are more efficient overall, yet the worst cases might be terrible. + +Heaps are also very useful in big disk sorts. You most probably all +know that a big sort implies producing "runs" (which are pre-sorted +sequences, which size is usually related to the amount of CPU memory), +followed by a merging passes for these runs, which merging is often +very cleverly organised[1]. It is very important that the initial +sort produces the longest runs possible. Tournaments are a good way +to that. If, using all the memory available to hold a tournament, you +replace and percolate items that happen to fit the current run, you'll +produce runs which are twice the size of the memory for random input, +and much better for input fuzzily ordered. + +Moreover, if you output the 0'th item on disk and get an input which +may not fit in the current tournament (because the value "wins" over +the last output value), it cannot fit in the heap, so the size of the +heap decreases. The freed memory could be cleverly reused immediately +for progressively building a second heap, which grows at exactly the +same rate the first heap is melting. When the first heap completely +vanishes, you switch heaps and start a new run. Clever and quite +effective! + +In a word, heaps are useful memory structures to know. I use them in +a few applications, and I think it is good to keep a `heap' module +around. :-) + +-------------------- +[1] The disk balancing algorithms which are current, nowadays, are +more annoying than clever, and this is a consequence of the seeking +capabilities of the disks. On devices which cannot seek, like big +tape drives, the story was quite different, and one had to be very +clever to ensure (far in advance) that each tape movement will be the +most effective possible (that is, will best participate at +"progressing" the merge). Some tapes were even able to read +backwards, and this was also used to avoid the rewinding time. +Believe me, real good tape sorts were quite spectacular to watch! +From all times, sorting has always been a Great Art! :-) +""" + +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', + 'nlargest', 'nsmallest', 'heappushpop'] + +def heappush(heap, item): + """Push item onto heap, maintaining the heap invariant.""" + heap.append(item) + _siftdown(heap, 0, len(heap)-1) + +def heappop(heap): + """Pop the smallest item off the heap, maintaining the heap invariant.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup(heap, 0) + return returnitem + return lastelt + +def heapreplace(heap, item): + """Pop and return the current smallest value, and add the new item. + + This is more efficient than heappop() followed by heappush(), and can be + more appropriate when using a fixed-size heap. Note that the value + returned may be larger than item! That constrains reasonable uses of + this routine unless written as part of a conditional replacement: + + if item > heap[0]: + item = heapreplace(heap, item) + """ + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup(heap, 0) + return returnitem + +def heappushpop(heap, item): + """Fast version of a heappush followed by a heappop.""" + if heap and heap[0] < item: + item, heap[0] = heap[0], item + _siftup(heap, 0) + return item + +def heapify(x): + """Transform list into a heap, in-place, in O(len(x)) time.""" + n = len(x) + # Transform bottom-up. The largest index there's any point to looking at + # is the largest with a child index in-range, so must have 2*i + 1 < n, + # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so + # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is + # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. + for i in reversed(range(n//2)): + _siftup(x, i) + +def _heappop_max(heap): + """Maxheap version of a heappop.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup_max(heap, 0) + return returnitem + return lastelt + +def _heapreplace_max(heap, item): + """Maxheap version of a heappop followed by a heappush.""" + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup_max(heap, 0) + return returnitem + +def _heapify_max(x): + """Transform list into a maxheap, in-place, in O(len(x)) time.""" + n = len(x) + for i in reversed(range(n//2)): + _siftup_max(x, i) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +def _siftdown(heap, startpos, pos): + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if newitem < parent: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +# The child indices of heap index pos are already heaps, and we want to make +# a heap at index pos too. We do this by bubbling the smaller child of +# pos up (and so on with that child's children, etc) until hitting a leaf, +# then using _siftdown to move the oddball originally at index pos into place. +# +# We *could* break out of the loop as soon as we find a pos where newitem <= +# both its children, but turns out that's not a good idea, and despite that +# many books write the algorithm that way. During a heap pop, the last array +# element is sifted in, and that tends to be large, so that comparing it +# against values starting from the root usually doesn't pay (= usually doesn't +# get us out of the loop early). See Knuth, Volume 3, where this is +# explained and quantified in an exercise. +# +# Cutting the # of comparisons is important, since these routines have no +# way to extract "the priority" from an array element, so that intelligence +# is likely to be hiding in custom comparison methods, or in array elements +# storing (priority, record) tuples. Comparisons are thus potentially +# expensive. +# +# On random arrays of length 1000, making this change cut the number of +# comparisons made by heapify() a little, and those made by exhaustive +# heappop() a lot, in accord with theory. Here are typical results from 3 +# runs (3 just to demonstrate how small the variance is): +# +# Compares needed by heapify Compares needed by 1000 heappops +# -------------------------- -------------------------------- +# 1837 cut to 1663 14996 cut to 8680 +# 1855 cut to 1659 14966 cut to 8678 +# 1847 cut to 1660 15024 cut to 8703 +# +# Building the heap by using heappush() 1000 times instead required +# 2198, 2148, and 2219 compares: heapify() is more efficient, when +# you can use it. +# +# The total compares needed by list.sort() on the same lists were 8627, +# 8627, and 8632 (this should be compared to the sum of heapify() and +# heappop() compares): list.sort() is (unsurprisingly!) more efficient +# for sorting. + +def _siftup(heap, pos): + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[childpos] < heap[rightpos]: + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown(heap, startpos, pos) + +def _siftdown_max(heap, startpos, pos): + 'Maxheap variant of _siftdown' + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if parent < newitem: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +def _siftup_max(heap, pos): + 'Maxheap variant of _siftup' + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the larger child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of larger child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[rightpos] < heap[childpos]: + childpos = rightpos + # Move the larger child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown_max(heap, startpos, pos) + +def merge(*iterables, key=None, reverse=False): + '''Merge multiple sorted inputs into a single sorted output. + + Similar to sorted(itertools.chain(*iterables)) but returns a generator, + does not pull the data into memory all at once, and assumes that each of + the input streams is already sorted (smallest to largest). + + >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) + [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] + + If *key* is not None, applies a key function to each element to determine + its sort order. + + >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) + ['dog', 'cat', 'fish', 'horse', 'kangaroo'] + + ''' + + h = [] + h_append = h.append + + if reverse: + _heapify = _heapify_max + _heappop = _heappop_max + _heapreplace = _heapreplace_max + direction = -1 + else: + _heapify = heapify + _heappop = heappop + _heapreplace = heapreplace + direction = 1 + + if key is None: + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + h_append([next(), order * direction, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + value, order, next = s = h[0] + yield value + s[0] = next() # raises StopIteration when exhausted + _heapreplace(h, s) # restore heap condition + except StopIteration: + _heappop(h) # remove empty iterator + if h: + # fast case when only a single iterator remains + value, order, next = h[0] + yield value + yield from next.__self__ + return + + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + value = next() + h_append([key(value), order * direction, value, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + key_value, order, value, next = s = h[0] + yield value + value = next() + s[0] = key(value) + s[2] = value + _heapreplace(h, s) + except StopIteration: + _heappop(h) + if h: + key_value, order, value, next = h[0] + yield value + yield from next.__self__ + + +# Algorithm notes for nlargest() and nsmallest() +# ============================================== +# +# Make a single pass over the data while keeping the k most extreme values +# in a heap. Memory consumption is limited to keeping k values in a list. +# +# Measured performance for random inputs: +# +# number of comparisons +# n inputs k-extreme values (average of 5 trials) % more than min() +# ------------- ---------------- --------------------- ----------------- +# 1,000 100 3,317 231.7% +# 10,000 100 14,046 40.5% +# 100,000 100 105,749 5.7% +# 1,000,000 100 1,007,751 0.8% +# 10,000,000 100 10,009,401 0.1% +# +# Theoretical number of comparisons for k smallest of n random inputs: +# +# Step Comparisons Action +# ---- -------------------------- --------------------------- +# 1 1.66 * k heapify the first k-inputs +# 2 n - k compare remaining elements to top of heap +# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap +# 4 k * lg2(k) - (k/2) final sort of the k most extreme values +# +# Combining and simplifying for a rough estimate gives: +# +# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) +# +# Computing the number of comparisons for step 3: +# ----------------------------------------------- +# * For the i-th new value from the iterable, the probability of being in the +# k most extreme values is k/i. For example, the probability of the 101st +# value seen being in the 100 most extreme values is 100/101. +# * If the value is a new extreme value, the cost of inserting it into the +# heap is 1 + log(k, 2). +# * The probability times the cost gives: +# (k/i) * (1 + log(k, 2)) +# * Summing across the remaining n-k elements gives: +# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) +# * This reduces to: +# (H(n) - H(k)) * k * (1 + log(k, 2)) +# * Where H(n) is the n-th harmonic number estimated by: +# gamma = 0.5772156649 +# H(n) = log(n, e) + gamma + 1 / (2 * n) +# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence +# * Substituting the H(n) formula: +# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) +# +# Worst-case for step 3: +# ---------------------- +# In the worst case, the input data is reversed sorted so that every new element +# must be inserted in the heap: +# +# comparisons = 1.66 * k + log(k, 2) * (n - k) +# +# Alternative Algorithms +# ---------------------- +# Other algorithms were not used because they: +# 1) Took much more auxiliary memory, +# 2) Made multiple passes over the data. +# 3) Made more comparisons in common cases (small k, large n, semi-random input). +# See the more detailed comparison of approach at: +# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest + +def nsmallest(n, iterable, key=None): + """Find the n smallest elements in a dataset. + + Equivalent to: sorted(iterable, key=key)[:n] + """ + + # Short-cut for n==1 is to use min() + if n == 1: + it = iter(iterable) + sentinel = object() + result = min(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + # put the range(n) first so that zip() doesn't + # consume one too many elements from the iterator + result = [(elem, i) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + if elem < top: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order += 1 + result.sort() + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + k = key(elem) + if k < top: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order += 1 + result.sort() + return [elem for (k, order, elem) in result] + +def nlargest(n, iterable, key=None): + """Find the n largest elements in a dataset. + + Equivalent to: sorted(iterable, key=key, reverse=True)[:n] + """ + + # Short-cut for n==1 is to use max() + if n == 1: + it = iter(iterable) + sentinel = object() + result = max(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key, reverse=True)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + if top < elem: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + k = key(elem) + if top < k: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (k, order, elem) in result] + +# If available, use C implementation +try: + from _heapq import * +except ImportError: + pass +try: + from _heapq import _heapreplace_max +except ImportError: + pass +try: + from _heapq import _heapify_max +except ImportError: + pass +try: + from _heapq import _heappop_max +except ImportError: + pass + + +if __name__ == "__main__": + + import doctest + print(doctest.testmod()) diff --git a/Lib/operator.py b/Lib/operator.py index f21920064..0e2e53efc 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -1,15 +1,464 @@ +""" +Operator Interface + +This module exports a set of functions corresponding to the intrinsic +operators of Python. For example, operator.add(x, y) is equivalent +to the expression x+y. The function names are those used for special +methods; variants without leading and trailing '__' are also provided +for convenience. + +This is the pure Python implementation of the module. +""" + +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', + 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', + 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', + 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', + 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', + 'setitem', 'sub', 'truediv', 'truth', 'xor'] + +from builtins import abs as _abs -# Comparison operations: +# Comparison Operations *******************************************************# + +def lt(a, b): + "Same as a < b." + return a < b + +def le(a, b): + "Same as a <= b." + return a <= b def eq(a, b): + "Same as a == b." return a == b - def ne(a, b): + "Same as a != b." return a != b - def ge(a, b): + "Same as a >= b." return a >= b +def gt(a, b): + "Same as a > b." + return a > b + +# Logical Operations **********************************************************# + +def not_(a): + "Same as not a." + return not a + +def truth(a): + "Return True if a is true, False otherwise." + return True if a else False + +def is_(a, b): + "Same as a is b." + return a is b + +def is_not(a, b): + "Same as a is not b." + return a is not b + +# Mathematical/Bitwise Operations *********************************************# + +def abs(a): + "Same as abs(a)." + return _abs(a) + +def add(a, b): + "Same as a + b." + return a + b + +def and_(a, b): + "Same as a & b." + return a & b + +def floordiv(a, b): + "Same as a // b." + return a // b + +def index(a): + "Same as a.__index__()." + return a.__index__() + +def inv(a): + "Same as ~a." + return ~a +invert = inv + +def lshift(a, b): + "Same as a << b." + return a << b + +def mod(a, b): + "Same as a % b." + return a % b + +def mul(a, b): + "Same as a * b." + return a * b + +def matmul(a, b): + "Same as a @ b." + return a @ b + +def neg(a): + "Same as -a." + return -a + +def or_(a, b): + "Same as a | b." + return a | b + +def pos(a): + "Same as +a." + return +a + +def pow(a, b): + "Same as a ** b." + return a ** b + +def rshift(a, b): + "Same as a >> b." + return a >> b + +def sub(a, b): + "Same as a - b." + return a - b + +def truediv(a, b): + "Same as a / b." + return a / b + +def xor(a, b): + "Same as a ^ b." + return a ^ b + +# Sequence Operations *********************************************************# + +def concat(a, b): + "Same as a + b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + return a + b + +def contains(a, b): + "Same as b in a (note reversed operands)." + return b in a + +def countOf(a, b): + "Return the number of times b occurs in a." + count = 0 + for i in a: + if i == b: + count += 1 + return count + +def delitem(a, b): + "Same as del a[b]." + del a[b] + +def getitem(a, b): + "Same as a[b]." + return a[b] + +def indexOf(a, b): + "Return the first index of b in a." + for i, j in enumerate(a): + if j == b: + return i + else: + raise ValueError('sequence.index(x): x not in sequence') + +def setitem(a, b, c): + "Same as a[b] = c." + a[b] = c + +def length_hint(obj, default=0): + """ + Return an estimate of the number of items in obj. + This is useful for presizing containers when building from an iterable. + + If the object supports len(), the result will be exact. Otherwise, it may + over- or under-estimate by an arbitrary amount. The result will be an + integer >= 0. + """ + if not isinstance(default, int): + msg = ("'%s' object cannot be interpreted as an integer" % + type(default).__name__) + raise TypeError(msg) + + try: + return len(obj) + except TypeError: + pass + + try: + hint = type(obj).__length_hint__ + except AttributeError: + return default + + try: + val = hint(obj) + except TypeError: + return default + if val is NotImplemented: + return default + if not isinstance(val, int): + msg = ('__length_hint__ must be integer, not %s' % + type(val).__name__) + raise TypeError(msg) + if val < 0: + msg = '__length_hint__() should return >= 0' + raise ValueError(msg) + return val + +# Generalized Lookup Objects **************************************************# + +class attrgetter: + """ + Return a callable object that fetches the given attribute(s) from its operand. + After f = attrgetter('name'), the call f(r) returns r.name. + After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). + After h = attrgetter('name.first', 'name.last'), the call h(r) returns + (r.name.first, r.name.last). + """ + __slots__ = ('_attrs', '_call') + + def __init__(self, attr, *attrs): + if not attrs: + if not isinstance(attr, str): + raise TypeError('attribute name must be a string') + self._attrs = (attr,) + names = attr.split('.') + def func(obj): + for name in names: + obj = getattr(obj, name) + return obj + self._call = func + else: + self._attrs = (attr,) + attrs + getters = tuple(map(attrgetter, self._attrs)) + def func(obj): + return tuple(getter(obj) for getter in getters) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__qualname__, + ', '.join(map(repr, self._attrs))) + + def __reduce__(self): + return self.__class__, self._attrs + +class itemgetter: + """ + Return a callable object that fetches the given item(s) from its operand. + After f = itemgetter(2), the call f(r) returns r[2]. + After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) + """ + __slots__ = ('_items', '_call') + + def __init__(self, item, *items): + if not items: + self._items = (item,) + def func(obj): + return obj[item] + self._call = func + else: + self._items = items = (item,) + items + def func(obj): + return tuple(obj[i] for i in items) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(map(repr, self._items))) + + def __reduce__(self): + return self.__class__, self._items + +class methodcaller: + """ + Return a callable object that calls the given method on its operand. + After f = methodcaller('name'), the call f(r) returns r.name(). + After g = methodcaller('name', 'date', foo=1), the call g(r) returns + r.name('date', foo=1). + """ + __slots__ = ('_name', '_args', '_kwargs') + + def __init__(*args, **kwargs): + if len(args) < 2: + msg = "methodcaller needs at least one argument, the method name" + raise TypeError(msg) + self = args[0] + self._name = args[1] + if not isinstance(self._name, str): + raise TypeError('method name must be a string') + self._args = args[2:] + self._kwargs = kwargs + + def __call__(self, obj): + return getattr(obj, self._name)(*self._args, **self._kwargs) + + def __repr__(self): + args = [repr(self._name)] + args.extend(map(repr, self._args)) + args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(args)) + + def __reduce__(self): + if not self._kwargs: + return self.__class__, (self._name,) + self._args + else: + from functools import partial + return partial(self.__class__, self._name, **self._kwargs), self._args + + +# In-place Operations *********************************************************# + +def iadd(a, b): + "Same as a += b." + a += b + return a + +def iand(a, b): + "Same as a &= b." + a &= b + return a + +def iconcat(a, b): + "Same as a += b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + a += b + return a + +def ifloordiv(a, b): + "Same as a //= b." + a //= b + return a + +def ilshift(a, b): + "Same as a <<= b." + a <<= b + return a + +def imod(a, b): + "Same as a %= b." + a %= b + return a + +def imul(a, b): + "Same as a *= b." + a *= b + return a + +def imatmul(a, b): + "Same as a @= b." + a @= b + return a + +def ior(a, b): + "Same as a |= b." + a |= b + return a + +def ipow(a, b): + "Same as a **= b." + a **=b + return a + +def irshift(a, b): + "Same as a >>= b." + a >>= b + return a + +def isub(a, b): + "Same as a -= b." + a -= b + return a + +def itruediv(a, b): + "Same as a /= b." + a /= b + return a + +def ixor(a, b): + "Same as a ^= b." + a ^= b + return a + + +try: + from _operator import * +except ImportError: + pass +else: + from _operator import __doc__ + +# All of these "__func__ = func" assignments have to happen after importing +# from _operator to make sure they're set to the right function +__lt__ = lt +__le__ = le +__eq__ = eq +__ne__ = ne +__ge__ = ge +__gt__ = gt +__not__ = not_ +__abs__ = abs +__add__ = add +__and__ = and_ +__floordiv__ = floordiv +__index__ = index +__inv__ = inv +__invert__ = invert +__lshift__ = lshift +__mod__ = mod +__mul__ = mul +__matmul__ = matmul +__neg__ = neg +__or__ = or_ +__pos__ = pos +__pow__ = pow +__rshift__ = rshift +__sub__ = sub +__truediv__ = truediv +__xor__ = xor +__concat__ = concat +__contains__ = contains +__delitem__ = delitem +__getitem__ = getitem +__setitem__ = setitem +__iadd__ = iadd +__iand__ = iand +__iconcat__ = iconcat +__ifloordiv__ = ifloordiv +__ilshift__ = ilshift +__imod__ = imod +__imul__ = imul +__imatmul__ = imatmul +__ior__ = ior +__ipow__ = ipow +__irshift__ = irshift +__isub__ = isub +__itruediv__ = itruediv +__ixor__ = ixor From e452d5523e8f029d43c2f13253d3b594c4a43174 Mon Sep 17 00:00:00 2001 From: Shitong Wen Date: Fri, 31 May 2019 14:16:31 +0800 Subject: [PATCH 642/658] use crate crc to replace crc table --- Cargo.lock | 16 ++++++++++++++ vm/Cargo.toml | 1 + vm/src/stdlib/binascii.rs | 46 +++++---------------------------------- 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 272786ef3..26521b259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,11 @@ dependencies = [ "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bumpalo" version = "2.3.0" @@ -195,6 +200,14 @@ dependencies = [ "python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "diff" version = "0.1.11" @@ -886,6 +899,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1429,6 +1443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49665c62e0e700857531fa5d3763e91b539ff1abeebd56808d378b495870d60d" "checksum block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d75255892aeb580d3c566f213a2b6fdc1c66667839f45719ee1d30ebf2aea591" +"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum bumpalo 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f6a6bc2ba935b1e2f810787ceba8dfe86cbc164cee55925cae715541186673c4" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" @@ -1439,6 +1454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b489034e723e7f5109fecd19b719e664f89ef925be785885252469e9822fa940" +"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" "checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5ee8e81c4..91c698822 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -30,6 +30,7 @@ lexical = "2.0.0" itertools = "^0.8.0" hexf = "0.1.0" indexmap = "1.0.2" +crc = "^1.0.0" # TODO: release and publish to crates.io [dependencies.unicode-casing] diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index f0c6cc8ca..049f69dfe 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -3,6 +3,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use crc::{crc32, Hasher32}; use num_traits::ToPrimitive; fn hex_nibble(n: u8) -> u8 { @@ -58,40 +59,6 @@ fn binascii_unhexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(unhex)) } -static CRC32_TABLE: [u32; 256] = [ - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, - 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, - 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, - 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, - 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, - 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, - 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, - 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, - 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, - 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, - 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, - 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, - 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, - 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, - 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, - 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, -]; fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -106,13 +73,10 @@ fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Some(value) => objint::get_value(&value).to_u32().unwrap(), }; - crc = !crc; - for b in bytes.iter() { - let b = *b as u32; - crc = CRC32_TABLE[((crc ^ b) & 0xff) as usize] ^ (crc >> 8); - } - crc = crc ^ 0xffffffff; - Ok(vm.ctx.new_int(crc)) + let mut digest = crc32::Digest::new_with_initial(crc32::IEEE, crc); + digest.write(&bytes); + + Ok(vm.ctx.new_int(digest.sum32())) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From 1949239592d6a02870b00c6fe63120863472ede6 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Fri, 31 May 2019 21:56:10 +0300 Subject: [PATCH 643/658] cleanups, type fixes and renames --- src/main.rs | 2 +- vm/src/frame.rs | 26 +++++++++++++------------- vm/src/stdlib/binascii.rs | 2 +- vm/src/symboltable.rs | 16 +++++++--------- vm/src/vm.rs | 23 +++++++++++------------ 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2100226c2..cd20462ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -159,7 +159,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com Ok(value) => { // Save non-None values as "_" - use rustpython_vm::pyobject::{IdProtocol, IntoPyObject, ItemProtocol}; + use rustpython_vm::pyobject::{IdProtocol, IntoPyObject}; if !value.is(&vm.get_none()) { let key = objstr::PyString::from("_").into_pyobject(vm); diff --git a/vm/src/frame.rs b/vm/src/frame.rs index c5e3b53ca..fef36bfec 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -124,15 +124,15 @@ impl Scope { self.locals.iter().next().cloned() } - pub fn child_scope_with_locals(&self, locals: PyDictRef) -> Scope { + pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope { Scope { locals: self.locals.clone().insert(locals), globals: self.globals.clone(), } } - pub fn child_scope(&self, ctx: &PyContext) -> Scope { - self.child_scope_with_locals(ctx.new_dict()) + pub fn new_child_scope(&self, ctx: &PyContext) -> Scope { + self.new_child_scope_with_locals(ctx.new_dict()) } } @@ -651,22 +651,22 @@ impl Frame { // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm + let func_obj = vm .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); let name = qualified_name.value.split('.').next_back().unwrap(); - vm.set_attr(&obj, "__name__", vm.new_str(name.to_string()))?; - vm.set_attr(&obj, "__qualname__", qualified_name)?; + vm.set_attr(&func_obj, "__name__", vm.new_str(name.to_string()))?; + vm.set_attr(&func_obj, "__qualname__", qualified_name)?; let module = self .scope .globals .get_item_option("__name__", vm)? .unwrap_or_else(|| vm.get_none()); - vm.set_attr(&obj, "__module__", module)?; - vm.set_attr(&obj, "__annotations__", annotations)?; + vm.set_attr(&func_obj, "__module__", module)?; + vm.set_attr(&func_obj, "__annotations__", annotations)?; - self.push_value(obj); + self.push_value(func_obj); Ok(None) } bytecode::Instruction::CallFunction { typ } => { @@ -1318,14 +1318,14 @@ impl Frame { || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { Ok(exception) - } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { - if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { - let exception = vm.new_empty_exception(exception)?; + } else if let Ok(exc_type) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exc_type, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exc_type)?; Ok(exception) } else { let msg = format!( "Can only raise BaseException derived types, not {}", - exception + exc_type ); Err(vm.new_type_error(msg)) } diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 049f69dfe..ff748aa01 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -68,7 +68,7 @@ fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { ); let bytes = objbytes::get_value(data); - let mut crc = match value { + let crc = match value { None => 0u32, Some(value) => objint::get_value(&value).to_u32().unwrap(), }; diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index ea9ae52a0..24407c751 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -16,7 +16,8 @@ pub fn make_symbol_table(program: &ast::Program) -> Result, @@ -101,7 +102,6 @@ fn analyze_symbol_table( symbol_scope: &SymbolScope, parent_symbol_scope: Option<&SymbolScope>, ) -> SymbolTableResult { - // println!("Analyzing {:?}, parent={:?} symbols={:?}", symbol_scope, parent_symbol_scope, symbol_scope.symbols); // Analyze sub scopes: for sub_scope in &symbol_scope.sub_scopes { analyze_symbol_table(&sub_scope, Some(symbol_scope))?; @@ -157,7 +157,6 @@ impl SymbolTableBuilder { } pub fn enter_scope(&mut self) { - // Create new scope and push into scope stack. let scope = SymbolScope::new(); self.scopes.push(scope); } @@ -494,7 +493,7 @@ impl SymbolTableBuilder { } fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { - // Evaulate eventual default parameters: + // Evaluate eventual default parameters: self.scan_expressions(&args.defaults)?; for kw_default in &args.kw_defaults { if let Some(expression) = kw_default { @@ -545,9 +544,8 @@ impl SymbolTableBuilder { let scope_depth = self.scopes.len(); let current_scope = self.scopes.last_mut().unwrap(); let location = Default::default(); - if let Some(_old_role) = current_scope.symbols.get(name) { + if current_scope.symbols.contains_key(name) { // Role already set.. - // debug!("TODO: {:?}", old_role); match role { SymbolRole::Global => { return Err(SymbolTableError { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 17d1caffb..7c2e80462 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -171,14 +171,13 @@ impl VirtualMachine { self.invoke(exc_type.into_object(), args) } + /// Create Python instance of `exc_type` with message pub fn new_exception(&self, exc_type: PyClassRef, msg: String) -> PyObjectRef { // TODO: exc_type may be user-defined exception, so we should return PyResult // TODO: maybe there is a clearer way to create an instance: info!("New exception created: {}", msg); let pymsg = self.new_str(msg); let args: Vec = vec![pymsg]; - - // Call function: self.invoke(exc_type.into_object(), args).unwrap() } @@ -402,13 +401,13 @@ impl VirtualMachine { scope: &Scope, defaults: &Option, kw_only_defaults: &Option, - args: PyFuncArgs, + func_args: PyFuncArgs, ) -> PyResult { - let scope = scope.child_scope(&self.ctx); + let scope = scope.new_child_scope(&self.ctx); self.fill_locals_from_args( &code.code, &scope.get_locals(), - args, + func_args, defaults, kw_only_defaults, )?; @@ -432,8 +431,8 @@ impl VirtualMachine { ) -> PyResult { if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope - .child_scope_with_locals(cells) - .child_scope_with_locals(locals); + .new_child_scope_with_locals(cells) + .new_child_scope_with_locals(locals); let frame = Frame::new(code.clone(), scope).into_ref(self); return self.run_frame_full(frame); } @@ -447,11 +446,11 @@ impl VirtualMachine { &self, code_object: &bytecode::CodeObject, locals: &PyDictRef, - args: PyFuncArgs, + func_args: PyFuncArgs, defaults: &Option, kw_only_defaults: &Option, ) -> PyResult<()> { - let nargs = args.args.len(); + let nargs = func_args.args.len(); let nexpected_args = code_object.arg_names.len(); // This parses the arguments from args and kwargs into @@ -469,7 +468,7 @@ impl VirtualMachine { // Copy positional arguments into local variables for i in 0..n { let arg_name = &code_object.arg_names[i]; - let arg = &args.args[i]; + let arg = &func_args.args[i]; locals.set_item(arg_name, arg.clone(), self)?; } @@ -478,7 +477,7 @@ impl VirtualMachine { bytecode::Varargs::Named(ref vararg_name) => { let mut last_args = vec![]; for i in n..nargs { - let arg = &args.args[i]; + let arg = &func_args.args[i]; last_args.push(arg.clone()); } let vararg_value = self.ctx.new_tuple(last_args); @@ -508,7 +507,7 @@ impl VirtualMachine { }; // Handle keyword arguments - for (name, value) in args.kwargs { + for (name, value) in func_args.kwargs { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { From bc35a64e5402566de8e8e9b4f01dfda87851429f Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 1 Jun 2019 03:19:22 +0300 Subject: [PATCH 644/658] fix most of clippy warnings --- parser/src/ast.rs | 1 + vm/src/builtins.rs | 11 +++++------ vm/src/dictdatatype.rs | 2 +- vm/src/frame.rs | 3 +-- vm/src/obj/objbytearray.rs | 2 +- vm/src/obj/objcode.rs | 1 + vm/src/obj/objcomplex.rs | 4 ++-- vm/src/obj/objfloat.rs | 2 +- vm/src/obj/objframe.rs | 1 + vm/src/obj/objint.rs | 10 +++++++--- vm/src/obj/objstr.rs | 14 +++++++------- vm/src/obj/objtuple.rs | 2 +- vm/src/pyhash.rs | 6 +++--- vm/src/pyobject.rs | 1 + vm/src/stdlib/ast.rs | 2 +- vm/src/stdlib/itertools.rs | 25 ++++++++++++++----------- vm/src/stdlib/os.rs | 13 ++++++++----- vm/src/stdlib/socket.rs | 4 ++-- vm/src/symboltable.rs | 14 +++++++++----- vm/src/vm.rs | 2 +- 20 files changed, 68 insertions(+), 52 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 1b985db2d..941e04131 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -44,6 +44,7 @@ pub struct Located { pub type LocatedStatement = Located; /// Abstract syntax tree nodes for python statements. +#[allow(clippy::large_enum_variant)] #[derive(Debug, PartialEq)] pub enum Statement { Break, diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index d467201a1..9afcd0e2c 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -612,13 +612,12 @@ impl Printer for std::io::StdoutLock<'_> { } } -pub fn builtin_exit(object: OptionalArg, vm: &VirtualMachine) -> PyResult<()> { - match object { - OptionalArg::Present(object) => match i32::try_from_object(&vm, object.clone()) { +pub fn builtin_exit(exit_code_arg: OptionalArg, vm: &VirtualMachine) -> PyResult<()> { + if let OptionalArg::Present(exit_code_obj) = exit_code_arg { + match i32::try_from_object(&vm, exit_code_obj.clone()) { Ok(code) => std::process::exit(code), - _ => println!("{}", vm.to_str(&object)?.as_str()), - }, - _ => {} + _ => println!("{}", vm.to_str(&exit_code_obj)?.as_str()), + } } std::process::exit(0); } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 788587266..ca20f2012 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -193,7 +193,7 @@ impl Dict { Box::new( self.entries .iter() - .filter_map(|v| v.as_ref().map_or(None, |v| Some(v.key.clone()))), + .filter_map(|v| v.as_ref().and_then(|v| Some(v.key.clone()))), ) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index fef36bfec..adc7bc46a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -173,8 +173,7 @@ impl NameProtocol for Scope { fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { self.locals .iter() - .skip(1) - .next() + .nth(1) .expect("no outer scope for non-local") .set_item(name, value, vm) .unwrap(); diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 2c72a3f46..51d608b09 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -340,7 +340,7 @@ impl PyByteArrayRef { let pos = bytes .iter() .position(|b| *b == x) - .ok_or(vm.new_value_error("value not found in bytearray".to_string()))?; + .ok_or_else(|| vm.new_value_error("value not found in bytearray".to_string()))?; bytes.remove(pos); diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index 60549cd3a..51a73d614 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -34,6 +34,7 @@ impl PyValue for PyCode { } impl PyCodeRef { + #[allow(clippy::new_ret_no_self)] fn new(_cls: PyClassRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("Cannot directly create code object".to_string())) } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index a349354a2..7a6df8705 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -126,12 +126,12 @@ impl PyComplex { #[pymethod(name = "__float__")] fn float(&self, vm: &VirtualMachine) -> PyResult { - return Err(vm.new_type_error(String::from("Can't convert complex to float"))); + Err(vm.new_type_error(String::from("Can't convert complex to float"))) } #[pymethod(name = "__int__")] fn int(&self, vm: &VirtualMachine) -> PyResult { - return Err(vm.new_type_error(String::from("Can't convert complex to int"))); + Err(vm.new_type_error(String::from("Can't convert complex to int"))) } #[pymethod(name = "__mul__")] diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index e300cf2ac..2d6287d7b 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -541,7 +541,7 @@ pub fn ufrexp(value: f64) -> (f64, i32) { } else { let bits = value.to_bits(); let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; - let mantissa_bits = bits & (0x000fffffffffffff) | (1022 << 52); + let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52); (f64::from_bits(mantissa_bits), exponent) } } diff --git a/vm/src/obj/objframe.rs b/vm/src/obj/objframe.rs index b4ed15f74..6669e3bb9 100644 --- a/vm/src/obj/objframe.rs +++ b/vm/src/obj/objframe.rs @@ -18,6 +18,7 @@ pub fn init(context: &PyContext) { } impl FrameRef { + #[allow(clippy::new_ret_no_self)] fn new(_class: FrameRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("Cannot directly create frame object".to_string())) } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index f8af3553f..2b703d30e 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -112,8 +112,9 @@ impl_try_from_object_int!( (u64, to_u64), ); +#[allow(clippy::collapsible_if)] fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { - Ok(if int2.value.is_negative() { + let result = if int2.value.is_negative() { let v1 = int1.float(vm)?; let v2 = int2.float(vm)?; vm.ctx.new_float(v1.pow(v2)) @@ -133,7 +134,8 @@ fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { // practically, exp over u64 is not possible to calculate anyway vm.ctx.not_implemented() } - }) + }; + Ok(result) } #[pyimpl] @@ -494,6 +496,7 @@ impl PyInt { } #[pymethod] + #[allow(clippy::match_bool)] fn from_bytes( bytes: PyByteInner, byteorder: PyStringRef, @@ -529,6 +532,7 @@ impl PyInt { Ok(x) } #[pymethod] + #[allow(clippy::match_bool)] fn to_bytes( &self, length: PyIntRef, @@ -547,7 +551,7 @@ impl PyInt { ); } } - if value.sign() == Sign::Minus && signed == false { + if value.sign() == Sign::Minus && !signed { return Err(vm.new_overflow_error("can't convert negative int to unsigned".to_string())); } let byte_len; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index b0d19e9c3..40d191296 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -863,7 +863,7 @@ impl PyString { None => false, }; // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(|c| UnicodeXID::is_xid_continue(c)) + is_identifier_start && chars.all(UnicodeXID::is_xid_continue) } // https://docs.python.org/3/library/stdtypes.html#str.translate @@ -876,18 +876,18 @@ impl PyString { match table.get_item(c as u32, vm) { Ok(value) => { if let Some(text) = value.payload::() { - translated.extend(text.value.chars()); - } else if let Some(_) = value.payload::() { - // Do Nothing + translated.push_str(&text.value); } else if let Some(bigint) = value.payload::() { match bigint.as_bigint().to_u32().and_then(std::char::from_u32) { Some(ch) => translated.push(ch as char), None => { - return Err(vm.new_value_error(format!( - "character mapping must be in range(0x110000)" - ))); + return Err(vm.new_value_error( + "character mapping must be in range(0x110000)".to_owned(), + )); } } + } else if let Some(_) = value.payload::() { + // Do Nothing } else { return Err(vm.new_type_error( "character mapping must return integer, None or str".to_owned(), diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 03596fbd2..40a5749ee 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -27,7 +27,7 @@ impl fmt::Debug for PyTuple { impl From> for PyTuple { fn from(elements: Vec) -> Self { - PyTuple { elements: elements } + PyTuple { elements } } } diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs index e717daa39..8bbd82168 100644 --- a/vm/src/pyhash.rs +++ b/vm/src/pyhash.rs @@ -9,11 +9,11 @@ pub type PyHash = i64; pub type PyUHash = u64; /// Prime multiplier used in string and various other hashes. -pub const MULTIPLIER: PyHash = 1000003; // 0xf4243 +pub const MULTIPLIER: PyHash = 1_000_003; // 0xf4243 /// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 pub const BITS: usize = 61; pub const MODULUS: PyUHash = (1 << BITS) - 1; -pub const INF: PyHash = 314159; +pub const INF: PyHash = 314_159; pub const NAN: PyHash = 0; pub const IMAG: PyHash = MULTIPLIER; @@ -42,7 +42,7 @@ pub fn hash_float(value: f64) -> PyHash { let mut x: PyUHash = 0; while m != 0.0 { x = ((x << 28) & MODULUS) | x >> (BITS - 28); - m *= 268435456.0; // 2**28 + m *= 268_435_456.0; // 2**28 e -= 28; let y = m as PyUHash; // pull out integer part m -= y as f64; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d88991b9e..70c8a84b4 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1238,6 +1238,7 @@ impl PyObject where T: Sized + PyObjectPayload, { + #[allow(clippy::new_ret_no_self)] pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ, dict, payload }.into_ref() } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 2ac36c81e..2b79dd706 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -457,7 +457,7 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult fn map_ast( f: fn(vm: &VirtualMachine, &T) -> PyResult, vm: &VirtualMachine, - items: &Vec, + items: &[T], ) -> PyResult { let list: PyResult> = items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index a8c333701..732020480 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -31,6 +31,7 @@ impl PyValue for PyItertoolsChain { #[pyimpl] impl PyItertoolsChain { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { Ok(PyItertoolsChain { iterables: args.args, @@ -88,6 +89,7 @@ impl PyValue for PyItertoolsCount { #[pyimpl] impl PyItertoolsCount { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, start: OptionalArg, @@ -105,7 +107,7 @@ impl PyItertoolsCount { Ok(PyItertoolsCount { cur: RefCell::new(start), - step: step, + step, } .into_ref(vm) .into_object()) @@ -140,6 +142,7 @@ impl PyValue for PyItertoolsRepeat { #[pyimpl] impl PyItertoolsRepeat { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, object: PyObjectRef, @@ -153,7 +156,7 @@ impl PyItertoolsRepeat { Ok(PyItertoolsRepeat { object: object.clone(), - times: times, + times, } .into_ref(vm) .into_object()) @@ -198,6 +201,7 @@ impl PyValue for PyItertoolsStarmap { #[pyimpl] impl PyItertoolsStarmap { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, function: PyObjectRef, @@ -206,12 +210,9 @@ impl PyItertoolsStarmap { ) -> PyResult { let iter = get_iter(vm, &iterable)?; - Ok(PyItertoolsStarmap { - function: function, - iter: iter, - } - .into_ref(vm) - .into_object()) + Ok(PyItertoolsStarmap { function, iter } + .into_ref(vm) + .into_object()) } #[pymethod(name = "__next__")] @@ -244,6 +245,7 @@ impl PyValue for PyItertoolsTakewhile { #[pyimpl] impl PyItertoolsTakewhile { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, predicate: PyObjectRef, @@ -253,7 +255,7 @@ impl PyItertoolsTakewhile { let iter = get_iter(vm, &iterable)?; Ok(PyItertoolsTakewhile { - predicate: predicate, + predicate, iterable: iter, stop_flag: RefCell::new(false), } @@ -314,6 +316,7 @@ fn pyobject_to_opt_usize(obj: PyObjectRef, vm: &VirtualMachine) -> Option #[pyimpl] impl PyItertoolsIslice { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { let (iter, start, stop, step) = match args.args.len() { 0 | 1 => { @@ -377,8 +380,8 @@ impl PyItertoolsIslice { iterable: iter, cur: RefCell::new(0), next: RefCell::new(start), - stop: stop, - step: step, + stop, + step, } .into_ref(vm) .into_object()) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f5a6ced96..0893c1e6d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -259,6 +259,7 @@ impl DirEntryRef { self.entry.path().to_str().unwrap().to_string() } + #[allow(clippy::match_bool)] fn perform_on_metadata( self, follow_symlinks: FollowSymlinks, @@ -434,6 +435,7 @@ fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { #[cfg(unix)] macro_rules! os_unix_stat_inner { ( $path:expr, $follow_symlinks:expr, $vm:expr ) => {{ + #[allow(clippy::match_bool)] fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { let meta = match follow_symlinks { true => fs::metadata(path)?, @@ -685,12 +687,13 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { where F: IntoPyNativeFunc, { + let func_obj = vm.ctx.new_rustfunc(func); Self { - name: name, - func_obj: vm.ctx.new_rustfunc(func), - fd: fd, - dir_fd: dir_fd, - follow_symlinks: follow_symlinks, + name, + func_obj, + fd, + dir_fd, + follow_symlinks, } } } diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 4f7cab5a1..68934bd01 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -218,7 +218,7 @@ impl SocketRef { } } - fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) -> () {} + fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) {} fn accept(self, vm: &VirtualMachine) -> PyResult { let ret = match self.con.borrow_mut().as_mut() { @@ -310,7 +310,7 @@ impl SocketRef { } } - fn close(self, _vm: &VirtualMachine) -> () { + fn close(self, _vm: &VirtualMachine) { self.con.borrow_mut().take(); } diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 24407c751..9687b0ec7 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -115,6 +115,7 @@ fn analyze_symbol_table( Ok(()) } +#[allow(clippy::single_match)] fn analyze_symbol( symbol_name: &str, symbol_role: &SymbolRole, @@ -298,13 +299,15 @@ impl SymbolTableBuilder { ast::Statement::Import { import_parts } => { for part in import_parts { if let Some(alias) = &part.alias { + // `import mymodule as myalias` + // `from mymodule import myimportname as myalias` self.register_name(alias, SymbolRole::Assigned)?; + } else if let Some(symbol) = &part.symbol { + // `from mymodule import myimport` + self.register_name(symbol, SymbolRole::Assigned)?; } else { - if let Some(symbol) = &part.symbol { - self.register_name(symbol, SymbolRole::Assigned)?; - } else { - self.register_name(&part.module, SymbolRole::Assigned)?; - } + // `import module` + self.register_name(&part.module, SymbolRole::Assigned)?; } } } @@ -540,6 +543,7 @@ impl SymbolTableBuilder { Ok(()) } + #[allow(clippy::single_match)] fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { let scope_depth = self.scopes.len(); let current_scope = self.scopes.last_mut().unwrap(); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7c2e80462..dfe89e640 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -951,7 +951,7 @@ impl VirtualMachine { } } - pub fn push_exception(&self, exc: PyObjectRef) -> () { + pub fn push_exception(&self, exc: PyObjectRef) { self.exceptions.borrow_mut().push(exc) } From 1294b56cbe5a7a7689ed5782feae10e42de48341 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 1 Jun 2019 14:32:23 +0300 Subject: [PATCH 645/658] Add _imp.create_builtin --- tests/snippets/imp.py | 13 +++++++++++++ vm/src/stdlib/imp.rs | 20 +++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py index a9d827b63..91b47daeb 100644 --- a/tests/snippets/imp.py +++ b/tests/snippets/imp.py @@ -1,4 +1,5 @@ import _imp +import time as import_time assert _imp.is_builtin("time") == True assert _imp.is_builtin("os") == False @@ -6,3 +7,15 @@ assert _imp.is_builtin("not existing module") == False assert _imp.is_frozen("__hello__") == True assert _imp.is_frozen("os") == False + +class FakeSpec: + def __init__(self, name): + self.name = name + +A = FakeSpec("time") + +imp_time = _imp.create_builtin(A) +assert imp_time.sleep == import_time.sleep + +B = FakeSpec("not existing module") +assert _imp.create_builtin(B) == None diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 5b249a135..f098d08fe 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,5 +1,6 @@ +use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{PyObjectRef, PyResult}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { @@ -29,6 +30,22 @@ fn imp_is_frozen(name: PyStringRef, vm: &VirtualMachine) -> bool { vm.frozen.borrow().contains_key(name.as_str()) } +fn imp_create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + + let name = &objstr::get_value(&vm.get_attribute(spec.clone(), "name")?); + + if let Ok(module) = sys_modules.get_item(name, vm) { + Ok(module) + } else { + if let Some(make_module_func) = vm.stdlib_inits.borrow().get(name) { + Ok(make_module_func(vm)) + } else { + Ok(vm.get_none()) + } + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -38,6 +55,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "lock_held" => ctx.new_rustfunc(imp_lock_held), "is_builtin" => ctx.new_rustfunc(imp_is_builtin), "is_frozen" => ctx.new_rustfunc(imp_is_frozen), + "create_builtin" => ctx.new_rustfunc(imp_create_builtin), }); module From 9a168b10d152507c9fd864205c53b6b622347c84 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 1 Jun 2019 14:49:12 +0300 Subject: [PATCH 646/658] Add _imp.exec_builtin that does nothing --- tests/snippets/imp.py | 2 ++ vm/src/stdlib/imp.rs | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py index 91b47daeb..fee4701e3 100644 --- a/tests/snippets/imp.py +++ b/tests/snippets/imp.py @@ -19,3 +19,5 @@ assert imp_time.sleep == import_time.sleep B = FakeSpec("not existing module") assert _imp.create_builtin(B) == None + +_imp.exec_builtin(imp_time) == 0 diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index f098d08fe..f285f9d60 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,3 +1,4 @@ +use crate::obj::objmodule::PyModuleRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; @@ -46,6 +47,11 @@ fn imp_create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { } } +fn imp_exec_builtin(_mod: PyModuleRef, _vm: &VirtualMachine) -> i32 { + // TOOD: Should we do something here? + 0 +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -56,6 +62,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "is_builtin" => ctx.new_rustfunc(imp_is_builtin), "is_frozen" => ctx.new_rustfunc(imp_is_frozen), "create_builtin" => ctx.new_rustfunc(imp_create_builtin), + "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), }); module From 4faaf2d6ca653cfd7362d2f9d111c31c81644060 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 1 Jun 2019 15:03:45 +0300 Subject: [PATCH 647/658] Add _imp.get_frozen_object --- tests/snippets/imp.py | 2 ++ vm/src/stdlib/imp.rs | 12 ++++++++++++ vm/src/vm.rs | 5 +++++ 3 files changed, 19 insertions(+) diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py index fee4701e3..0f60c8082 100644 --- a/tests/snippets/imp.py +++ b/tests/snippets/imp.py @@ -21,3 +21,5 @@ B = FakeSpec("not existing module") assert _imp.create_builtin(B) == None _imp.exec_builtin(imp_time) == 0 + +_imp.get_frozen_object("__hello__") diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index f285f9d60..6aa233b8d 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,3 +1,5 @@ +use crate::compile; +use crate::obj::objcode::PyCodeRef; use crate::obj::objmodule::PyModuleRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; @@ -52,6 +54,15 @@ fn imp_exec_builtin(_mod: PyModuleRef, _vm: &VirtualMachine) -> i32 { 0 } +fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + if let Some(frozen) = vm.frozen.borrow().get(name.as_str()) { + compile::compile(vm, frozen, &compile::Mode::Exec, "frozen".to_string()) + .map_err(|err| vm.new_syntax_error(&err)) + } else { + Err(vm.new_import_error(format!("No such frozen object named {}", name.as_str()))) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -63,6 +74,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "is_frozen" => ctx.new_rustfunc(imp_is_frozen), "create_builtin" => ctx.new_rustfunc(imp_create_builtin), "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), + "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), }); module diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 17d1caffb..f256b2279 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -256,6 +256,11 @@ impl VirtualMachine { syntax_error } + pub fn new_import_error(&self, msg: String) -> PyObjectRef { + let import_error = self.ctx.exceptions.import_error.clone(); + self.new_exception(import_error, msg) + } + pub fn new_scope_with_builtins(&self) -> Scope { Scope::with_builtins(None, self.ctx.new_dict(), self) } From 0acce07631ea8fcdea4c7c100105eaeca4047690 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 1 Jun 2019 15:11:55 +0300 Subject: [PATCH 648/658] Add _imp.init_frozen --- tests/snippets/imp.py | 3 +++ vm/src/stdlib/imp.rs | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py index 0f60c8082..44e7e1042 100644 --- a/tests/snippets/imp.py +++ b/tests/snippets/imp.py @@ -23,3 +23,6 @@ assert _imp.create_builtin(B) == None _imp.exec_builtin(imp_time) == 0 _imp.get_frozen_object("__hello__") + +hello = _imp.init_frozen("__hello__") +assert hello.initialized == True diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 6aa233b8d..ca266aa27 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,4 +1,5 @@ use crate::compile; +use crate::import::import_file; use crate::obj::objcode::PyCodeRef; use crate::obj::objmodule::PyModuleRef; use crate::obj::objstr; @@ -63,6 +64,14 @@ fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult PyResult { + if let Some(frozen) = vm.frozen.borrow().get(name.as_str()) { + import_file(vm, name.as_str(), "frozen".to_string(), frozen.to_string()) + } else { + Err(vm.new_import_error(format!("No such frozen object named {}", name.as_str()))) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -75,6 +84,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "create_builtin" => ctx.new_rustfunc(imp_create_builtin), "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), + "init_frozen" => ctx.new_rustfunc(imp_init_frozen), }); module From 106f5f7054e499a1312819a0a0392ccbe0b85f71 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 1 Jun 2019 15:14:17 +0300 Subject: [PATCH 649/658] Add _imp.is_frozen_package --- vm/src/stdlib/imp.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index ca266aa27..ae40c16c5 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -72,6 +72,11 @@ fn imp_init_frozen(name: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +fn imp_is_frozen_package(_name: PyStringRef, _vm: &VirtualMachine) -> bool { + // TODO: Support frozen package. + false +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -85,6 +90,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), "init_frozen" => ctx.new_rustfunc(imp_init_frozen), + "is_frozen_package" => ctx.new_rustfunc(imp_is_frozen_package), }); module From 3b5cc1ad2f19db30ce0fc1cd88da793014ba0058 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 2 Jun 2019 01:12:05 +0300 Subject: [PATCH 650/658] more doc comments, split with_exit() method for context manager __exit__ into two methods --- vm/src/bytecode.rs | 1 + vm/src/frame.rs | 79 ++++++++++++++++++++++--------------------- vm/src/obj/objbool.rs | 1 + vm/src/obj/objtype.rs | 2 +- vm/src/pyobject.rs | 2 ++ 5 files changed, 45 insertions(+), 40 deletions(-) diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index fb8cab98b..5df445d74 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -17,6 +17,7 @@ use std::fmt; #[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct CodeObject { pub instructions: Vec, + /// Jump targets. pub label_map: HashMap, pub locations: Vec, pub arg_names: Vec, // Names of positional arguments diff --git a/vm/src/frame.rs b/vm/src/frame.rs index adc7bc46a..88a1b31ad 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -243,7 +243,7 @@ pub enum ExecutionResult { Yield(PyObjectRef), } -// A valid execution result, or an exception +/// A valid execution result, or an exception pub type FrameResult = Result, PyObjectRef>; impl Frame { @@ -285,9 +285,11 @@ impl Frame { Ok(Some(value)) => { break Ok(value); } + // Instruction raised an exception Err(exception) => { - // unwind block stack on exception and find any handlers. - // Add an entry in the traceback: + // 1. Extract traceback from exception's '__traceback__' attr. + // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. + // 3. Unwind block stack till appropriate handler is found. assert!(objtype::isinstance( &exception, &vm.ctx.exceptions.base_exception_type @@ -296,13 +298,12 @@ impl Frame { .get_attribute(exception.clone(), "__traceback__") .unwrap(); trace!("Adding to traceback: {:?} {:?}", traceback, lineno); - let pos = vm.ctx.new_tuple(vec![ + let raise_location = vm.ctx.new_tuple(vec![ vm.ctx.new_str(filename.clone()), vm.ctx.new_int(lineno.get_row()), vm.ctx.new_str(run_obj_name.clone()), ]); - objlist::PyListRef::try_from_object(vm, traceback)?.append(pos, vm); - // exception.__trace + objlist::PyListRef::try_from_object(vm, traceback)?.append(raise_location, vm); match self.unwind_exception(vm, exception) { None => {} Some(exception) => { @@ -333,7 +334,7 @@ impl Frame { ins2 } - // Execute a single instruction: + /// Execute a single instruction. fn execute_instruction(&self, vm: &VirtualMachine) -> FrameResult { let instruction = self.fetch_instruction(); { @@ -565,9 +566,7 @@ impl Frame { } = &block.typ { debug_assert!(end1 == end2); - - // call exit now with no exception: - self.with_exit(vm, &context_manager, None)?; + self.call_context_manager_exit_no_exception(vm, &context_manager)?; } else { unreachable!("Block stack is incorrect, expected a with block"); } @@ -949,7 +948,7 @@ impl Frame { BlockType::With { context_manager, .. } => { - match self.with_exit(vm, &context_manager, None) { + match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { // __exit__ went wrong, @@ -976,7 +975,7 @@ impl Frame { } BlockType::With { context_manager, .. - } => match self.with_exit(vm, &context_manager, None) { + } => match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { panic!("Exception in with __exit__ {:?}", exc); @@ -1006,12 +1005,12 @@ impl Frame { end, context_manager, } => { - match self.with_exit(vm, &context_manager, Some(exc.clone())) { - Ok(exit_action) => { - match objbool::boolval(vm, exit_action) { - Ok(handle_exception) => { - if handle_exception { - // We handle the exception, so return! + match self.call_context_manager_exit(vm, &context_manager, exc.clone()) { + Ok(exit_result_obj) => { + match objbool::boolval(vm, exit_result_obj) { + // If __exit__ method returned True, suppress the exception and continue execution. + Ok(suppress_exception) => { + if suppress_exception { self.jump(end); return None; } else { @@ -1022,7 +1021,6 @@ impl Frame { return Some(exit_exc); } } - // if objtype::isinstance } Err(exit_exc) => { // TODO: what about original exception? @@ -1031,36 +1029,39 @@ impl Frame { } } BlockType::Loop { .. } => {} - // Exception was already poped on Raised. + // Exception was already popped on Raised. BlockType::ExceptHandler => {} } } Some(exc) } - fn with_exit( + fn call_context_manager_exit_no_exception( &self, vm: &VirtualMachine, context_manager: &PyObjectRef, - exc: Option, ) -> PyResult { - // Assume top of stack is __exit__ method: // TODO: do we want to put the exit call on the stack? - // let exit_method = self.pop_value(); - // let args = PyFuncArgs::default(); - // TODO: what happens when we got an error during handling exception? - let args = if let Some(exc) = exc { - let exc_type = exc.class().into_object(); - let exc_val = exc.clone(); - let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? - vec![exc_type, exc_val, exc_tb] - } else { - let exc_type = vm.ctx.none(); - let exc_val = vm.ctx.none(); - let exc_tb = vm.ctx.none(); - vec![exc_type, exc_val, exc_tb] - }; - vm.call_method(context_manager, "__exit__", args) + // TODO: what happens when we got an error during execution of __exit__? + vm.call_method( + context_manager, + "__exit__", + vec![vm.ctx.none(), vm.ctx.none(), vm.ctx.none()], + ) + } + + fn call_context_manager_exit( + &self, + vm: &VirtualMachine, + context_manager: &PyObjectRef, + exc: PyObjectRef, + ) -> PyResult { + // TODO: do we want to put the exit call on the stack? + // TODO: what happens when we got an error during execution of __exit__? + let exc_type = exc.class().into_object(); + let exc_val = exc.clone(); + let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? + vm.call_method(context_manager, "__exit__", vec![exc_type, exc_val, exc_tb]) } fn store_name( @@ -1131,7 +1132,7 @@ impl Frame { fn jump(&self, label: bytecode::Label) { let target_pc = self.code.label_map[&label]; - trace!("program counter from {:?} to {:?}", self.lasti, target_pc); + trace!("jump from {:?} to {:?}", self.lasti, target_pc); *self.lasti.borrow_mut() = target_pc; } diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index c1a999e45..8276a46f7 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -19,6 +19,7 @@ impl TryFromObject for bool { } } +/// Convert Python bool into Rust bool. pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { Ok(if let Ok(f) = vm.get_method(obj.clone(), "__bool__") { let bool_res = vm.invoke(f, PyFuncArgs::default())?; diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 29c66ffd1..0416d665b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -289,7 +289,7 @@ fn type_dict_setter(_instance: PyClassRef, _value: PyObjectRef, vm: &VirtualMach )) } -// This is the internal get_attr implementation for fast lookup on a class. +/// This is the internal get_attr implementation for fast lookup on a class. pub fn class_get_attr(class: &PyClassRef, attr_name: &str) -> Option { if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { return Some(item); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 70c8a84b4..3023573b0 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -87,6 +87,7 @@ pub type PyResult = Result; // A valid value, o /// For attributes we do not use a dict, but a hashmap. This is probably /// faster, unordered, and only supports strings as keys. +/// TODO: class attributes should maintain insertion order (use IndexMap here) pub type PyAttributes = HashMap; impl fmt::Display for PyObject { @@ -833,6 +834,7 @@ impl PyRef { pub fn as_object(&self) -> &PyObjectRef { &self.obj } + pub fn into_object(self) -> PyObjectRef { self.obj } From 58b1f4a8ee66324fe53bb54d7c83df0c606f1fe5 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 2 Jun 2019 18:21:52 +0300 Subject: [PATCH 651/658] add indented source line in traceback --- vm/src/exceptions.rs | 94 +++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index e02cd2894..464aea079 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -4,6 +4,8 @@ use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use std::fs::File; +use std::io::{BufRead, BufReader}; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let zelf = args.args[0].clone(); @@ -18,7 +20,7 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -// print excption chain +/// Print exception chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { let mut had_cause = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { @@ -39,41 +41,71 @@ pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { print_exception_inner(vm, exc) } -// Print exception including traceback: +fn print_source_line(filename: String, lineno: usize) { + // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/master/Python/traceback.c#L393 + // TODO: support different encodings + let file = match File::open(filename) { + Ok(file) => file, + Err(_) => { + return; + } + }; + let file = BufReader::new(file); + + for (i, line) in file.lines().enumerate() { + if i + 1 == lineno { + if let Ok(line) = line { + // Indented with 4 spaces + println!(" {}", line.trim_start()); + } + return; + } + } +} + +/// Print exception occurrence location from traceback element +fn print_traceback_entry(vm: &VirtualMachine, tb_entry: &PyObjectRef) { + if objtype::isinstance(&tb_entry, &vm.ctx.tuple_type()) { + let location_attrs = objsequence::get_elements_tuple(&tb_entry); + let filename = if let Ok(x) = vm.to_str(&location_attrs[0]) { + x.value.clone() + } else { + "".to_string() + }; + + let lineno = if let Ok(x) = vm.to_str(&location_attrs[1]) { + x.value.clone() + } else { + "".to_string() + }; + + let obj_name = if let Ok(x) = vm.to_str(&location_attrs[2]) { + x.value.clone() + } else { + "".to_string() + }; + + println!( + r##" File "{}", line {}, in {}"##, + filename, lineno, obj_name + ); + print_source_line(filename, lineno.parse().unwrap()); + } else { + println!(" File ??"); + return; + } +} + +/// Print exception with traceback pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let mut elements = objsequence::get_elements_list(&tb).to_vec(); - elements.reverse(); - for element in elements.iter() { - if objtype::isinstance(&element, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements_tuple(&element); - let filename = if let Ok(x) = vm.to_str(&element[0]) { - x.value.clone() - } else { - "".to_string() - }; + let mut tb_entries = objsequence::get_elements_list(&tb).to_vec(); + tb_entries.reverse(); - let lineno = if let Ok(x) = vm.to_str(&element[1]) { - x.value.clone() - } else { - "".to_string() - }; - - let obj_name = if let Ok(x) = vm.to_str(&element[2]) { - x.value.clone() - } else { - "".to_string() - }; - - println!( - r##" File "{}", line {}, in {}"##, - filename, lineno, obj_name - ); - } else { - println!(" File ??"); - } + for exc_location in tb_entries.iter() { + print_traceback_entry(vm, exc_location); } } } else { From bc33921fdb6ae7ddf75ae7c8f59375635fe05158 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Tue, 4 Jun 2019 14:56:44 +0300 Subject: [PATCH 652/658] make empty exception message format compatible with CPython --- vm/src/exceptions.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 464aea079..4f1f94744 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -12,7 +12,8 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let msg = if args.args.len() > 1 { args.args[1].clone() } else { - vm.new_str("No msg".to_string()) + let empty_string = String::default(); + vm.new_str(empty_string) }; let traceback = vm.ctx.new_list(Vec::new()); vm.set_attr(&zelf, "msg", msg)?; @@ -132,8 +133,11 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { panic!("Error message must be set"); }; - let s = format!("{}: {}", exc.class().name, msg); - Ok(vm.new_str(s)) + let mut exc_repr = exc.class().name.clone(); + if !msg.is_empty() { + &exc_repr.push_str(&format!(": {}", msg)); + } + Ok(vm.new_str(exc_repr)) } #[derive(Debug)] From b2cdeab0c6942a7f035f966819a006ab675d0ac5 Mon Sep 17 00:00:00 2001 From: Michel Heily Date: Wed, 5 Jun 2019 01:43:32 +0300 Subject: [PATCH 653/658] Fix memoryview.obj misimplementation. The memoryview skeleton did not correctly assign the "obj" attribute; The "obj" attribute was assigned to the memoryview class and not the memryview object, leading to a funny bug that can be reproduced as follows: ```python m1 = memoryview(b'1234') m2 = memoryview(b'abcd') print(m1.obj) ``` the result would be the value inside `m2` instead that of `m1` --- vm/src/obj/objmemory.rs | 47 +++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 88040ae14..8c2af26f0 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,18 +1,37 @@ use crate::obj::objbyteinner::try_as_byte; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass(name = "memoryview")] +#[derive(Debug)] +pub struct PyMemoryView { + obj_ref: PyObjectRef, +} + pub type PyMemoryViewRef = PyRef; -#[derive(Debug)] -pub struct PyMemoryView { - obj: PyObjectRef, -} - +#[pyimpl] impl PyMemoryView { pub fn get_obj_value(&self) -> Option> { - try_as_byte(&self.obj) + try_as_byte(&self.obj_ref) + } + + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + bytes_object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + PyMemoryView { + obj_ref: bytes_object.clone(), + } + .into_ref_with_type(vm, cls) + } + + #[pyproperty] + fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef { + self.obj_ref.clone() } } @@ -22,18 +41,6 @@ impl PyValue for PyMemoryView { } } -pub fn new_memory_view( - cls: PyClassRef, - bytes_object: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - vm.set_attr(cls.as_object(), "obj", bytes_object.clone())?; - PyMemoryView { obj: bytes_object }.into_ref_with_type(vm, cls) -} - pub fn init(ctx: &PyContext) { - let memoryview_type = &ctx.memoryview_type; - extend_class!(ctx, memoryview_type, { - "__new__" => ctx.new_rustfunc(new_memory_view) - }); + PyMemoryView::extend_class(ctx, &ctx.memoryview_type) } From d2a82b1acdcb83fac8c7ab7f02482611ea29f66a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 7 Jun 2019 15:14:27 +0300 Subject: [PATCH 654/658] Add _warnings --- tests/snippets/warnings.py | 3 +++ vm/src/stdlib/mod.rs | 2 ++ vm/src/stdlib/warnings.rs | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 tests/snippets/warnings.py create mode 100644 vm/src/stdlib/warnings.rs diff --git a/tests/snippets/warnings.py b/tests/snippets/warnings.py new file mode 100644 index 000000000..95092e506 --- /dev/null +++ b/tests/snippets/warnings.py @@ -0,0 +1,3 @@ +import _warnings + +_warnings.warn("Test") diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index ef2d7f3a5..096d14845 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -16,6 +16,7 @@ mod string; mod thread; mod time_module; mod tokenize; +mod warnings; mod weakref; use std::collections::HashMap; @@ -55,6 +56,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); modules.insert("_weakref".to_string(), Box::new(weakref::make_module)); modules.insert("_imp".to_string(), Box::new(imp::make_module)); + modules.insert("_warnings".to_string(), Box::new(warnings::make_module)); // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs new file mode 100644 index 000000000..bf0055d90 --- /dev/null +++ b/vm/src/stdlib/warnings.rs @@ -0,0 +1,37 @@ +use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +#[derive(FromArgs)] +struct WarnArgs { + #[pyarg(positional_only, optional = false)] + message: PyStringRef, + #[pyarg(positional_or_keyword, optional = true)] + category: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + stacklevel: OptionalArg, +} + +fn warnings_warn(args: WarnArgs, _vm: &VirtualMachine) { + // TODO: Implement correctly + let level = match args.stacklevel { + OptionalArg::Present(l) => l, + OptionalArg::Missing => 1, + }; + println!( + "Warning: {} , category: {:?}, level: {}", + args.message.as_str(), + args.category, + level + ) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_warnings", { + "warn" => ctx.new_rustfunc(warnings_warn), + }); + + module +} From c0e1702869f21d3512307b4bb72195671ea73282 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Fri, 7 Jun 2019 16:34:16 +0300 Subject: [PATCH 655/658] split mode/type for io.open properly, fix open('r') case --- tests/snippets/builtin_open.py | 13 +- vm/src/stdlib/io.rs | 228 +++++++++++++++++++++++++-------- 2 files changed, 189 insertions(+), 52 deletions(-) diff --git a/tests/snippets/builtin_open.py b/tests/snippets/builtin_open.py index 41bcf2b80..d7c310a92 100644 --- a/tests/snippets/builtin_open.py +++ b/tests/snippets/builtin_open.py @@ -6,5 +6,14 @@ assert 'RustPython' in fd.read() assert_raises(FileNotFoundError, lambda: open('DoesNotExist')) # Use open as a context manager -with open('README.md') as fp: - fp.read() +with open('README.md', 'rt') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'r') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'rb') as fp: + contents = fp.read() + assert type(contents) == bytes, "type is " + str(type(contents)) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 8b203cc78..80b9ffb7f 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,9 +1,7 @@ /* * I/O core tools. */ - use std::cell::RefCell; -use std::collections::HashSet; use std::fs::File; use std::io::prelude::*; use std::io::BufReader; @@ -294,6 +292,63 @@ fn text_io_base_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn split_mode_string(mode_string: String) -> Result<(String, String), String> { + let mut mode: char = '\0'; + let mut typ: char = '\0'; + let mut plus_is_set = false; + + for ch in mode_string.chars() { + match ch { + '+' => { + if plus_is_set { + return Err(format!("invalid mode: '{}'", mode_string)); + } + plus_is_set = true; + } + 't' | 'b' => { + if typ != '\0' { + if typ == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err("can't have text and binary mode at once".to_string()); + } + } + typ = ch; + } + 'a' | 'r' | 'w' => { + if mode != '\0' { + if mode == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err( + "must have exactly one of create/read/write/append mode".to_string() + ); + } + } + mode = ch; + } + _ => return Err(format!("invalid mode: '{}'", mode_string)), + } + } + + if mode == '\0' { + return Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string(), + ); + } + let mut mode = mode.to_string(); + if plus_is_set { + mode.push('+'); + } + if typ == '\0' { + typ = 't'; + } + Ok((mode, typ.to_string())) +} + pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -301,69 +356,55 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(file, Some(vm.ctx.str_type()))], optional = [(mode, Some(vm.ctx.str_type()))] ); + // mode is optional: 'rt' is the default mode (open from reading text) + let mode_string = mode.map_or("rt".to_string(), |m| objstr::get_value(m)); + let (mode, typ) = match split_mode_string(mode_string) { + Ok((mode, typ)) => (mode, typ), + Err(error_message) => { + return Err(vm.new_value_error(error_message)); + } + }; let module = import::import_module(vm, PathBuf::default(), "io").unwrap(); - //mode is optional: 'rt' is the default mode (open from reading text) - let rust_mode = mode.map_or("rt".to_string(), |m| objstr::get_value(m)); - - let mut raw_modes = HashSet::new(); - - //add raw modes - raw_modes.insert("a".to_string()); - raw_modes.insert("r".to_string()); - raw_modes.insert("x".to_string()); - raw_modes.insert("w".to_string()); - - //This is not a terribly elegant way to separate the file mode from - //the "type" flag - this should be improved. The intention here is to - //match a valid flag for the file_io_init call: - //https://docs.python.org/3/library/io.html#io.FileIO - let modes: Vec = rust_mode - .chars() - .filter(|a| raw_modes.contains(&a.to_string())) - .collect(); - - if modes.is_empty() || modes.len() > 1 { - return Err(vm.new_value_error("Invalid Mode".to_string())); - } - - //Class objects (potentially) consumed by io.open - //RawIO: FileIO - //Buffered: BufferedWriter, BufferedReader - //Text: TextIOWrapper + // Class objects (potentially) consumed by io.open + // RawIO: FileIO + // Buffered: BufferedWriter, BufferedReader + // Text: TextIOWrapper let file_io_class = vm.get_attribute(module.clone(), "FileIO").unwrap(); let buffered_writer_class = vm.get_attribute(module.clone(), "BufferedWriter").unwrap(); let buffered_reader_class = vm.get_attribute(module.clone(), "BufferedReader").unwrap(); let text_io_wrapper_class = vm.get_attribute(module, "TextIOWrapper").unwrap(); - //Construct a FileIO (subclass of RawIOBase) - //This is subsequently consumed by a Buffered Class. - let file_args = vec![file.clone(), vm.ctx.new_str(modes[0].to_string())]; - let file_io = vm.invoke(file_io_class, file_args)?; + // Construct a FileIO (subclass of RawIOBase) + // This is subsequently consumed by a Buffered Class. + let file_args = vec![file.clone(), vm.ctx.new_str(mode.clone())]; + let file_io_obj = vm.invoke(file_io_class, file_args)?; - //Create Buffered class to consume FileIO. The type of buffered class depends on - //the operation in the mode. - //There are 3 possible classes here, each inheriting from the RawBaseIO + // Create Buffered class to consume FileIO. The type of buffered class depends on + // the operation in the mode. + // There are 3 possible classes here, each inheriting from the RawBaseIO // creating || writing || appending => BufferedWriter - let buffered = if rust_mode.contains('w') { - vm.invoke(buffered_writer_class, vec![file_io.clone()]) - // reading => BufferedReader - } else { - vm.invoke(buffered_reader_class, vec![file_io.clone()]) + let buffered = match mode.chars().next().unwrap() { + 'w' => vm.invoke(buffered_writer_class, vec![file_io_obj.clone()]), + 'r' => vm.invoke(buffered_reader_class, vec![file_io_obj.clone()]), //TODO: updating => PyBufferedRandom + _ => unimplemented!("'a' mode is not yet implemented"), }; - if rust_mode.contains('t') { - //If the mode is text this buffer type is consumed on construction of - //a TextIOWrapper which is subsequently returned. - vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) - } else { + let io_obj = match typ.chars().next().unwrap() { + // If the mode is text this buffer type is consumed on construction of + // a TextIOWrapper which is subsequently returned. + 't' => vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]), + // If the mode is binary this Buffered class is returned directly at // this point. - //For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field - buffered - } + // For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field + 'b' => buffered, + + _ => unreachable!(), + }; + io_obj } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -439,3 +480,90 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "BytesIO" => bytes_io, }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_mode_split_into(mode_string: &str, expected_mode: &str, expected_typ: &str) { + let (mode, typ) = split_mode_string(mode_string.to_string()).unwrap(); + assert_eq!(mode, expected_mode); + assert_eq!(typ, expected_typ); + } + + #[test] + fn test_split_mode_valid_cases() { + assert_mode_split_into("r", "r", "t"); + assert_mode_split_into("rb", "r", "b"); + assert_mode_split_into("rt", "r", "t"); + assert_mode_split_into("r+t", "r+", "t"); + assert_mode_split_into("w+t", "w+", "t"); + assert_mode_split_into("r+b", "r+", "b"); + assert_mode_split_into("w+b", "w+", "b"); + } + + #[test] + fn test_invalid_mode() { + assert_eq!( + split_mode_string("rbsss".to_string()), + Err("invalid mode: 'rbsss'".to_string()) + ); + assert_eq!( + split_mode_string("rrb".to_string()), + Err("invalid mode: 'rrb'".to_string()) + ); + assert_eq!( + split_mode_string("rbb".to_string()), + Err("invalid mode: 'rbb'".to_string()) + ); + } + + #[test] + fn test_mode_not_specified() { + assert_eq!( + split_mode_string("".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("b".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("t".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + } + + #[test] + fn test_text_and_binary_at_once() { + assert_eq!( + split_mode_string("rbt".to_string()), + Err("can't have text and binary mode at once".to_string()) + ); + } + + #[test] + fn test_exactly_one_mode() { + assert_eq!( + split_mode_string("rwb".to_string()), + Err("must have exactly one of create/read/write/append mode".to_string()) + ); + } + + #[test] + fn test_at_most_one_plus() { + assert_eq!( + split_mode_string("a++".to_string()), + Err("invalid mode: 'a++'".to_string()) + ); + } +} From 8fccad1e09a5c1bf34ef292fee5379b5891c747e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 7 Jun 2019 18:59:41 +0300 Subject: [PATCH 656/658] Print warning to stderr Co-Authored-By: coolreader18 <33094578+coolreader18@users.noreply.github.com> --- vm/src/stdlib/warnings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs index bf0055d90..cb97ead48 100644 --- a/vm/src/stdlib/warnings.rs +++ b/vm/src/stdlib/warnings.rs @@ -19,7 +19,7 @@ fn warnings_warn(args: WarnArgs, _vm: &VirtualMachine) { OptionalArg::Present(l) => l, OptionalArg::Missing => 1, }; - println!( + eprintln!( "Warning: {} , category: {:?}, level: {}", args.message.as_str(), args.category, From 50252b4e809f7277670b284d168157299572fd87 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 7 Jun 2019 11:50:15 -0500 Subject: [PATCH 657/658] Disable the pwd module on android --- vm/Cargo.toml | 2 +- vm/src/stdlib/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ddc8ee20c..922163d01 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -38,5 +38,5 @@ bincode = "1.1.4" git = "https://github.com/OddCoincidence/unicode-casing" rev = "90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" -[target.'cfg(unix)'.dependencies] +[target.'cfg(all(unix, not(target_os = "android")))'.dependencies] pwd = "1" diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 096d14845..7c72a1b61 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -26,7 +26,7 @@ use crate::vm::VirtualMachine; pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "android")))] mod pwd; use crate::pyobject::PyObjectRef; @@ -67,7 +67,7 @@ pub fn get_module_inits() -> HashMap { } // Unix-only - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "android")))] { modules.insert("pwd".to_string(), Box::new(pwd::make_module)); } From baf62d56d2a128dd0cec9df8aaea416a47c3831e Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 8 Jun 2019 02:42:06 +0300 Subject: [PATCH 658/658] get io reader/writer classes only when needed --- vm/src/stdlib/io.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 80b9ffb7f..da8bedf53 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -365,29 +365,33 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } }; - let module = import::import_module(vm, PathBuf::default(), "io").unwrap(); - - // Class objects (potentially) consumed by io.open - // RawIO: FileIO - // Buffered: BufferedWriter, BufferedReader - // Text: TextIOWrapper - let file_io_class = vm.get_attribute(module.clone(), "FileIO").unwrap(); - let buffered_writer_class = vm.get_attribute(module.clone(), "BufferedWriter").unwrap(); - let buffered_reader_class = vm.get_attribute(module.clone(), "BufferedReader").unwrap(); - let text_io_wrapper_class = vm.get_attribute(module, "TextIOWrapper").unwrap(); + let io_module = import::import_module(vm, PathBuf::default(), "io").unwrap(); // Construct a FileIO (subclass of RawIOBase) // This is subsequently consumed by a Buffered Class. - let file_args = vec![file.clone(), vm.ctx.new_str(mode.clone())]; - let file_io_obj = vm.invoke(file_io_class, file_args)?; + let file_io_class = vm.get_attribute(io_module.clone(), "FileIO").unwrap(); + let file_io_obj = vm.invoke( + file_io_class, + vec![file.clone(), vm.ctx.new_str(mode.clone())], + )?; // Create Buffered class to consume FileIO. The type of buffered class depends on // the operation in the mode. // There are 3 possible classes here, each inheriting from the RawBaseIO // creating || writing || appending => BufferedWriter let buffered = match mode.chars().next().unwrap() { - 'w' => vm.invoke(buffered_writer_class, vec![file_io_obj.clone()]), - 'r' => vm.invoke(buffered_reader_class, vec![file_io_obj.clone()]), + 'w' => { + let buffered_writer_class = vm + .get_attribute(io_module.clone(), "BufferedWriter") + .unwrap(); + vm.invoke(buffered_writer_class, vec![file_io_obj.clone()]) + } + 'r' => { + let buffered_reader_class = vm + .get_attribute(io_module.clone(), "BufferedReader") + .unwrap(); + vm.invoke(buffered_reader_class, vec![file_io_obj.clone()]) + } //TODO: updating => PyBufferedRandom _ => unimplemented!("'a' mode is not yet implemented"), }; @@ -395,13 +399,14 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let io_obj = match typ.chars().next().unwrap() { // If the mode is text this buffer type is consumed on construction of // a TextIOWrapper which is subsequently returned. - 't' => vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]), - + 't' => { + let text_io_wrapper_class = vm.get_attribute(io_module, "TextIOWrapper").unwrap(); + vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) + } // If the mode is binary this Buffered class is returned directly at // this point. // For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field 'b' => buffered, - _ => unreachable!(), }; io_obj