From b0d7960cc59ed1a9a1a8f794818349024eac3f4f Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:14:37 -0700 Subject: [PATCH 001/223] bytes: move methods to impl block --- vm/src/obj/objbytes.rs | 202 +++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 100 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8d64cc507..5b725be50 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -57,16 +57,16 @@ pub fn init(context: &PyContext) { - an integer"; extend_class!(context, bytes_type, { - "__eq__" => context.new_rustfunc(bytes_eq), - "__lt__" => context.new_rustfunc(bytes_lt), - "__le__" => context.new_rustfunc(bytes_le), - "__gt__" => context.new_rustfunc(bytes_gt), - "__ge__" => context.new_rustfunc(bytes_ge), - "__hash__" => context.new_rustfunc(bytes_hash), "__new__" => context.new_rustfunc(bytes_new), - "__repr__" => context.new_rustfunc(bytes_repr), - "__len__" => context.new_rustfunc(bytes_len), - "__iter__" => context.new_rustfunc(bytes_iter), + "__eq__" => context.new_rustfunc(PyBytesRef::eq), + "__lt__" => context.new_rustfunc(PyBytesRef::lt), + "__le__" => context.new_rustfunc(PyBytesRef::le), + "__gt__" => context.new_rustfunc(PyBytesRef::gt), + "__ge__" => context.new_rustfunc(PyBytesRef::ge), + "__hash__" => context.new_rustfunc(PyBytesRef::hash), + "__repr__" => context.new_rustfunc(PyBytesRef::repr), + "__len__" => context.new_rustfunc(PyBytesRef::len), + "__iter__" => context.new_rustfunc(PyBytesRef::iter), "__doc__" => context.new_str(bytes_doc.to_string()) }); } @@ -93,111 +93,113 @@ fn bytes_new( PyBytes::new(value).into_ref_with_type(vm, cls) } -fn bytes_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); +impl PyBytesRef { + fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() == get_value(b).to_vec() + } else { + false + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() >= get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() > get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() <= get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + fn lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(a, Some(vm.ctx.bytes_type())), (b, None)] + ); - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { + get_value(a).to_vec() < get_value(b).to_vec() + } else { + return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); + }; + Ok(vm.ctx.new_bool(result)) + } -fn bytes_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); + fn len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + let byte_vec = get_value(a).to_vec(); + Ok(vm.ctx.new_int(byte_vec.len())) + } -fn bytes_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) + fn hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); + let data = get_value(zelf); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + let hash = hasher.finish(); + Ok(vm.ctx.new_int(hash)) + } + + fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); + let value = get_value(obj); + let data = String::from_utf8(value.to_vec()).unwrap(); + Ok(vm.new_str(format!("b'{}'", data))) + } + + fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { + PyIteratorValue { + position: Cell::new(0), + iterated_obj: obj.into_object(), + } + } } pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().value } - -fn bytes_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) -} - -fn bytes_iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { - position: Cell::new(0), - iterated_obj: obj.into_object(), - } -} From e4272126cf5589ecd24a6a89a77daba364968c49 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:15:38 -0700 Subject: [PATCH 002/223] bytes: return NotImplemented where appropriate --- vm/src/obj/objbytes.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 5b725be50..bed57157e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -104,7 +104,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() == get_value(b).to_vec() } else { - false + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -119,7 +119,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() >= get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -134,7 +134,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() > get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -149,7 +149,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() <= get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } @@ -164,7 +164,7 @@ impl PyBytesRef { let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { get_value(a).to_vec() < get_value(b).to_vec() } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); + return Ok(vm.ctx.not_implemented()); }; Ok(vm.ctx.new_bool(result)) } From e0aca86473738ea8cf592fce5b7e4769293b6227 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 27 Mar 2019 19:17:27 -0700 Subject: [PATCH 003/223] bytes: convert methods to new args style --- vm/src/obj/objbytes.rs | 121 +++++++++++++---------------------------- 1 file changed, 39 insertions(+), 82 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index bed57157e..9390e7070 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,17 +1,16 @@ use std::cell::Cell; +use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::ops::Deref; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; -use super::objtype::{self, PyClassRef}; +use super::objtype::PyClassRef; #[derive(Debug)] pub struct PyBytes { @@ -94,102 +93,60 @@ fn bytes_new( } impl PyBytesRef { - fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value == other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value >= other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value > other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value <= other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if let Ok(other) = other.downcast::() { + vm.ctx.new_bool(self.value < other.value) } else { - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.not_implemented() + } } - fn len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); - - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) + fn len(self, _vm: &VirtualMachine) -> usize { + self.value.len() } - fn hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) + fn hash(self, _vm: &VirtualMachine) -> u64 { + let mut hasher = DefaultHasher::new(); + self.value.hash(&mut hasher); + hasher.finish() } - fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) + fn repr(self, _vm: &VirtualMachine) -> String { + // TODO: don't just unwrap + let data = String::from_utf8(self.value.clone()).unwrap(); + format!("b'{}'", data) } fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { From 73cd680d8d761bd6f0616f09bfab1f5e87be9d56 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 28 Mar 2019 17:50:41 +0100 Subject: [PATCH 004/223] Add support for trailing comma in function defs. Parser code simplications. --- parser/src/lexer.rs | 17 +++++++-- parser/src/python.lalrpop | 72 ++++++++++++-------------------------- tests/snippets/function.py | 5 +++ 3 files changed, 42 insertions(+), 52 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index defa5e232..edf21c973 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -466,8 +466,16 @@ where loop { match self.next_char() { Some('\\') => { - if is_raw { + if self.chr0 == Some(quote_char) { + string_content.push(quote_char); + self.next_char(); + } else if is_raw { string_content.push('\\'); + if let Some(c) = self.next_char() { + string_content.push(c) + } else { + return Err(LexicalError::StringError); + } } else { match self.next_char() { Some('\\') => { @@ -711,7 +719,6 @@ where let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); - println!("Emoji: {}", c); return Some(Ok(( tok_start, Tok::Name { @@ -1438,7 +1445,7 @@ mod tests { #[test] fn test_string() { - let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g'"#); + let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#); let tokens = lex_source(&source); assert_eq!( tokens, @@ -1467,6 +1474,10 @@ mod tests { value: String::from("\\g"), is_fstring: false, }, + Tok::String { + value: String::from("raw\'"), + is_fstring: false, + }, ] ); } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 9e5b309b1..7f010278d 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -337,10 +337,7 @@ CompoundStatement: ast::LocatedStatement = { IfStatement: ast::LocatedStatement = { "if" ":" => { // Determine last else: - let mut last = match s3 { - Some(s) => Some(s.2), - None => None, - }; + let mut last = s3.map(|s| s.2); // handle elif: for i in s2.into_iter().rev() { @@ -360,10 +357,7 @@ IfStatement: ast::LocatedStatement = { WhileStatement: ast::LocatedStatement = { "while" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::While { test: e, body: s, orelse: or_else }, @@ -373,10 +367,7 @@ WhileStatement: ast::LocatedStatement = { ForStatement: ast::LocatedStatement = { "for" "in" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::For { @@ -389,14 +380,8 @@ ForStatement: ast::LocatedStatement = { TryStatement: ast::LocatedStatement = { "try" ":" => { - let or_else = match else_suite { - Some(s) => Some(s.2), - None => None, - }; - let finalbody = match finally { - Some(s) => Some(s.2), - None => None, - }; + let or_else = else_suite.map(|s| s.2); + let finalbody = finally.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::Try { @@ -437,10 +422,7 @@ WithStatement: ast::LocatedStatement = { WithItem: ast::WithItem = { => { - let optional_vars = match n { - Some(val) => Some(val.1), - None => None, - }; + let optional_vars = n.map(|val| val.1); ast::WithItem { context_expr: t, optional_vars } }, }; @@ -461,27 +443,19 @@ FuncDef: ast::LocatedStatement = { }; Parameters: ast::Parameters = { - "(" )?> ")" => { - match a { - Some(a) => a, - None => Default::default(), - } + "(" )?> ")" => { + a.unwrap_or_else(Default::default) }, }; -// parameters are (String, None), kwargs are (String, Some(Test)) where Test is -// the default // Note that this is a macro which is used once for function defs, and // once for lambda defs. -TypedArgsList: ast::Parameters = { - > )?> => { +ParameterList: ast::Parameters = { + > )?> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: - let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 { - Some((_, x)) => x, - None => (None, vec![], vec![], None), - }; + let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); ast::Parameters { args: names, @@ -492,7 +466,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > )> => { + > )> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: @@ -510,7 +484,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { let (vararg, kwonlyargs, kw_defaults, kwarg) = params; ast::Parameters { args: vec![], @@ -521,7 +495,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { ast::Parameters { args: vec![], kwonlyargs: vec![], @@ -535,8 +509,8 @@ TypedArgsList: ast::Parameters = { // Use inline here to make sure the "," is not creating an ambiguity. #[inline] -TypedParameters: (Vec, Vec) = { - > )*> => { +ParameterDefs: (Vec, Vec) = { + > )*> => { // Combine first parameters: let mut args = vec![param1]; args.extend(param2.into_iter().map(|x| x.1)); @@ -564,7 +538,7 @@ TypedParameters: (Vec, Vec) = { } }; -TypedParameterDef: (ast::Parameter, Option) = { +ParameterDef: (ast::Parameter, Option) = { => (i, None), "=" => (i, Some(e)), }; @@ -580,8 +554,11 @@ TypedParameter: ast::Parameter = { }, }; +// Use inline here to make sure the "," is not creating an ambiguity. +// TODO: figure out another grammar that makes this inline no longer required. +#[inline] ParameterListStarArgs: (Option>, Vec, Vec>, Option>) = { - "*" )*> )?> => { + "*" )*> )?> => { // Extract keyword arguments: let mut kwonlyargs = vec![]; let mut kw_defaults = vec![]; @@ -590,10 +567,7 @@ ParameterListStarArgs: (Option>, Vec Some(name), - None => None, - }; + let kwarg = kwarg.map(|n| n.1); (Some(va), kwonlyargs, kw_defaults, kwarg) } @@ -684,7 +658,7 @@ Test: ast::Expression = { }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), body:Box::new(b) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index a69178bb7..b1ea96860 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -2,3 +2,8 @@ def foo(): return 42 assert foo() == 42 + +def my_func(a,): + return a+2 + +assert my_func(2) == 4 From ff85838556989d9dccc79f3d8baa1ae44202a65e Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 00:16:34 -0500 Subject: [PATCH 005/223] Add #[py_class] attribute proc macro --- derive/Cargo.toml | 2 +- derive/src/lib.rs | 198 ++++++++++++++- vm/src/builtins.rs | 1 + vm/src/lib.rs | 2 +- vm/src/obj/objstr.rs | 572 +++++++++++++++++++++++-------------------- vm/src/pyobject.rs | 20 ++ 6 files changed, 517 insertions(+), 278 deletions(-) diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c15d47ad5..91fdc150e 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -8,6 +8,6 @@ edition = "2018" proc-macro = true [dependencies] -syn = "0.15.29" +syn = { version = "0.15.29", features = ["full"] } quote = "0.6.11" proc-macro2 = "0.4.27" diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ad743e72f..64853170f 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,21 +1,57 @@ extern crate proc_macro; use proc_macro::TokenStream; -use proc_macro2::TokenStream as TokenStream2; +use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; -use syn::{Data, DeriveInput, Fields}; +use syn::{ + parse_macro_input, AttributeArgs, Data, DeriveInput, Fields, Ident, ImplItem, Item, Lit, Meta, + NestedMeta, +}; -#[proc_macro_derive(FromArgs)] +fn rustpython_path(inside_vm: bool) -> syn::Path { + let path = if inside_vm { + quote!(crate) + } else { + quote!(::rustpython_vm) + }; + syn::parse2(path).unwrap() +} + +/// Does the item have the #[__inside_vm] attribute on it, signifying that the derive target is +/// being derived from inside the `rustpython_vm` crate. +fn rustpython_path_derive(input: &DeriveInput) -> syn::Path { + rustpython_path( + input + .attrs + .iter() + .any(|attr| attr.path.is_ident("__inside_vm")), + ) +} + +fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { + rustpython_path(attr.iter().any(|meta| { + if let syn::NestedMeta::Meta(meta) = meta { + if let syn::Meta::Word(ident) = meta { + ident == "__inside_vm" + } else { + false + } + } else { + false + } + })) +} + +#[proc_macro_derive(FromArgs, attributes(__inside_vm))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); - let gen = impl_from_args(&ast); + let gen = impl_from_args(ast); gen.to_string().parse().unwrap() } -fn impl_from_args(input: &DeriveInput) -> TokenStream2 { - // FIXME: This references types using `crate` instead of `rustpython_vm` - // so that it can be used in the latter. How can we support both? +fn impl_from_args(input: DeriveInput) -> TokenStream2 { + let rp_path = rustpython_path_derive(&input); let fields = match input.data { Data::Struct(ref data) => { match data.fields { @@ -36,7 +72,7 @@ fn impl_from_args(input: &DeriveInput) -> TokenStream2 { let name = &input.ident; quote! { - impl crate::function::FromArgs for #name { + impl #rp_path::function::FromArgs for #name { fn from_args( vm: &crate::vm::VirtualMachine, args: &mut crate::function::PyFuncArgs @@ -46,3 +82,149 @@ fn impl_from_args(input: &DeriveInput) -> TokenStream2 { } } } + +#[proc_macro_attribute] +pub fn py_class(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + impl_py_class(attr, item).into() +} + +enum MethodKind { + Method, + Property, +} + +impl MethodKind { + fn to_ctx_constructor_fn(&self) -> Ident { + let f = match self { + MethodKind::Method => "new_rustfunc", + MethodKind::Property => "new_property", + }; + Ident::new(f, Span::call_site()) + } +} + +struct Method { + fn_name: Ident, + py_name: String, + kind: MethodKind, +} + +fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter().filter_map(|item| { + if let ImplItem::Method(meth) = item { + let mut py_name = None; + let mut kind = MethodKind::Method; + let metas_iter = meth + .attrs + .iter() + .filter_map(|attr| { + if attr.path.is_ident("py_class") { + let meta = attr.parse_meta().expect("Invalid attribute"); + if let Meta::List(list) = meta { + Some(list) + } else { + panic!( + "#[py_class] attribute on a method should be a list, like \ + #[py_class(...)]" + ) + } + } else { + None + } + }) + .flat_map(|attr| attr.nested); + for meta in metas_iter { + if let NestedMeta::Meta(meta) = meta { + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => match ident.to_string().as_str() { + "property" => kind = MethodKind::Property, + _ => {} + }, + _ => {} + } + } + } + let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); + Some(Method { + fn_name: meth.sig.ident.clone(), + py_name, + kind, + }) + } else { + None + } + }) +} + +fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { + let imp = if let Item::Impl(imp) = item { + imp + } else { + return quote!(#item); + }; + let rp_path = rustpython_path_attr(&attr); + let mut class_name = None; + let mut doc = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } else if name_value.ident == "doc" { + if let Lit::Str(s) = name_value.lit { + doc = Some(s.value()); + } else { + panic!("#[py_class(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.expect("#[py_class] must have a name"); + let doc = match doc { + Some(doc) => quote!(Some(#doc)), + None => quote!(None), + }; + let ty = &imp.self_ty; + let methods = item_impl_to_methods(&imp).map( + |Method { + py_name, + fn_name, + kind, + }| { + let constructor_fn = kind.to_ctx_constructor_fn(); + quote! { + ctx.set_attr(class, #py_name, ctx.#constructor_fn(#ty::#fn_name)); + } + }, + ); + + quote! { + #imp + impl #rp_path::pyobject::IntoPyClass for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + fn _extend_class( + ctx: &#rp_path::pyobject::PyContext, + class: &#rp_path::obj::objtype::PyClassRef, + ) { + #(#methods)* + } + } + } +} diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 9d5980759..6b5dda16b 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -536,6 +536,7 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } #[derive(Debug, FromArgs)] +#[__inside_vm] pub struct PrintOptions { sep: Option, end: Option, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 3892fe2a4..29124ad11 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -30,7 +30,7 @@ extern crate statrs; extern crate rustpython_parser; #[macro_use] -extern crate rustpython_derive; +pub extern crate rustpython_derive; //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index aa62812a2..82ac806e6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -10,8 +10,8 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TryIntoRef, TypeProtocol, + IdProtocol, IntoPyClass, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, + PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -48,80 +48,138 @@ impl TryIntoRef for &str { } } -impl PyStringRef { - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(format!("{}{}", self.value, get_value(&rhs))) +fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + let rep_str = match rep { + OptionalArg::Present(ref st) => &st.value, + OptionalArg::Missing => " ", + }; + if rep_str.len() == 1 { + Ok(rep_str) + } else { + Err(vm.new_type_error("The fill character must be exactly one character long".to_string())) + } +} + +#[py_class( + __inside_vm, + name = "str", + doc = "str(object='') -> str\n\ + str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ + \n\ + Create a new string object from the given object. If encoding or\n\ + errors is specified, then the object must expose a data buffer\n\ + that will be decoded using the given encoding and error handler.\n\ + Otherwise, returns the result of object.__str__() (if defined)\n\ + or repr(object).\n\ + encoding defaults to sys.getdefaultencoding().\n\ + errors defaults to 'strict'." +)] +impl PyString { + // TODO: should with following format + // class str(object='') + // class str(object=b'', encoding='utf-8', errors='strict') + #[py_class(name = "__new__")] + fn new( + cls: PyClassRef, + object: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let string = match object { + OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), + OptionalArg::Missing => vm.new_str("".to_string()), + }; + if string.class().is(&cls) { + TryFromObject::try_from_object(vm, string) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", self, rhs))) + let payload = string.payload::().unwrap(); + payload.clone().into_ref_with_type(vm, cls) + } + } + #[py_class(name = "__add__")] + fn add(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&rhs, &vm.ctx.str_type()) { + Ok(format!("{}{}", zelf.value, get_value(&rhs))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", zelf, rhs))) } } - fn bool(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() + #[py_class(name = "__bool__")] + fn bool(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() } - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + #[py_class(name = "__eq__")] + fn eq(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - self.value == get_value(&rhs) + zelf.value == get_value(&rhs) } else { false } } - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { - self.value.contains(&needle.value) + #[py_class(name = "__contains__")] + fn contains(zelf: PyStringRef, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + zelf.value.contains(&needle.value) } - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - subscript(vm, &self.value, needle) + #[py_class(name = "__getitem__")] + fn getitem(zelf: PyStringRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + subscript(vm, &zelf.value, needle) } - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__gt__")] + fn gt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value > get_value(&rhs)) + Ok(zelf.value > get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__ge__")] + fn ge(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value >= get_value(&rhs)) + Ok(zelf.value >= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__lt__")] + fn lt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value < get_value(&rhs)) + Ok(zelf.value < get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__le__")] + fn le(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(self.value <= get_value(&rhs)) + Ok(zelf.value <= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) } } - fn hash(self, _vm: &VirtualMachine) -> usize { + #[py_class(name = "__hash__")] + fn hash(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); + zelf.value.hash(&mut hasher); hasher.finish() as usize } - fn len(self, _vm: &VirtualMachine) -> usize { - self.value.chars().count() + #[py_class(name = "__len__")] + fn len(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { + zelf.value.chars().count() } - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[py_class(name = "__mul__")] + fn mul(zelf: PyStringRef, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &self.value; + let value = &zelf.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); let mut result = String::new(); for _x in 0..multiplier { @@ -129,16 +187,18 @@ impl PyStringRef { } Ok(result) } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) + Err(vm.new_type_error(format!("Cannot multiply {} and {}", zelf, val))) } } - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + #[py_class(name = "__str__")] + fn str(zelf: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + zelf } - fn repr(self, _vm: &VirtualMachine) -> String { - let value = &self.value; + #[py_class(name = "__repr__")] + fn repr(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let value = &zelf.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' } else { @@ -167,31 +227,31 @@ impl PyStringRef { formatted } - fn lower(self, _vm: &VirtualMachine) -> String { - self.value.to_lowercase() + fn lower(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(self, _vm: &VirtualMachine) -> String { - caseless::default_case_fold_str(&self.value) + fn casefold(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + caseless::default_case_fold_str(&zelf.value) } - fn upper(self, _vm: &VirtualMachine) -> String { - self.value.to_uppercase() + fn upper(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.to_uppercase() } - fn capitalize(self, _vm: &VirtualMachine) -> String { - let (first_part, lower_str) = self.value.split_at(1); + fn capitalize(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let (first_part, lower_str) = zelf.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } fn split( - self, - pattern: OptionalArg, + zelf: PyStringRef, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &self.value; + let value = &zelf.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -207,12 +267,12 @@ impl PyStringRef { } fn rsplit( - self, - pattern: OptionalArg, + zelf: PyStringRef, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &self.value; + let value = &zelf.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -227,85 +287,113 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn strip(self, _vm: &VirtualMachine) -> String { - self.value.trim().to_string() + fn strip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim().to_string() } - fn lstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_start().to_string() + fn lstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim_start().to_string() } - fn rstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_end().to_string() + fn rstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + zelf.value.trim_end().to_string() } fn endswith( - self, + zelf: PyStringRef, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].ends_with(&suffix.value) + if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { + zelf.value[start..end].ends_with(&suffix.value) } else { false } } fn startswith( - self, + zelf: PyStringRef, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].starts_with(&prefix.value) + if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { + zelf.value[start..end].starts_with(&prefix.value) } else { false } } - fn isalnum(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) + fn isalnum(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) } - fn isnumeric(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_numeric) + fn isnumeric(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_numeric) } - fn isdigit(self, _vm: &VirtualMachine) -> bool { + fn isdigit(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, ]; - if self.value.is_empty() { + if zelf.value.is_empty() { false } else { - self.value + zelf.value .chars() .filter(|c| !c.is_digit(10)) .all(|c| valid_unicodes.contains(&(c as u16))) } } - fn isdecimal(self, _vm: &VirtualMachine) -> bool { - if self.value.is_empty() { + fn isdecimal(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + if zelf.value.is_empty() { false } else { - self.value.chars().all(|c| c.is_ascii_digit()) + zelf.value.chars().all(|c| c.is_ascii_digit()) } } - fn title(self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + if args.args.is_empty() { + return Err(vm.new_type_error( + "descriptor 'format' of 'str' object needs an argument".to_string(), + )); + } + + let zelf = &args.args[0]; + if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { + let zelf_typ = zelf.class(); + let actual_type = vm.to_pystr(&zelf_typ)?; + return Err(vm.new_type_error(format!( + "descriptor 'format' requires a 'str' object but received a '{}'", + actual_type + ))); + } + let format_string_text = get_value(zelf); + match FormatString::from_str(format_string_text.as_str()) { + Ok(format_string) => perform_format(vm, &format_string, &args), + Err(err) => match err { + FormatParseError::UnmatchedBracket => { + Err(vm.new_value_error("expected '}' before end of string".to_string())) + } + _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), + }, + } } - fn swapcase(self, _vm: &VirtualMachine) -> String { - let mut swapped_str = String::with_capacity(self.value.len()); - for c in self.value.chars() { + fn title(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + make_title(&zelf.value) + } + + fn swapcase(zelf: PyStringRef, _vm: &VirtualMachine) -> String { + let mut swapped_str = String::with_capacity(zelf.value.len()); + for c in zelf.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { swapped_str.push(c.to_ascii_uppercase()); @@ -318,54 +406,54 @@ impl PyStringRef { swapped_str } - fn isalpha(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) + fn isalpha(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) } fn replace( - self, - old: Self, - new: Self, + zelf: PyStringRef, + old: PyStringRef, + new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { match num.into_option() { - Some(num) => self.value.replacen(&old.value, &new.value, num), - None => self.value.replace(&old.value, &new.value), + Some(num) => zelf.value.replacen(&old.value, &new.value, num), + None => zelf.value.replace(&old.value, &new.value), } } // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) + fn isspace(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() - && self + fn isupper(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() + && zelf .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_uppercase) } - fn islower(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() - && self + fn islower(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() + && zelf .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_lowercase) } - fn isascii(self, _vm: &VirtualMachine) -> bool { - !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) + fn isascii(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { - let elements = self + fn splitlines(zelf: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let elements = zelf .value .split('\n') .map(|e| vm.ctx.new_str(e.to_string())) @@ -373,13 +461,17 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join( + zelf: PyStringRef, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { let elem = elem?; if idx != 0 { - joined.push_str(&self.value); + joined.push_str(&zelf.value); } joined.push_str(&elem.value) } @@ -388,13 +480,13 @@ impl PyStringRef { } fn find( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => (start + num) as isize, @@ -406,13 +498,13 @@ impl PyStringRef { } fn rfind( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => (start + num) as isize, @@ -424,13 +516,13 @@ impl PyStringRef { } fn index( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => Ok(start + num), @@ -442,13 +534,13 @@ impl PyStringRef { } fn rindex( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => Ok(start + num), @@ -459,8 +551,8 @@ impl PyStringRef { } } - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &self.value; + fn partition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &zelf.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -477,8 +569,8 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &self.value; + fn rpartition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &zelf.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -496,31 +588,31 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn istitle(self, _vm: &VirtualMachine) -> bool { - if self.value.is_empty() { + fn istitle(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + if zelf.value.is_empty() { false } else { - self.value.split(' ').all(|word| word == make_title(word)) + zelf.value.split(' ').all(|word| word == make_title(word)) } } fn count( - self, - sub: Self, + zelf: PyStringRef, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> usize { - let value = &self.value; + let value = &zelf.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { - self.value[start..end].matches(&sub.value).count() + zelf.value[start..end].matches(&sub.value).count() } else { 0 } } - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { - let value = &self.value; + fn zfill(zelf: PyStringRef, len: usize, _vm: &VirtualMachine) -> String { + let value = &zelf.value; if len <= value.len() { value.to_string() } else { @@ -528,35 +620,36 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { - let rep_str = match rep { - OptionalArg::Present(ref st) => &st.value, - OptionalArg::Missing => " ", - }; - if rep_str.len() == 1 { - Ok(rep_str) - } else { - Err(vm.new_type_error( - "The fill character must be exactly one character long".to_string(), - )) - } - } - - fn ljust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn ljust( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } - fn rjust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn rjust( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } - fn center(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { - let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + fn center( + zelf: PyStringRef, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let value = &zelf.value; + let rep_char = get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; Ok(format!( @@ -567,12 +660,12 @@ impl PyStringRef { )) } - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(zelf: PyStringRef, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; let mut col_count = 0 as usize; - for ch in self.value.chars() { + for ch in zelf.value.chars() { // 0x0009 is tab if ch == 0x0009 as char { let num_spaces = tab_size - col_count; @@ -590,8 +683,8 @@ impl PyStringRef { expanded_str } - fn isidentifier(self, _vm: &VirtualMachine) -> bool { - let value = &self.value; + fn isidentifier(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + let value = &zelf.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) && !value.chars().nth(0).unwrap().is_digit(10) @@ -623,75 +716,66 @@ impl IntoPyObject for String { #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let str_type = &context.str_type; - let str_doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'."; - extend_class!(context, str_type, { - "__add__" => context.new_rustfunc(PyStringRef::add), - "__bool__" => context.new_rustfunc(PyStringRef::bool), - "__contains__" => context.new_rustfunc(PyStringRef::contains), - "__doc__" => context.new_str(str_doc.to_string()), - "__eq__" => context.new_rustfunc(PyStringRef::eq), - "__ge__" => context.new_rustfunc(PyStringRef::ge), - "__getitem__" => context.new_rustfunc(PyStringRef::getitem), - "__gt__" => context.new_rustfunc(PyStringRef::gt), - "__hash__" => context.new_rustfunc(PyStringRef::hash), - "__lt__" => context.new_rustfunc(PyStringRef::lt), - "__le__" => context.new_rustfunc(PyStringRef::le), - "__len__" => context.new_rustfunc(PyStringRef::len), - "__mul__" => context.new_rustfunc(PyStringRef::mul), - "__new__" => context.new_rustfunc(str_new), - "__repr__" => context.new_rustfunc(PyStringRef::repr), - "__str__" => context.new_rustfunc(PyStringRef::str), - "capitalize" => context.new_rustfunc(PyStringRef::capitalize), - "casefold" => context.new_rustfunc(PyStringRef::casefold), - "center" => context.new_rustfunc(PyStringRef::center), - "count" => context.new_rustfunc(PyStringRef::count), - "endswith" => context.new_rustfunc(PyStringRef::endswith), - "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), - "find" => context.new_rustfunc(PyStringRef::find), - "format" => context.new_rustfunc(str_format), - "index" => context.new_rustfunc(PyStringRef::index), - "isalnum" => context.new_rustfunc(PyStringRef::isalnum), - "isalpha" => context.new_rustfunc(PyStringRef::isalpha), - "isascii" => context.new_rustfunc(PyStringRef::isascii), - "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), - "isdigit" => context.new_rustfunc(PyStringRef::isdigit), - "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), - "islower" => context.new_rustfunc(PyStringRef::islower), - "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), - "isspace" => context.new_rustfunc(PyStringRef::isspace), - "isupper" => context.new_rustfunc(PyStringRef::isupper), - "istitle" => context.new_rustfunc(PyStringRef::istitle), - "join" => context.new_rustfunc(PyStringRef::join), - "lower" => context.new_rustfunc(PyStringRef::lower), - "ljust" => context.new_rustfunc(PyStringRef::ljust), - "lstrip" => context.new_rustfunc(PyStringRef::lstrip), - "partition" => context.new_rustfunc(PyStringRef::partition), - "replace" => context.new_rustfunc(PyStringRef::replace), - "rfind" => context.new_rustfunc(PyStringRef::rfind), - "rindex" => context.new_rustfunc(PyStringRef::rindex), - "rjust" => context.new_rustfunc(PyStringRef::rjust), - "rpartition" => context.new_rustfunc(PyStringRef::rpartition), - "rsplit" => context.new_rustfunc(PyStringRef::rsplit), - "rstrip" => context.new_rustfunc(PyStringRef::rstrip), - "split" => context.new_rustfunc(PyStringRef::split), - "splitlines" => context.new_rustfunc(PyStringRef::splitlines), - "startswith" => context.new_rustfunc(PyStringRef::startswith), - "strip" => context.new_rustfunc(PyStringRef::strip), - "swapcase" => context.new_rustfunc(PyStringRef::swapcase), - "title" => context.new_rustfunc(PyStringRef::title), - "upper" => context.new_rustfunc(PyStringRef::upper), - "zfill" => context.new_rustfunc(PyStringRef::zfill), - }); + PyString::extend_class(context, str_type); + // extend_class!(context, str_type, { + // "__add__" => context.new_rustfunc(PyStringRef::add), + // "__bool__" => context.new_rustfunc(PyStringRef::bool), + // "__contains__" => context.new_rustfunc(PyStringRef::contains), + // "__doc__" => context.new_str(str_doc.to_string()), + // "__eq__" => context.new_rustfunc(PyStringRef::eq), + // "__ge__" => context.new_rustfunc(PyStringRef::ge), + // "__getitem__" => context.new_rustfunc(PyStringRef::getitem), + // "__gt__" => context.new_rustfunc(PyStringRef::gt), + // "__hash__" => context.new_rustfunc(PyStringRef::hash), + // "__lt__" => context.new_rustfunc(PyStringRef::lt), + // "__le__" => context.new_rustfunc(PyStringRef::le), + // "__len__" => context.new_rustfunc(PyStringRef::len), + // "__mul__" => context.new_rustfunc(PyStringRef::mul), + // "__new__" => context.new_rustfunc(str_new), + // "__repr__" => context.new_rustfunc(PyStringRef::repr), + // "__str__" => context.new_rustfunc(PyStringRef::str), + // "capitalize" => context.new_rustfunc(PyStringRef::capitalize), + // "casefold" => context.new_rustfunc(PyStringRef::casefold), + // "center" => context.new_rustfunc(PyStringRef::center), + // "count" => context.new_rustfunc(PyStringRef::count), + // "endswith" => context.new_rustfunc(PyStringRef::endswith), + // "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), + // "find" => context.new_rustfunc(PyStringRef::find), + // "format" => context.new_rustfunc(str_format), + // "index" => context.new_rustfunc(PyStringRef::index), + // "isalnum" => context.new_rustfunc(PyStringRef::isalnum), + // "isalpha" => context.new_rustfunc(PyStringRef::isalpha), + // "isascii" => context.new_rustfunc(PyStringRef::isascii), + // "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), + // "isdigit" => context.new_rustfunc(PyStringRef::isdigit), + // "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), + // "islower" => context.new_rustfunc(PyStringRef::islower), + // "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), + // "isspace" => context.new_rustfunc(PyStringRef::isspace), + // "isupper" => context.new_rustfunc(PyStringRef::isupper), + // "istitle" => context.new_rustfunc(PyStringRef::istitle), + // "join" => context.new_rustfunc(PyStringRef::join), + // "lower" => context.new_rustfunc(PyStringRef::lower), + // "ljust" => context.new_rustfunc(PyStringRef::ljust), + // "lstrip" => context.new_rustfunc(PyStringRef::lstrip), + // "partition" => context.new_rustfunc(PyStringRef::partition), + // "replace" => context.new_rustfunc(PyStringRef::replace), + // "rfind" => context.new_rustfunc(PyStringRef::rfind), + // "rindex" => context.new_rustfunc(PyStringRef::rindex), + // "rjust" => context.new_rustfunc(PyStringRef::rjust), + // "rpartition" => context.new_rustfunc(PyStringRef::rpartition), + // "rsplit" => context.new_rustfunc(PyStringRef::rsplit), + // "rstrip" => context.new_rustfunc(PyStringRef::rstrip), + // "split" => context.new_rustfunc(PyStringRef::split), + // "splitlines" => context.new_rustfunc(PyStringRef::splitlines), + // "startswith" => context.new_rustfunc(PyStringRef::startswith), + // "strip" => context.new_rustfunc(PyStringRef::strip), + // "swapcase" => context.new_rustfunc(PyStringRef::swapcase), + // "title" => context.new_rustfunc(PyStringRef::title), + // "upper" => context.new_rustfunc(PyStringRef::upper), + // "zfill" => context.new_rustfunc(PyStringRef::zfill), + // }); } pub fn get_value(obj: &PyObjectRef) -> String { @@ -706,34 +790,6 @@ fn count_char(s: &str, c: char) -> usize { s.chars().filter(|x| *x == c).count() } -fn str_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - return Err( - vm.new_type_error("descriptor 'format' of 'str' object needs an argument".to_string()) - ); - } - - let zelf = &args.args[0]; - if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { - let zelf_typ = zelf.class(); - let actual_type = vm.to_pystr(&zelf_typ)?; - return Err(vm.new_type_error(format!( - "descriptor 'format' requires a 'str' object but received a '{}'", - actual_type - ))); - } - let format_string_text = get_value(zelf); - match FormatString::from_str(format_string_text.as_str()) { - Ok(format_string) => perform_format(vm, &format_string, &args), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_string())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), - }, - } -} - fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { let returned_type = vm.ctx.new_str(format_spec.to_string()); let result = vm.call_method(&argument, "__format__", vec![returned_type])?; @@ -797,26 +853,6 @@ fn perform_format( Ok(vm.ctx.new_str(final_string)) } -// TODO: should with following format -// class str(object='') -// class str(object=b'', encoding='utf-8', errors='strict') -fn str_new( - cls: PyClassRef, - object: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let string = match object { - OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), - OptionalArg::Missing => vm.new_str("".to_string()), - }; - if string.class().is(&cls) { - TryFromObject::try_from_object(vm, string) - } else { - let payload = string.payload::().unwrap(); - payload.clone().into_ref_with_type(vm, cls) - } -} - impl PySliceableSequence for String { fn do_slice(&self, range: Range) -> Self { to_graphemes(self) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index aa3348be5..a0e737d4c 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1259,6 +1259,26 @@ where } } +pub trait IntoPyClass { + const NAME: &'static str; + const DOC: Option<&'static str> = None; + + fn _extend_class(ctx: &PyContext, class: &PyClassRef); + + fn extend_class(ctx: &PyContext, class: &PyClassRef) { + Self::_extend_class(ctx, class); + if let Some(doc) = Self::DOC { + ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); + } + } + + fn make_class(ctx: &PyContext) -> PyClassRef { + let py_class = ctx.new_class(Self::NAME, ctx.object()); + Self::extend_class(ctx, &py_class); + py_class + } +} + #[cfg(test)] mod tests { use super::*; From ab6031871cff7602e6160817465fbf561df8b597 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 14:51:42 -0500 Subject: [PATCH 006/223] impl PyStringRef using py_class --- derive/src/lib.rs | 2 +- vm/src/obj/objstr.rs | 252 +++++++++++++++++++++---------------------- 2 files changed, 125 insertions(+), 129 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index be16403ef..f3464b5f1 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -374,7 +374,7 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { }| { let constructor_fn = kind.to_ctx_constructor_fn(); quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(#ty::#fn_name)); + ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); } }, ); diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 82ac806e6..66590fdb2 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -74,7 +74,7 @@ fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> encoding defaults to sys.getdefaultencoding().\n\ errors defaults to 'strict'." )] -impl PyString { +impl PyStringRef { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') @@ -96,90 +96,90 @@ impl PyString { } } #[py_class(name = "__add__")] - fn add(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(format!("{}{}", zelf.value, get_value(&rhs))) + Ok(format!("{}{}", self.value, get_value(&rhs))) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot add {} and {}", self, rhs))) } } #[py_class(name = "__bool__")] - fn bool(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() } #[py_class(name = "__eq__")] - fn eq(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - zelf.value == get_value(&rhs) + self.value == get_value(&rhs) } else { false } } #[py_class(name = "__contains__")] - fn contains(zelf: PyStringRef, needle: PyStringRef, _vm: &VirtualMachine) -> bool { - zelf.value.contains(&needle.value) + fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + self.value.contains(&needle.value) } #[py_class(name = "__getitem__")] - fn getitem(zelf: PyStringRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - subscript(vm, &zelf.value, needle) + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + subscript(vm, &self.value, needle) } #[py_class(name = "__gt__")] - fn gt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value > get_value(&rhs)) + Ok(self.value > get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__ge__")] - fn ge(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value >= get_value(&rhs)) + Ok(self.value >= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__lt__")] - fn lt(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value < get_value(&rhs)) + Ok(self.value < get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__le__")] - fn le(zelf: PyStringRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { - Ok(zelf.value <= get_value(&rhs)) + Ok(self.value <= get_value(&rhs)) } else { - Err(vm.new_type_error(format!("Cannot compare {} and {}", zelf, rhs))) + Err(vm.new_type_error(format!("Cannot compare {} and {}", self, rhs))) } } #[py_class(name = "__hash__")] - fn hash(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { + fn hash(self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); - zelf.value.hash(&mut hasher); + self.value.hash(&mut hasher); hasher.finish() as usize } #[py_class(name = "__len__")] - fn len(zelf: PyStringRef, _vm: &VirtualMachine) -> usize { - zelf.value.chars().count() + fn len(self, _vm: &VirtualMachine) -> usize { + self.value.chars().count() } #[py_class(name = "__mul__")] - fn mul(zelf: PyStringRef, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &zelf.value; + let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); let mut result = String::new(); for _x in 0..multiplier { @@ -187,18 +187,18 @@ impl PyString { } Ok(result) } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", zelf, val))) + Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) } } #[py_class(name = "__str__")] - fn str(zelf: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { - zelf + fn str(self, _vm: &VirtualMachine) -> PyStringRef { + self } #[py_class(name = "__repr__")] - fn repr(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let value = &zelf.value; + fn repr(self, _vm: &VirtualMachine) -> String { + let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' } else { @@ -227,31 +227,31 @@ impl PyString { formatted } - fn lower(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.to_lowercase() + fn lower(self, _vm: &VirtualMachine) -> String { + self.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - caseless::default_case_fold_str(&zelf.value) + fn casefold(self, _vm: &VirtualMachine) -> String { + caseless::default_case_fold_str(&self.value) } - fn upper(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.to_uppercase() + fn upper(self, _vm: &VirtualMachine) -> String { + self.value.to_uppercase() } - fn capitalize(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let (first_part, lower_str) = zelf.value.split_at(1); + fn capitalize(self, _vm: &VirtualMachine) -> String { + let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } fn split( - zelf: PyStringRef, + self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &zelf.value; + let value = &self.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -267,12 +267,12 @@ impl PyString { } fn rsplit( - zelf: PyStringRef, + self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - let value = &zelf.value; + let value = &self.value; let pattern = match pattern { OptionalArg::Present(ref s) => &s.value, OptionalArg::Missing => " ", @@ -287,75 +287,75 @@ impl PyString { vm.ctx.new_list(elements) } - fn strip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim().to_string() + fn strip(self, _vm: &VirtualMachine) -> String { + self.value.trim().to_string() } - fn lstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim_start().to_string() + fn lstrip(self, _vm: &VirtualMachine) -> String { + self.value.trim_start().to_string() } - fn rstrip(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - zelf.value.trim_end().to_string() + fn rstrip(self, _vm: &VirtualMachine) -> String { + self.value.trim_end().to_string() } fn endswith( - zelf: PyStringRef, + self, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { - zelf.value[start..end].ends_with(&suffix.value) + if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { + self.value[start..end].ends_with(&suffix.value) } else { false } } fn startswith( - zelf: PyStringRef, + self, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> bool { - if let Some((start, end)) = adjust_indices(start, end, zelf.value.len()) { - zelf.value[start..end].starts_with(&prefix.value) + if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { + self.value[start..end].starts_with(&prefix.value) } else { false } } - fn isalnum(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) + fn isalnum(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } - fn isnumeric(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_numeric) + fn isnumeric(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_numeric) } - fn isdigit(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { + fn isdigit(self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, ]; - if zelf.value.is_empty() { + if self.value.is_empty() { false } else { - zelf.value + self.value .chars() .filter(|c| !c.is_digit(10)) .all(|c| valid_unicodes.contains(&(c as u16))) } } - fn isdecimal(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - if zelf.value.is_empty() { + fn isdecimal(self, _vm: &VirtualMachine) -> bool { + if self.value.is_empty() { false } else { - zelf.value.chars().all(|c| c.is_ascii_digit()) + self.value.chars().all(|c| c.is_ascii_digit()) } } @@ -387,13 +387,13 @@ impl PyString { } } - fn title(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - make_title(&zelf.value) + fn title(self, _vm: &VirtualMachine) -> String { + make_title(&self.value) } - fn swapcase(zelf: PyStringRef, _vm: &VirtualMachine) -> String { - let mut swapped_str = String::with_capacity(zelf.value.len()); - for c in zelf.value.chars() { + fn swapcase(self, _vm: &VirtualMachine) -> String { + let mut swapped_str = String::with_capacity(self.value.len()); + for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { swapped_str.push(c.to_ascii_uppercase()); @@ -406,54 +406,54 @@ impl PyString { swapped_str } - fn isalpha(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(char::is_alphanumeric) + fn isalpha(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } fn replace( - zelf: PyStringRef, + self, old: PyStringRef, new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { match num.into_option() { - Some(num) => zelf.value.replacen(&old.value, &new.value, num), - None => zelf.value.replace(&old.value, &new.value), + Some(num) => self.value.replacen(&old.value, &new.value, num), + None => self.value.replace(&old.value, &new.value), } } // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii_whitespace()) + fn isspace(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() - && zelf + fn isupper(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() + && self .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_uppercase) } - fn islower(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() - && zelf + fn islower(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() + && self .value .chars() .filter(|x| !x.is_ascii_whitespace()) .all(char::is_lowercase) } - fn isascii(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - !zelf.value.is_empty() && zelf.value.chars().all(|c| c.is_ascii()) + fn isascii(self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(zelf: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let elements = zelf + fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + let elements = self .value .split('\n') .map(|e| vm.ctx.new_str(e.to_string())) @@ -461,17 +461,13 @@ impl PyString { vm.ctx.new_list(elements) } - fn join( - zelf: PyStringRef, - iterable: PyIterable, - vm: &VirtualMachine, - ) -> PyResult { + fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { let elem = elem?; if idx != 0 { - joined.push_str(&zelf.value); + joined.push_str(&self.value); } joined.push_str(&elem.value) } @@ -480,13 +476,13 @@ impl PyString { } fn find( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => (start + num) as isize, @@ -498,13 +494,13 @@ impl PyString { } fn rfind( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> isize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => (start + num) as isize, @@ -516,13 +512,13 @@ impl PyString { } fn index( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].find(&sub.value) { Some(num) => Ok(start + num), @@ -534,13 +530,13 @@ impl PyString { } fn rindex( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { match value[start..end].rfind(&sub.value) { Some(num) => Ok(start + num), @@ -551,8 +547,8 @@ impl PyString { } } - fn partition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &zelf.value; + fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -569,8 +565,8 @@ impl PyString { vm.ctx.new_tuple(new_tup) } - fn rpartition(zelf: PyStringRef, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { - let value = &zelf.value; + fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); if value.contains(sub) { @@ -588,31 +584,31 @@ impl PyString { vm.ctx.new_tuple(new_tup) } - fn istitle(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - if zelf.value.is_empty() { + fn istitle(self, _vm: &VirtualMachine) -> bool { + if self.value.is_empty() { false } else { - zelf.value.split(' ').all(|word| word == make_title(word)) + self.value.split(' ').all(|word| word == make_title(word)) } } fn count( - zelf: PyStringRef, + self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, ) -> usize { - let value = &zelf.value; + let value = &self.value; if let Some((start, end)) = adjust_indices(start, end, value.len()) { - zelf.value[start..end].matches(&sub.value).count() + self.value[start..end].matches(&sub.value).count() } else { 0 } } - fn zfill(zelf: PyStringRef, len: usize, _vm: &VirtualMachine) -> String { - let value = &zelf.value; + fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + let value = &self.value; if len <= value.len() { value.to_string() } else { @@ -621,34 +617,34 @@ impl PyString { } fn ljust( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } fn rjust( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } fn center( - zelf: PyStringRef, + self, len: usize, rep: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let value = &zelf.value; + let value = &self.value; let rep_char = get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; @@ -660,12 +656,12 @@ impl PyString { )) } - fn expandtabs(zelf: PyStringRef, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; let mut col_count = 0 as usize; - for ch in zelf.value.chars() { + for ch in self.value.chars() { // 0x0009 is tab if ch == 0x0009 as char { let num_spaces = tab_size - col_count; @@ -683,8 +679,8 @@ impl PyString { expanded_str } - fn isidentifier(zelf: PyStringRef, _vm: &VirtualMachine) -> bool { - let value = &zelf.value; + fn isidentifier(self, _vm: &VirtualMachine) -> bool { + let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) && !value.chars().nth(0).unwrap().is_digit(10) @@ -717,7 +713,7 @@ impl IntoPyObject for String { pub fn init(context: &PyContext) { let str_type = &context.str_type; - PyString::extend_class(context, str_type); + PyStringRef::extend_class(context, str_type); // extend_class!(context, str_type, { // "__add__" => context.new_rustfunc(PyStringRef::add), // "__bool__" => context.new_rustfunc(PyStringRef::bool), From 8376ec2d982bbe90ace709fe7585298f72743871 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 15:17:58 -0500 Subject: [PATCH 007/223] Use doc comments for python __doc__ --- derive/src/lib.rs | 31 +++++++++++++++++++++++-------- vm/src/obj/objstr.rs | 25 +++++++++++-------------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index f3464b5f1..52cfc7567 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -340,7 +340,6 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { }; let rp_path = rustpython_path_attr(&attr); let mut class_name = None; - let mut doc = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { if let Meta::NameValue(name_value) = meta { @@ -350,19 +349,35 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } else { panic!("#[py_class(name = ...)] must be a string"); } - } else if name_value.ident == "doc" { - if let Lit::Str(s) = name_value.lit { - doc = Some(s.value()); - } else { - panic!("#[py_class(name = ...)] must be a string"); - } } } } } let class_name = class_name.expect("#[py_class] must have a name"); + let mut doc: Option> = None; + for attr in imp.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } else { + panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); + } + } + } let doc = match doc { - Some(doc) => quote!(Some(#doc)), + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } None => quote!(None), }; let ty = &imp.self_ty; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 66590fdb2..5e4035c5a 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -60,20 +60,17 @@ fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> } } -#[py_class( - __inside_vm, - name = "str", - doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'." -)] +#[py_class(__inside_vm, name = "str")] +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." impl PyStringRef { // TODO: should with following format // class str(object='') From d9efe4ea3752b8c0f460786d487cacd74f62e772 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 29 Mar 2019 16:16:31 -0500 Subject: [PATCH 008/223] Require a #[pymethod] for a method to be recognized --- derive/src/lib.rs | 62 ++++++++++++++++---------- vm/src/obj/objstr.rs | 104 ++++++++++++++++++++++++++++++------------- 2 files changed, 112 insertions(+), 54 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 52cfc7567..374463f06 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -249,7 +249,7 @@ fn impl_from_args(input: DeriveInput) -> TokenStream2 { } #[proc_macro_attribute] -pub fn py_class(attr: TokenStream, item: TokenStream) -> TokenStream { +pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); impl_py_class(attr, item).into() @@ -276,31 +276,37 @@ struct Method { kind: MethodKind, } -fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter().filter_map(|item| { +/// Parse an impl block into an iterator of methods +fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter_mut().filter_map(|item| { if let ImplItem::Method(meth) = item { let mut py_name = None; let mut kind = MethodKind::Method; - let metas_iter = meth + let mut pymethod_to_remove = Vec::new(); + let metas = meth .attrs .iter() - .filter_map(|attr| { - if attr.path.is_ident("py_class") { + .enumerate() + .filter_map(|(i, attr)| { + if attr.path.is_ident("pymethod") { let meta = attr.parse_meta().expect("Invalid attribute"); - if let Meta::List(list) = meta { - Some(list) - } else { - panic!( - "#[py_class] attribute on a method should be a list, like \ - #[py_class(...)]" - ) + // remove #[pymethod] because there's no actual proc macro + // implementation for it + pymethod_to_remove.push(i); + match meta { + Meta::List(list) => Some(list), + Meta::Word(_) => None, + Meta::NameValue(_) => panic!( + "#[pymethod = ...] attribute on a method should be a list, like \ + #[pymethod(...)]" + ), } } else { None } }) .flat_map(|attr| attr.nested); - for meta in metas_iter { + for meta in metas { if let NestedMeta::Meta(meta) = meta { match meta { Meta::NameValue(name_value) => { @@ -308,18 +314,27 @@ fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator match ident.to_string().as_str() { - "property" => kind = MethodKind::Property, - _ => {} - }, + Meta::Word(ident) => { + if ident == "property" { + kind = MethodKind::Property + } + } _ => {} } } } + // if there are no #[pymethods]s, then it's not a method, so exclude it from + // the final result + if pymethod_to_remove.is_empty() { + return None; + } + for i in pymethod_to_remove { + meth.attrs.remove(i); + } let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); Some(Method { fn_name: meth.sig.ident.clone(), @@ -333,7 +348,7 @@ fn item_impl_to_methods<'a>(imp: &'a syn::ItemImpl) -> impl Iterator TokenStream2 { - let imp = if let Item::Impl(imp) = item { + let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); @@ -347,13 +362,13 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { if let Lit::Str(s) = name_value.lit { class_name = Some(s.value()); } else { - panic!("#[py_class(name = ...)] must be a string"); + panic!("#[pyclass(name = ...)] must be a string"); } } } } } - let class_name = class_name.expect("#[py_class] must have a name"); + let class_name = class_name.expect("#[pyclass] must have a name"); let mut doc: Option> = None; for attr in imp.attrs.iter() { if attr.path.is_ident("doc") { @@ -380,8 +395,9 @@ fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } None => quote!(None), }; + let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); let ty = &imp.self_ty; - let methods = item_impl_to_methods(&imp).map( + let methods = methods.iter().map( |Method { py_name, fn_name, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 5e4035c5a..70c420665 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -48,19 +48,7 @@ impl TryIntoRef for &str { } } -fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { - let rep_str = match rep { - OptionalArg::Present(ref st) => &st.value, - OptionalArg::Missing => " ", - }; - if rep_str.len() == 1 { - Ok(rep_str) - } else { - Err(vm.new_type_error("The fill character must be exactly one character long".to_string())) - } -} - -#[py_class(__inside_vm, name = "str")] +#[pyclass(__inside_vm, name = "str")] /// str(object='') -> str /// str(bytes_or_buffer[, encoding[, errors]]) -> str /// @@ -75,7 +63,7 @@ impl PyStringRef { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') - #[py_class(name = "__new__")] + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, object: OptionalArg, @@ -92,7 +80,7 @@ impl PyStringRef { payload.clone().into_ref_with_type(vm, cls) } } - #[py_class(name = "__add__")] + #[pymethod(name = "__add__")] fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) @@ -101,12 +89,12 @@ impl PyStringRef { } } - #[py_class(name = "__bool__")] + #[pymethod(name = "__bool__")] fn bool(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() } - #[py_class(name = "__eq__")] + #[pymethod(name = "__eq__")] fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) @@ -115,17 +103,17 @@ impl PyStringRef { } } - #[py_class(name = "__contains__")] + #[pymethod(name = "__contains__")] fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } - #[py_class(name = "__getitem__")] + #[pymethod(name = "__getitem__")] fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } - #[py_class(name = "__gt__")] + #[pymethod(name = "__gt__")] fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) @@ -134,7 +122,7 @@ impl PyStringRef { } } - #[py_class(name = "__ge__")] + #[pymethod(name = "__ge__")] fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) @@ -143,7 +131,7 @@ impl PyStringRef { } } - #[py_class(name = "__lt__")] + #[pymethod(name = "__lt__")] fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) @@ -152,7 +140,7 @@ impl PyStringRef { } } - #[py_class(name = "__le__")] + #[pymethod(name = "__le__")] fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) @@ -161,19 +149,19 @@ impl PyStringRef { } } - #[py_class(name = "__hash__")] + #[pymethod(name = "__hash__")] fn hash(self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() as usize } - #[py_class(name = "__len__")] + #[pymethod(name = "__len__")] fn len(self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } - #[py_class(name = "__mul__")] + #[pymethod(name = "__mul__")] fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; @@ -188,12 +176,12 @@ impl PyStringRef { } } - #[py_class(name = "__str__")] + #[pymethod(name = "__str__")] fn str(self, _vm: &VirtualMachine) -> PyStringRef { self } - #[py_class(name = "__repr__")] + #[pymethod(name = "__repr__")] fn repr(self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { @@ -224,24 +212,29 @@ impl PyStringRef { formatted } + #[pymethod] fn lower(self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower + #[pymethod] fn casefold(self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } + #[pymethod] fn upper(self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } + #[pymethod] fn capitalize(self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } + #[pymethod] fn split( self, pattern: OptionalArg, @@ -263,6 +256,7 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn rsplit( self, pattern: OptionalArg, @@ -284,18 +278,22 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn strip(self, _vm: &VirtualMachine) -> String { self.value.trim().to_string() } + #[pymethod] fn lstrip(self, _vm: &VirtualMachine) -> String { self.value.trim_start().to_string() } + #[pymethod] fn rstrip(self, _vm: &VirtualMachine) -> String { self.value.trim_end().to_string() } + #[pymethod] fn endswith( self, suffix: PyStringRef, @@ -310,6 +308,7 @@ impl PyStringRef { } } + #[pymethod] fn startswith( self, prefix: PyStringRef, @@ -324,14 +323,17 @@ impl PyStringRef { } } + #[pymethod] fn isalnum(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn isnumeric(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } + #[pymethod] fn isdigit(self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ @@ -348,6 +350,7 @@ impl PyStringRef { } } + #[pymethod] fn isdecimal(self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false @@ -356,6 +359,7 @@ impl PyStringRef { } } + #[pymethod] fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if args.args.is_empty() { return Err(vm.new_type_error( @@ -384,10 +388,12 @@ impl PyStringRef { } } + #[pymethod] fn title(self, _vm: &VirtualMachine) -> String { make_title(&self.value) } + #[pymethod] fn swapcase(self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { @@ -403,10 +409,12 @@ impl PyStringRef { swapped_str } + #[pymethod] fn isalpha(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn replace( self, old: PyStringRef, @@ -422,10 +430,12 @@ impl PyStringRef { // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower + #[pymethod] fn isspace(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } + #[pymethod] fn isupper(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self @@ -435,6 +445,7 @@ impl PyStringRef { .all(char::is_uppercase) } + #[pymethod] fn islower(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self @@ -444,11 +455,13 @@ impl PyStringRef { .all(char::is_lowercase) } + #[pymethod] fn isascii(self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet + #[pymethod] fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value @@ -458,6 +471,7 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); @@ -472,6 +486,7 @@ impl PyStringRef { Ok(joined) } + #[pymethod] fn find( self, sub: PyStringRef, @@ -490,6 +505,7 @@ impl PyStringRef { } } + #[pymethod] fn rfind( self, sub: PyStringRef, @@ -508,6 +524,7 @@ impl PyStringRef { } } + #[pymethod] fn index( self, sub: PyStringRef, @@ -526,6 +543,7 @@ impl PyStringRef { } } + #[pymethod] fn rindex( self, sub: PyStringRef, @@ -544,6 +562,7 @@ impl PyStringRef { } } + #[pymethod] fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; @@ -562,6 +581,7 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } + #[pymethod] fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; @@ -581,6 +601,7 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } + #[pymethod] fn istitle(self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false @@ -589,6 +610,7 @@ impl PyStringRef { } } + #[pymethod] fn count( self, sub: PyStringRef, @@ -604,6 +626,7 @@ impl PyStringRef { } } + #[pymethod] fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { @@ -613,6 +636,21 @@ impl PyStringRef { } } + fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + let rep_str = match rep { + OptionalArg::Present(ref st) => &st.value, + OptionalArg::Missing => " ", + }; + if rep_str.len() == 1 { + Ok(rep_str) + } else { + Err(vm.new_type_error( + "The fill character must be exactly one character long".to_string(), + )) + } + } + + #[pymethod] fn ljust( self, len: usize, @@ -620,10 +658,11 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } + #[pymethod] fn rjust( self, len: usize, @@ -631,10 +670,11 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } + #[pymethod] fn center( self, len: usize, @@ -642,7 +682,7 @@ impl PyStringRef { vm: &VirtualMachine, ) -> PyResult { let value = &self.value; - let rep_char = get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; let left_buff: usize = (len - value.len()) / 2; let right_buff = len - value.len() - left_buff; Ok(format!( @@ -653,6 +693,7 @@ impl PyStringRef { )) } + #[pymethod] fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); @@ -676,6 +717,7 @@ impl PyStringRef { expanded_str } + #[pymethod] fn isidentifier(self, _vm: &VirtualMachine) -> bool { let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number From c2d04f97d80a0eb2916ac7c4ce630b43a528f26b Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 30 Mar 2019 13:33:50 -0700 Subject: [PATCH 009/223] bytes: add tests for NotImplemented --- tests/snippets/bytes.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/snippets/bytes.py diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py new file mode 100644 index 000000000..19ebef68a --- /dev/null +++ b/tests/snippets/bytes.py @@ -0,0 +1,6 @@ +assert b'foobar'.__eq__(2) == NotImplemented +assert b'foobar'.__ne__(2) == NotImplemented +assert b'foobar'.__gt__(2) == NotImplemented +assert b'foobar'.__ge__(2) == NotImplemented +assert b'foobar'.__lt__(2) == NotImplemented +assert b'foobar'.__le__(2) == NotImplemented From 7d18a1ce604a8a36d57e3b3caf34c7816ee545b3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 30 Mar 2019 21:47:20 +0100 Subject: [PATCH 010/223] Fix #746: invalid slice with start or stop =-1 when step<0 --- tests/snippets/builtin_slice.py | 75 +- tests/snippets/slice_res.py | 2252 +++++++++++++++++++++++++++++++ vm/src/obj/objsequence.rs | 21 +- 3 files changed, 2334 insertions(+), 14 deletions(-) create mode 100644 tests/snippets/slice_res.py diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 1d8a93b47..619356211 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -2,24 +2,24 @@ from testutils import assert_raises a = [] assert a[:] == [] -assert a[:2**100] == [] -assert a[-2**100:] == [] -assert a[::2**100] == [] +assert a[: 2 ** 100] == [] +assert a[-2 ** 100 :] == [] +assert a[:: 2 ** 100] == [] assert a[10:20] == [] assert a[-20:-10] == [] b = [1, 2] assert b[:] == [1, 2] -assert b[:2**100] == [1, 2] -assert b[-2**100:] == [1, 2] -assert b[2**100:] == [] -assert b[::2**100] == [1] +assert b[: 2 ** 100] == [1, 2] +assert b[-2 ** 100 :] == [1, 2] +assert b[2 ** 100 :] == [] +assert b[:: 2 ** 100] == [1] assert b[-10:1] == [1] assert b[0:0] == [] assert b[1:0] == [] -assert_raises(ValueError, lambda: b[::0], 'zero step slice') +assert_raises(ValueError, lambda: b[::0], "zero step slice") assert b[::-1] == [2, 1] assert b[1::-1] == [2, 1] @@ -33,7 +33,7 @@ c = list(range(10)) assert c[9:6:-3] == [9] assert c[9::-3] == [9, 6, 3, 0] assert c[9::-4] == [9, 5, 1] -assert c[8::-2**100] == [8] +assert c[8 :: -2 ** 100] == [8] assert c[7:7:-2] == [] assert c[7:8:-2] == [] @@ -43,6 +43,7 @@ d = "123456" assert d[3::-1] == "4321" assert d[4::-3] == "52" +assert [1, 2, 3, 5, 6][-1:-5:-1] == [6, 5, 3, 2] # #746 slice_a = slice(5) assert slice_a.start is None @@ -71,3 +72,59 @@ class SubScript(object): ss = SubScript() _ = ss[:] ss[:1] = 1 + + +def test_all_slices(): + """ + test all possible slices except big number + """ + MODE = None # set to "build" to rebuild slice_res.py + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + + for i in [start, end, step]: + i.append(None) + + def build(): + # loop used to build slices_res.py with cpython + with open("slice_res.py", "wt") as f: + for s in start: + for e in end: + for t in step: + f.write(str(ll[s:e:t]) + "\n") + + def run(): + # test utility + from slice_res import SLICES_RES + + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == SLICES_RES[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, SLICES_RES[count] + ) + ) + count += 1 + + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") + + if MODE == "build": + build() + else: + run() + + +test_all_slices() diff --git a/tests/snippets/slice_res.py b/tests/snippets/slice_res.py new file mode 100644 index 000000000..38eb87729 --- /dev/null +++ b/tests/snippets/slice_res.py @@ -0,0 +1,2252 @@ +SLICES_RES = [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [2, 3], + [2], + [2], + [2], + [2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [3], + [3], + [3], + [3], + [3], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [0], + [0], + [0], + [0], + [0], + [], + [], + [], + [], + [], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [0], + [0], + [0], + [0], + [0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1, 0], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [1], + [1], + [1], + [1], + [1], + [], + [], + [], + [], + [], + [1, 2], + [1], + [1], + [1], + [1, 2], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [], + [], + [], + [], + [], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [1], + [1], + [1], + [1], + [1, 0], + [1, 2, 3], + [1, 3], + [1], + [1], + [1, 2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2], + [2, 1], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [2], + [2], + [2], + [2], + [2], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [], + [], + [], + [], + [], + [2, 3], + [2], + [2], + [2], + [2, 3], + [2], + [2], + [2], + [2, 0], + [2, 1, 0], + [2, 3], + [2], + [2], + [2], + [2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [3], + [3], + [3], + [3], + [3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [0], + [0], + [0], + [0], + [0], + [3], + [3], + [3], + [3], + [3], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [3], + [3], + [3], + [3, 1], + [3, 2, 1], + [], + [], + [], + [], + [], + [3], + [3], + [3], + [3], + [3, 2], + [0], + [0], + [0], + [0], + [0], + [3], + [3], + [3], + [3], + [3], + [0, 1], + [0], + [0], + [0], + [0, 1], + [], + [], + [], + [], + [], + [0, 1, 2], + [0, 2], + [0], + [0], + [0, 1, 2], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [], + [], + [], + [], + [], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], + [3], + [3], + [3, 0], + [3, 1], + [3, 2, 1, 0], + [0, 1, 2, 3], + [0, 2], + [0, 3], + [0], + [0, 1, 2, 3], +] diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a07b2454d..55c56d47c 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -2,11 +2,10 @@ use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; - use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use super::objbool; use super::objint::PyInt; @@ -86,8 +85,20 @@ pub trait PySliceableSequence { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { From f11f04da709d564114d8fcaec07abb130c7de214 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 18:43:31 -0500 Subject: [PATCH 011/223] Split rustpython_derive into multiple files --- derive/src/from_args.rs | 202 +++++++++++++++++++++ derive/src/lib.rs | 382 +--------------------------------------- derive/src/pyclass.rs | 179 +++++++++++++++++++ vm/src/lib.rs | 4 +- 4 files changed, 390 insertions(+), 377 deletions(-) create mode 100644 derive/src/from_args.rs create mode 100644 derive/src/pyclass.rs diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs new file mode 100644 index 000000000..7845b2880 --- /dev/null +++ b/derive/src/from_args.rs @@ -0,0 +1,202 @@ +use super::rustpython_path_derive; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; + +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +enum ParameterKind { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, +} + +impl ParameterKind { + fn from_ident(ident: &Ident) -> ParameterKind { + if ident == "positional_only" { + ParameterKind::PositionalOnly + } else if ident == "positional_or_keyword" { + ParameterKind::PositionalOrKeyword + } else if ident == "keyword_only" { + ParameterKind::KeywordOnly + } else { + panic!("Unrecognised attribute") + } + } +} + +struct ArgAttribute { + kind: ParameterKind, + default: Option, + optional: bool, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option { + if !attr.path.is_ident("pyarg") { + return None; + } + + match attr.parse_meta().unwrap() { + Meta::List(list) => { + let mut iter = list.nested.iter(); + let first_arg = iter.next().expect("at least one argument in pyarg list"); + let kind = match first_arg { + NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + + let mut attribute = ArgAttribute { + kind, + default: None, + optional: false, + }; + + while let Some(arg) = iter.next() { + attribute.parse_argument(arg); + } + + assert!( + attribute.default.is_none() || !attribute.optional, + "Can't set both a default value and optional" + ); + + Some(attribute) + } + _ => panic!("Bad syntax for pyarg attribute"), + } + } + + fn parse_argument(&mut self, arg: &NestedMeta) { + match arg { + NestedMeta::Meta(Meta::Word(ident)) => { + if ident == "default" { + assert!(self.default.is_none(), "Default already set"); + let expr = syn::parse_str::("Default::default()").unwrap(); + self.default = Some(expr); + } else if ident == "optional" { + self.optional = true; + } else { + panic!("Unrecognised pyarg attribute '{}'", ident); + } + } + NestedMeta::Meta(Meta::NameValue(name_value)) => { + if name_value.ident == "default" { + assert!(self.default.is_none(), "Default already set"); + + match name_value.lit { + Lit::Str(ref val) => { + let expr = val + .parse::() + .expect("a valid expression for default argument"); + self.default = Some(expr); + } + _ => panic!("Expected string value for default argument"), + } + } else if name_value.ident == "optional" { + match name_value.lit { + Lit::Bool(ref val) => { + self.optional = val.value; + } + _ => panic!("Expected boolean value for optional argument"), + } + } else { + panic!("Unrecognised pyarg attribute '{}'", name_value.ident); + } + } + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + } +} + +fn generate_field(field: &Field) -> TokenStream2 { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(ArgAttribute::from_attribute) + .collect::>(); + let attr = if pyarg_attrs.is_empty() { + ArgAttribute { + kind: ParameterKind::PositionalOrKeyword, + default: None, + optional: false, + } + } else if pyarg_attrs.len() == 1 { + pyarg_attrs.remove(0) + } else { + panic!( + "Multiple pyarg attributes on field '{}'", + field.ident.as_ref().unwrap() + ); + }; + + let name = &field.ident; + let middle = quote! { + .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + }; + let ending = if let Some(default) = attr.default { + quote! { + .unwrap_or_else(|| #default) + } + } else if attr.optional { + quote! { + .map(crate::function::OptionalArg::Present) + .unwrap_or(crate::function::OptionalArg::Missing) + } + } else { + let err = match attr.kind { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { + crate::function::ArgumentError::TooFewArgs + }, + ParameterKind::KeywordOnly => quote! { + crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + }, + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + match attr.kind { + ParameterKind::PositionalOnly => { + quote! { + #name: args.take_positional()#middle#ending, + } + } + ParameterKind::PositionalOrKeyword => { + quote! { + #name: args.take_positional_keyword(stringify!(#name))#middle#ending, + } + } + ParameterKind::KeywordOnly => { + quote! { + #name: args.take_keyword(stringify!(#name))#middle#ending, + } + } + } +} + +pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { + let rp_path = rustpython_path_derive(&input); + let fields = match input.data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => fields.named.iter().map(generate_field), + Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message + }; + + let name = &input.ident; + quote! { + impl #rp_path::function::FromArgs for #name { + fn from_args( + vm: &crate::vm::VirtualMachine, + args: &mut crate::function::PyFuncArgs + ) -> Result { + Ok(#name { #(#fields)* }) + } + } + } +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 374463f06..5a26030e4 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,12 +1,11 @@ extern crate proc_macro; use proc_macro::TokenStream; -use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; -use syn::{ - parse_macro_input, Attribute, AttributeArgs, Data, DeriveInput, Expr, Field, Fields, Ident, - ImplItem, Item, Lit, Meta, NestedMeta, -}; +use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; + +mod from_args; +mod pyclass; fn rustpython_path(inside_vm: bool) -> syn::Path { let path = if inside_vm { @@ -46,381 +45,12 @@ fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); - let gen = impl_from_args(ast); - gen.to_string().parse().unwrap() -} - -/// The kind of the python parameter, this corresponds to the value of Parameter.kind -/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) -enum ParameterKind { - PositionalOnly, - PositionalOrKeyword, - KeywordOnly, -} - -impl ParameterKind { - fn from_ident(ident: &Ident) -> ParameterKind { - if ident == "positional_only" { - ParameterKind::PositionalOnly - } else if ident == "positional_or_keyword" { - ParameterKind::PositionalOrKeyword - } else if ident == "keyword_only" { - ParameterKind::KeywordOnly - } else { - panic!("Unrecognised attribute") - } - } -} - -struct ArgAttribute { - kind: ParameterKind, - default: Option, - optional: bool, -} - -impl ArgAttribute { - fn from_attribute(attr: &Attribute) -> Option { - if !attr.path.is_ident("pyarg") { - return None; - } - - match attr.parse_meta().unwrap() { - Meta::List(list) => { - let mut iter = list.nested.iter(); - let first_arg = iter.next().expect("at least one argument in pyarg list"); - let kind = match first_arg { - NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - - let mut attribute = ArgAttribute { - kind, - default: None, - optional: false, - }; - - while let Some(arg) = iter.next() { - attribute.parse_argument(arg); - } - - assert!( - attribute.default.is_none() || !attribute.optional, - "Can't set both a default value and optional" - ); - - Some(attribute) - } - _ => panic!("Bad syntax for pyarg attribute"), - } - } - - fn parse_argument(&mut self, arg: &NestedMeta) { - match arg { - NestedMeta::Meta(Meta::Word(ident)) => { - if ident == "default" { - assert!(self.default.is_none(), "Default already set"); - let expr = syn::parse_str::("Default::default()").unwrap(); - self.default = Some(expr); - } else if ident == "optional" { - self.optional = true; - } else { - panic!("Unrecognised pyarg attribute '{}'", ident); - } - } - NestedMeta::Meta(Meta::NameValue(name_value)) => { - if name_value.ident == "default" { - assert!(self.default.is_none(), "Default already set"); - - match name_value.lit { - Lit::Str(ref val) => { - let expr = val - .parse::() - .expect("a valid expression for default argument"); - self.default = Some(expr); - } - _ => panic!("Expected string value for default argument"), - } - } else if name_value.ident == "optional" { - match name_value.lit { - Lit::Bool(ref val) => { - self.optional = val.value; - } - _ => panic!("Expected boolean value for optional argument"), - } - } else { - panic!("Unrecognised pyarg attribute '{}'", name_value.ident); - } - } - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - } -} - -fn generate_field(field: &Field) -> TokenStream2 { - let mut pyarg_attrs = field - .attrs - .iter() - .filter_map(ArgAttribute::from_attribute) - .collect::>(); - let attr = if pyarg_attrs.is_empty() { - ArgAttribute { - kind: ParameterKind::PositionalOrKeyword, - default: None, - optional: false, - } - } else if pyarg_attrs.len() == 1 { - pyarg_attrs.remove(0) - } else { - panic!( - "Multiple pyarg attributes on field '{}'", - field.ident.as_ref().unwrap() - ); - }; - - let name = &field.ident; - let middle = quote! { - .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? - }; - let ending = if let Some(default) = attr.default { - quote! { - .unwrap_or_else(|| #default) - } - } else if attr.optional { - quote! { - .map(crate::function::OptionalArg::Present) - .unwrap_or(crate::function::OptionalArg::Missing) - } - } else { - let err = match attr.kind { - ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - crate::function::ArgumentError::TooFewArgs - }, - ParameterKind::KeywordOnly => quote! { - crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) - }, - }; - quote! { - .ok_or_else(|| #err)? - } - }; - - match attr.kind { - ParameterKind::PositionalOnly => { - quote! { - #name: args.take_positional()#middle#ending, - } - } - ParameterKind::PositionalOrKeyword => { - quote! { - #name: args.take_positional_keyword(stringify!(#name))#middle#ending, - } - } - ParameterKind::KeywordOnly => { - quote! { - #name: args.take_keyword(stringify!(#name))#middle#ending, - } - } - } -} - -fn impl_from_args(input: DeriveInput) -> TokenStream2 { - let rp_path = rustpython_path_derive(&input); - let fields = match input.data { - Data::Struct(ref data) => { - match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(generate_field), - Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message - } - } - Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message - }; - - let name = &input.ident; - quote! { - impl #rp_path::function::FromArgs for #name { - fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { - Ok(#name { #(#fields)* }) - } - } - } + from_args::impl_from_args(ast).into() } #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - impl_py_class(attr, item).into() -} - -enum MethodKind { - Method, - Property, -} - -impl MethodKind { - fn to_ctx_constructor_fn(&self) -> Ident { - let f = match self { - MethodKind::Method => "new_rustfunc", - MethodKind::Property => "new_property", - }; - Ident::new(f, Span::call_site()) - } -} - -struct Method { - fn_name: Ident, - py_name: String, - kind: MethodKind, -} - -/// Parse an impl block into an iterator of methods -fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter_mut().filter_map(|item| { - if let ImplItem::Method(meth) = item { - let mut py_name = None; - let mut kind = MethodKind::Method; - let mut pymethod_to_remove = Vec::new(); - let metas = meth - .attrs - .iter() - .enumerate() - .filter_map(|(i, attr)| { - if attr.path.is_ident("pymethod") { - let meta = attr.parse_meta().expect("Invalid attribute"); - // remove #[pymethod] because there's no actual proc macro - // implementation for it - pymethod_to_remove.push(i); - match meta { - Meta::List(list) => Some(list), - Meta::Word(_) => None, - Meta::NameValue(_) => panic!( - "#[pymethod = ...] attribute on a method should be a list, like \ - #[pymethod(...)]" - ), - } - } else { - None - } - }) - .flat_map(|attr| attr.nested); - for meta in metas { - if let NestedMeta::Meta(meta) = meta { - match meta { - Meta::NameValue(name_value) => { - if name_value.ident == "name" { - if let Lit::Str(s) = &name_value.lit { - py_name = Some(s.value()); - } else { - panic!("#[pymethod(name = ...)] must be a string"); - } - } - } - Meta::Word(ident) => { - if ident == "property" { - kind = MethodKind::Property - } - } - _ => {} - } - } - } - // if there are no #[pymethods]s, then it's not a method, so exclude it from - // the final result - if pymethod_to_remove.is_empty() { - return None; - } - for i in pymethod_to_remove { - meth.attrs.remove(i); - } - let py_name = py_name.unwrap_or_else(|| meth.sig.ident.to_string()); - Some(Method { - fn_name: meth.sig.ident.clone(), - py_name, - kind, - }) - } else { - None - } - }) -} - -fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { - let mut imp = if let Item::Impl(imp) = item { - imp - } else { - return quote!(#item); - }; - let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; - for attr in attr { - if let NestedMeta::Meta(meta) = attr { - if let Meta::NameValue(name_value) = meta { - if name_value.ident == "name" { - if let Lit::Str(s) = name_value.lit { - class_name = Some(s.value()); - } else { - panic!("#[pyclass(name = ...)] must be a string"); - } - } - } - } - } - let class_name = class_name.expect("#[pyclass] must have a name"); - let mut doc: Option> = None; - for attr in imp.attrs.iter() { - if attr.path.is_ident("doc") { - let meta = attr.parse_meta().expect("expected doc attr to be a meta"); - if let Meta::NameValue(name_value) = meta { - if let Lit::Str(s) = name_value.lit { - let val = s.value().trim().to_string(); - match doc { - Some(ref mut doc) => doc.push(val), - None => doc = Some(vec![val]), - } - } else { - panic!("expected #[doc = ...] to be a string") - } - } else { - panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); - } - } - } - let doc = match doc { - Some(doc) => { - let doc = doc.join("\n"); - quote!(Some(#doc)) - } - None => quote!(None), - }; - let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); - let ty = &imp.self_ty; - let methods = methods.iter().map( - |Method { - py_name, - fn_name, - kind, - }| { - let constructor_fn = kind.to_ctx_constructor_fn(); - quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); - } - }, - ); - - quote! { - #imp - impl #rp_path::pyobject::IntoPyClass for #ty { - const NAME: &'static str = #class_name; - const DOC: Option<&'static str> = #doc; - fn _extend_class( - ctx: &#rp_path::pyobject::PyContext, - class: &#rp_path::obj::objtype::PyClassRef, - ) { - #(#methods)* - } - } - } + pyclass::impl_py_class(attr, item).into() } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs new file mode 100644 index 000000000..4497424f2 --- /dev/null +++ b/derive/src/pyclass.rs @@ -0,0 +1,179 @@ +use super::rustpython_path_attr; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; + +enum MethodKind { + Method, + Property, +} + +impl MethodKind { + fn to_ctx_constructor_fn(&self) -> Ident { + let f = match self { + MethodKind::Method => "new_rustfunc", + MethodKind::Property => "new_property", + }; + Ident::new(f, Span::call_site()) + } +} + +struct Method { + fn_name: Ident, + py_name: String, + kind: MethodKind, +} + +impl Method { + fn from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + let mut py_name = None; + let mut kind = MethodKind::Method; + let mut pymethod_to_remove = Vec::new(); + let metas = attrs + .iter() + .enumerate() + .filter_map(|(i, attr)| { + if attr.path.is_ident("pymethod") { + let meta = attr.parse_meta().expect("Invalid attribute"); + // remove #[pymethod] because there's no actual proc macro + // implementation for it + pymethod_to_remove.push(i); + match meta { + Meta::List(list) => Some(list), + Meta::Word(_) => None, + Meta::NameValue(_) => panic!( + "#[pymethod = ...] attribute on a method should be a list, like \ + #[pymethod(...)]" + ), + } + } else { + None + } + }) + .flat_map(|attr| attr.nested); + for meta in metas { + if let NestedMeta::Meta(meta) = meta { + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pymethod(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => { + if ident == "property" { + kind = MethodKind::Property + } + } + _ => {} + } + } + } + // if there are no #[pymethods]s, then it's not a method, so exclude it from + // the final result + if pymethod_to_remove.is_empty() { + return None; + } + for i in pymethod_to_remove { + attrs.remove(i); + } + let py_name = py_name.unwrap_or_else(|| sig.ident.to_string()); + Some(Method { + fn_name: sig.ident.clone(), + py_name, + kind, + }) + } +} + +/// Parse an impl block into an iterator of methods +fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { + imp.items.iter_mut().filter_map(|item| { + if let ImplItem::Method(meth) = item { + Method::from_syn(&mut meth.attrs, &meth.sig) + } else { + None + } + }) +} + +pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { + let mut imp = if let Item::Impl(imp) = item { + imp + } else { + return quote!(#item); + }; + let rp_path = rustpython_path_attr(&attr); + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[pyclass(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.expect("#[pyclass] must have a name"); + let mut doc: Option> = None; + for attr in imp.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } else { + panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); + let ty = &imp.self_ty; + let methods = methods.iter().map( + |Method { + py_name, + fn_name, + kind, + }| { + let constructor_fn = kind.to_ctx_constructor_fn(); + quote! { + ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); + } + }, + ); + + quote! { + #imp + impl #rp_path::pyobject::IntoPyClass for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + fn _extend_class( + ctx: &#rp_path::pyobject::PyContext, + class: &#rp_path::obj::objtype::PyClassRef, + ) { + #(#methods)* + } + } + } +} diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 29124ad11..4a300dcef 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -30,7 +30,9 @@ extern crate statrs; extern crate rustpython_parser; #[macro_use] -pub extern crate rustpython_derive; +extern crate rustpython_derive; + +pub use rustpython_derive::*; //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; From 15cffc4d2b023ec31aade0c72d8821f72be74265 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:30:14 -0500 Subject: [PATCH 012/223] Clean up a bit --- derive/src/pyclass.rs | 23 ++++++++------- vm/src/obj/objstr.rs | 65 ++----------------------------------------- 2 files changed, 13 insertions(+), 75 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 4497424f2..ad03b58c9 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -89,17 +89,6 @@ impl Method { } } -/// Parse an impl block into an iterator of methods -fn item_impl_to_methods<'a>(imp: &'a mut syn::ItemImpl) -> impl Iterator + 'a { - imp.items.iter_mut().filter_map(|item| { - if let ImplItem::Method(meth) = item { - Method::from_syn(&mut meth.attrs, &meth.sig) - } else { - None - } - }) -} - pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp @@ -148,7 +137,17 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } None => quote!(None), }; - let methods: Vec<_> = item_impl_to_methods(&mut imp).collect(); + let methods = imp + .items + .iter_mut() + .filter_map(|item| { + if let ImplItem::Method(meth) = item { + Method::from_syn(&mut meth.attrs, &meth.sig) + } else { + None + } + }) + .collect::>(); let ty = &imp.self_ty; let methods = methods.iter().map( |Method { diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 70c420665..9520b8af9 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -748,69 +748,8 @@ impl IntoPyObject for String { } } -#[rustfmt::skip] // to avoid line splitting -pub fn init(context: &PyContext) { - let str_type = &context.str_type; - - PyStringRef::extend_class(context, str_type); - // extend_class!(context, str_type, { - // "__add__" => context.new_rustfunc(PyStringRef::add), - // "__bool__" => context.new_rustfunc(PyStringRef::bool), - // "__contains__" => context.new_rustfunc(PyStringRef::contains), - // "__doc__" => context.new_str(str_doc.to_string()), - // "__eq__" => context.new_rustfunc(PyStringRef::eq), - // "__ge__" => context.new_rustfunc(PyStringRef::ge), - // "__getitem__" => context.new_rustfunc(PyStringRef::getitem), - // "__gt__" => context.new_rustfunc(PyStringRef::gt), - // "__hash__" => context.new_rustfunc(PyStringRef::hash), - // "__lt__" => context.new_rustfunc(PyStringRef::lt), - // "__le__" => context.new_rustfunc(PyStringRef::le), - // "__len__" => context.new_rustfunc(PyStringRef::len), - // "__mul__" => context.new_rustfunc(PyStringRef::mul), - // "__new__" => context.new_rustfunc(str_new), - // "__repr__" => context.new_rustfunc(PyStringRef::repr), - // "__str__" => context.new_rustfunc(PyStringRef::str), - // "capitalize" => context.new_rustfunc(PyStringRef::capitalize), - // "casefold" => context.new_rustfunc(PyStringRef::casefold), - // "center" => context.new_rustfunc(PyStringRef::center), - // "count" => context.new_rustfunc(PyStringRef::count), - // "endswith" => context.new_rustfunc(PyStringRef::endswith), - // "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), - // "find" => context.new_rustfunc(PyStringRef::find), - // "format" => context.new_rustfunc(str_format), - // "index" => context.new_rustfunc(PyStringRef::index), - // "isalnum" => context.new_rustfunc(PyStringRef::isalnum), - // "isalpha" => context.new_rustfunc(PyStringRef::isalpha), - // "isascii" => context.new_rustfunc(PyStringRef::isascii), - // "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), - // "isdigit" => context.new_rustfunc(PyStringRef::isdigit), - // "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), - // "islower" => context.new_rustfunc(PyStringRef::islower), - // "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), - // "isspace" => context.new_rustfunc(PyStringRef::isspace), - // "isupper" => context.new_rustfunc(PyStringRef::isupper), - // "istitle" => context.new_rustfunc(PyStringRef::istitle), - // "join" => context.new_rustfunc(PyStringRef::join), - // "lower" => context.new_rustfunc(PyStringRef::lower), - // "ljust" => context.new_rustfunc(PyStringRef::ljust), - // "lstrip" => context.new_rustfunc(PyStringRef::lstrip), - // "partition" => context.new_rustfunc(PyStringRef::partition), - // "replace" => context.new_rustfunc(PyStringRef::replace), - // "rfind" => context.new_rustfunc(PyStringRef::rfind), - // "rindex" => context.new_rustfunc(PyStringRef::rindex), - // "rjust" => context.new_rustfunc(PyStringRef::rjust), - // "rpartition" => context.new_rustfunc(PyStringRef::rpartition), - // "rsplit" => context.new_rustfunc(PyStringRef::rsplit), - // "rstrip" => context.new_rustfunc(PyStringRef::rstrip), - // "split" => context.new_rustfunc(PyStringRef::split), - // "splitlines" => context.new_rustfunc(PyStringRef::splitlines), - // "startswith" => context.new_rustfunc(PyStringRef::startswith), - // "strip" => context.new_rustfunc(PyStringRef::strip), - // "swapcase" => context.new_rustfunc(PyStringRef::swapcase), - // "title" => context.new_rustfunc(PyStringRef::title), - // "upper" => context.new_rustfunc(PyStringRef::upper), - // "zfill" => context.new_rustfunc(PyStringRef::zfill), - // }); +pub fn init(ctx: &PyContext) { + PyStringRef::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { From 1b57f0783466d5e364b4307f6943fa8679426a30 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:56:37 -0500 Subject: [PATCH 013/223] Process lalrpop in cargo's target directory --- parser/.gitignore | 4 ---- parser/build.rs | 5 +---- parser/src/lib.rs | 4 ++-- parser/src/python.lalrpop | 1 - 4 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 parser/.gitignore diff --git a/parser/.gitignore b/parser/.gitignore deleted file mode 100644 index 401e8cc22..000000000 --- a/parser/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -src/python.rs -target/ -Cargo.lock - diff --git a/parser/build.rs b/parser/build.rs index 65147ed0e..19f102721 100644 --- a/parser/build.rs +++ b/parser/build.rs @@ -1,8 +1,5 @@ use lalrpop; fn main() { - lalrpop::Configuration::new() - .generate_in_source_tree() - .process() - .unwrap(); + lalrpop::process_root().unwrap() } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index b10c3d551..4f0105a9a 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,11 +1,11 @@ #[macro_use] extern crate log; +use lalrpop_util::lalrpop_mod; pub mod ast; pub mod error; mod fstring; pub mod lexer; pub mod parser; -#[cfg_attr(rustfmt, rustfmt_skip)] -mod python; +lalrpop_mod!(python); pub mod token; diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 9e5b309b1..437d001fe 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -2,7 +2,6 @@ // See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4 // See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword -#![allow(unknown_lints,clippy)] use std::iter::FromIterator; From 60ad5489f15d8911afead86b4c84907fbe20eeec Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Mar 2019 22:58:46 -0500 Subject: [PATCH 014/223] Ignore clippy lints in the lalrpop generated file --- parser/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 4f0105a9a..5cc8ca3c4 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -7,5 +7,8 @@ pub mod error; mod fstring; pub mod lexer; pub mod parser; -lalrpop_mod!(python); +lalrpop_mod!( + #[allow(clippy::all)] + python +); pub mod token; From f994f8660e487de93f1ad1080512e4af24e92fee Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:10:24 +0200 Subject: [PATCH 015/223] Function support __doc__ --- tests/snippets/function.py | 3 +++ vm/src/compile.rs | 32 +++++++++++++++++++++++++++++++- vm/src/frame.rs | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index b1ea96860..f327d88cb 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,7 +1,10 @@ def foo(): + """test""" return 42 assert foo() == 42 +assert foo.__doc__ == "test" + def my_func(a,): return a+2 diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 00bafe727..037d9e95c 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -597,7 +597,20 @@ impl Compiler { self.in_loop = false; self.in_function_def = true; let mut flags = self.enter_function(name, args)?; - self.compile_statements(body)?; + + let doc = get_doc(body); + let (new_body, doc_str) = match doc { + Some(val) => { + if let Some((_, body_rest)) = body.split_first() { + (body_rest, val) + } else { + (body, "".to_string()) + } + } + None => (body, "".to_string()), + }; + + self.compile_statements(new_body)?; // Emit None at end: self.emit(Instruction::LoadConst { @@ -644,6 +657,10 @@ impl Compiler { }); } + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { value: doc_str }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1511,6 +1528,19 @@ impl Compiler { } } +fn get_doc(body: &[ast::LocatedStatement]) -> Option { + if let Some(val) = body.get(0) { + if let ast::Statement::Expression { ref expression } = val.node { + if let ast::Expression::String { ref value } = expression { + if let ast::StringGroup::Constant { ref value } = value { + return Some(value.to_string()); + } + } + } + } + None +} + #[cfg(test)] mod tests { use super::Compiler; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 88a619eb3..5a22aabe9 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -568,6 +568,8 @@ impl Frame { .downcast() .expect("Second to top value on the stack must be a code object"); + let doc = self.pop_value(); + let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { @@ -586,6 +588,7 @@ impl Frame { let obj = vm.ctx.new_function(code_obj, scope, defaults); vm.ctx.set_attr(&obj, "__annotations__", annotations); + vm.ctx.set_attr(&obj, "__doc__", doc); self.push_value(obj); Ok(None) From c76b31866d2fae1d6b8112be47b6ccbaca1f5cf5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:15:39 +0200 Subject: [PATCH 016/223] Add more tests for function doc --- tests/snippets/function.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index f327d88cb..7d3f25709 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -5,8 +5,35 @@ def foo(): assert foo() == 42 assert foo.__doc__ == "test" - def my_func(a,): return a+2 assert my_func(2) == 4 + + +def f1(): + + """test1""" + pass + +assert f1.__doc__ == "test1" + +def f2(): + '''test2''' + pass + +assert f2.__doc__ == "test2" + +def f3(): + """ + test3 + """ + pass + +assert f3.__doc__ == "\n test3\n " + +def f4(): + "test4" + pass + +assert f4.__doc__ == "test4" From ab3cde77b4b2b7d71ffe7e5d05b94e3c45c677e3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:45:45 +0200 Subject: [PATCH 017/223] Support class __doc__ --- tests/snippets/class.py | 3 +-- vm/src/builtins.rs | 2 ++ vm/src/compile.rs | 47 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 06e125f21..bd94e0516 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -37,8 +37,7 @@ class Bar: assert x == 3 -# TODO: -# assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == " W00t " bar = Bar(42) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index ed656c0fb..705ec6905 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -769,6 +769,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { let function = args.shift(); let name_arg = args.shift(); + let doc = args.shift(); let bases = args.args.clone(); let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? @@ -801,5 +802,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu vec![name_arg, bases, namespace.into_object()], )?; cells.set_item(&vm.ctx, "__class__", class.clone()); + vm.ctx.set_attr(&class, "__doc__", doc); Ok(class) } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 037d9e95c..58a582ae7 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -706,13 +706,34 @@ impl Compiler { line_number, name.to_string(), )); - self.compile_statements(body)?; + + let doc = get_doc(body); + let (new_body, doc_str) = match doc { + Some(val) => { + if let Some((_, body_rest)) = body.split_first() { + (body_rest, val) + } else { + (body, "".to_string()) + } + } + None => (body, "".to_string()), + }; + + self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -735,6 +756,11 @@ impl Compiler { }, }); + // class doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { value: doc_str }, + }); + for base in bases { self.compile_expression(base)?; } @@ -759,11 +785,11 @@ impl Compiler { }, }); self.emit(Instruction::CallFunction { - typ: CallType::Keyword(2 + keywords.len() + bases.len()), + typ: CallType::Keyword(3 + keywords.len() + bases.len()), }); } else { self.emit(Instruction::CallFunction { - typ: CallType::Positional(2 + bases.len()), + typ: CallType::Positional(3 + bases.len()), }); } @@ -1142,6 +1168,14 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1428,6 +1462,13 @@ impl Compiler { // Fetch code for listcomp function: let code = self.pop_code_object(); + // function doc + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: "".to_string(), + }, + }); + // List comprehension code: self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { From 6d53fe692408912c3558e827c296cb98a9f98e68 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 28 Mar 2019 23:49:40 +0200 Subject: [PATCH 018/223] Add more tests for class doc --- tests/snippets/class.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index bd94e0516..9dbce566d 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -116,3 +116,31 @@ assert isinstance(a, super) assert type(a) is super assert a.conjugate() == 1 + +class T1: + "test1" + +assert T1.__doc__ == "test1" + +class T2: + '''test2''' + +assert T2.__doc__ == "test2" + +class T3: + """ + test3 + """ + +assert T3.__doc__ == "\n test3\n " + +class T4: + + """test4""" + + def t1(self): + """t1""" + pass + +assert T4.__doc__ == "test4" +assert T4.t1.__doc__ == "t1" From 4e99350e5c7e8850d0557b2fac7c19773f7b1dab Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 31 Mar 2019 18:49:04 +0300 Subject: [PATCH 019/223] Use StoreAttr for __doc__ --- vm/src/builtins.rs | 2 - vm/src/compile.rs | 94 ++++++++++++++++++---------------------------- vm/src/frame.rs | 3 -- 3 files changed, 36 insertions(+), 63 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 705ec6905..ed656c0fb 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -769,7 +769,6 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { let function = args.shift(); let name_arg = args.shift(); - let doc = args.shift(); let bases = args.args.clone(); let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? @@ -802,6 +801,5 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu vec![name_arg, bases, namespace.into_object()], )?; cells.set_item(&vm.ctx, "__class__", class.clone()); - vm.ctx.set_attr(&class, "__doc__", doc); Ok(class) } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 58a582ae7..f79980af9 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -598,17 +598,7 @@ impl Compiler { self.in_function_def = true; let mut flags = self.enter_function(name, args)?; - let doc = get_doc(body); - let (new_body, doc_str) = match doc { - Some(val) => { - if let Some((_, body_rest)) = body.split_first() { - (body_rest, val) - } else { - (body, "".to_string()) - } - } - None => (body, "".to_string()), - }; + let (new_body, doc_str) = get_doc(body); self.compile_statements(new_body)?; @@ -657,10 +647,6 @@ impl Compiler { }); } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: doc_str }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -679,6 +665,20 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + + if let Some(doc_string) = doc_str { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + self.emit(Instruction::LoadName { + name: name.to_string(), + }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -707,17 +707,7 @@ impl Compiler { name.to_string(), )); - let doc = get_doc(body); - let (new_body, doc_str) = match doc { - Some(val) => { - if let Some((_, body_rest)) = body.split_first() { - (body_rest, val) - } else { - (body, "".to_string()) - } - } - None => (body, "".to_string()), - }; + let (new_body, doc_str) = get_doc(body); self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { @@ -727,13 +717,6 @@ impl Compiler { let code = self.pop_code_object(); - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -756,11 +739,6 @@ impl Compiler { }, }); - // class doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { value: doc_str }, - }); - for base in bases { self.compile_expression(base)?; } @@ -785,11 +763,11 @@ impl Compiler { }, }); self.emit(Instruction::CallFunction { - typ: CallType::Keyword(3 + keywords.len() + bases.len()), + typ: CallType::Keyword(2 + keywords.len() + bases.len()), }); } else { self.emit(Instruction::CallFunction { - typ: CallType::Positional(3 + bases.len()), + typ: CallType::Positional(2 + bases.len()), }); } @@ -798,6 +776,19 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + if let Some(doc_string) = doc_str { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + self.emit(Instruction::LoadName { + name: name.to_string(), + }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } self.in_loop = was_in_loop; Ok(()) } @@ -1168,14 +1159,6 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1462,13 +1445,6 @@ impl Compiler { // Fetch code for listcomp function: let code = self.pop_code_object(); - // function doc - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: "".to_string(), - }, - }); - // List comprehension code: self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { @@ -1569,17 +1545,19 @@ impl Compiler { } } -fn get_doc(body: &[ast::LocatedStatement]) -> Option { +fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option) { if let Some(val) = body.get(0) { if let ast::Statement::Expression { ref expression } = val.node { if let ast::Expression::String { ref value } = expression { if let ast::StringGroup::Constant { ref value } = value { - return Some(value.to_string()); + if let Some((_, body_rest)) = body.split_first() { + return (body_rest, Some(value.to_string())); + } } } } } - None + (body, None) } #[cfg(test)] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 5a22aabe9..88a619eb3 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -568,8 +568,6 @@ impl Frame { .downcast() .expect("Second to top value on the stack must be a code object"); - let doc = self.pop_value(); - let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { @@ -588,7 +586,6 @@ impl Frame { let obj = vm.ctx.new_function(code_obj, scope, defaults); vm.ctx.set_attr(&obj, "__annotations__", annotations); - vm.ctx.set_attr(&obj, "__doc__", doc); self.push_value(obj); Ok(None) From 53b46a7b3245741a27ec05b30a23f1a9f26538ee Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 17:23:33 -0500 Subject: [PATCH 020/223] Split some of #[pyclass] off into #[pyimpl] --- derive/src/lib.rs | 9 +++- derive/src/pyclass.rs | 112 +++++++++++++++++++++++++----------------- vm/src/obj/objstr.rs | 25 +++++----- vm/src/pyobject.rs | 16 ++++-- 4 files changed, 100 insertions(+), 62 deletions(-) diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 5a26030e4..adde078f0 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -52,5 +52,12 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr as AttributeArgs); let item = parse_macro_input!(item as Item); - pyclass::impl_py_class(attr, item).into() + pyclass::impl_pyclass(attr, item).into() +} + +#[proc_macro_attribute] +pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + pyclass::impl_pyimpl(attr, item).into() } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index ad03b58c9..56e216f35 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -89,54 +89,15 @@ impl Method { } } -pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); }; + let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; - for attr in attr { - if let NestedMeta::Meta(meta) = attr { - if let Meta::NameValue(name_value) = meta { - if name_value.ident == "name" { - if let Lit::Str(s) = name_value.lit { - class_name = Some(s.value()); - } else { - panic!("#[pyclass(name = ...)] must be a string"); - } - } - } - } - } - let class_name = class_name.expect("#[pyclass] must have a name"); - let mut doc: Option> = None; - for attr in imp.attrs.iter() { - if attr.path.is_ident("doc") { - let meta = attr.parse_meta().expect("expected doc attr to be a meta"); - if let Meta::NameValue(name_value) = meta { - if let Lit::Str(s) = name_value.lit { - let val = s.value().trim().to_string(); - match doc { - Some(ref mut doc) => doc.push(val), - None => doc = Some(vec![val]), - } - } else { - panic!("expected #[doc = ...] to be a string") - } - } else { - panic!("expected #[doc] to be a NameValue, e.g. #[doc = \"...\""); - } - } - } - let doc = match doc { - Some(doc) => { - let doc = doc.join("\n"); - quote!(Some(#doc)) - } - None => quote!(None), - }; + let methods = imp .items .iter_mut() @@ -164,10 +125,8 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #imp - impl #rp_path::pyobject::IntoPyClass for #ty { - const NAME: &'static str = #class_name; - const DOC: Option<&'static str> = #doc; - fn _extend_class( + impl #rp_path::pyobject::PyClassImpl for #ty { + fn impl_extend_class( ctx: &#rp_path::pyobject::PyContext, class: &#rp_path::obj::objtype::PyClassRef, ) { @@ -176,3 +135,64 @@ pub fn impl_py_class(attr: AttributeArgs, item: Item) -> TokenStream2 { } } } + +pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { + let struc = if let Item::Struct(struc) = item { + struc + } else { + panic!("#[pyclass] can only be on a struct declaration"); + }; + + let rp_path = rustpython_path_attr(&attr); + + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[pyclass(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.unwrap_or_else(|| struc.ident.to_string()); + + let mut doc: Option> = None; + for attr in struc.attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + + let ty = &struc.ident; + + quote! { + #struc + impl #rp_path::pyobject::PyClassDef for #ty { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + } + } +} diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 9520b8af9..922306a9e 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -10,7 +10,7 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyClass, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -20,6 +20,17 @@ use super::objsequence::PySliceableSequence; use super::objslice::PySlice; use super::objtype::{self, PyClassRef}; +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." +#[pyclass(name = "str", __inside_vm)] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public @@ -48,17 +59,7 @@ impl TryIntoRef for &str { } } -#[pyclass(__inside_vm, name = "str")] -/// str(object='') -> str -/// str(bytes_or_buffer[, encoding[, errors]]) -> str -/// -/// Create a new string object from the given object. If encoding or -/// errors is specified, then the object must expose a data buffer -/// that will be decoded using the given encoding and error handler. -/// Otherwise, returns the result of object.__str__() (if defined) -/// or repr(object). -/// encoding defaults to sys.getdefaultencoding(). -/// errors defaults to 'strict'." +#[pyimpl(__inside_vm)] impl PyStringRef { // TODO: should with following format // class str(object='') diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index a0e737d4c..801522ea0 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1259,14 +1259,24 @@ where } } -pub trait IntoPyClass { +pub trait PyClassDef { const NAME: &'static str; const DOC: Option<&'static str> = None; +} - fn _extend_class(ctx: &PyContext, class: &PyClassRef); +impl PyClassDef for PyRef +where + T: PyClassDef, +{ + const NAME: &'static str = T::NAME; + const DOC: Option<&'static str> = T::DOC; +} + +pub trait PyClassImpl: PyClassDef { + fn impl_extend_class(ctx: &PyContext, class: &PyClassRef); fn extend_class(ctx: &PyContext, class: &PyClassRef) { - Self::_extend_class(ctx, class); + Self::impl_extend_class(ctx, class); if let Some(doc) = Self::DOC { ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); } From 42768b20e57a1a699340c78d21a5268910dc0b93 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 17:53:53 -0500 Subject: [PATCH 021/223] Small demo improvements --- wasm/demo/snippets/mandelbrot.py | 9 +++++---- wasm/demo/src/style.css | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index 2d664fded..b4010c753 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -2,8 +2,7 @@ w = 50.0 h = 50.0 def mandel(): - """Print a mandelbrot fractal to the console, yielding after each character - is printed""" + """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 while y < h: x = 0.0 @@ -32,12 +31,14 @@ def mandel(): y += 1 yield +# run the mandelbrot + try: from browser import request_animation_frame except: request_animation_frame = None gen = mandel() def gen_cb(_time=None): - gen.__next__() + for _ in range(4): gen.__next__() request_animation_frame(gen_cb) if request_animation_frame: gen_cb() -else: list(gen) +else: any(gen) diff --git a/wasm/demo/src/style.css b/wasm/demo/src/style.css index a8a5de500..8fc18ecce 100644 --- a/wasm/demo/src/style.css +++ b/wasm/demo/src/style.css @@ -1,6 +1,6 @@ textarea { font-family: monospace; - resize: none; + resize: vertical; } #code, From 51b766089f4f9f245499c280ff2a33627c024bb9 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 19:53:52 -0500 Subject: [PATCH 022/223] Allow an enum with #[pyclass] --- derive/src/pyclass.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 56e216f35..7e196a2a1 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -137,10 +137,10 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { } pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { - let struc = if let Item::Struct(struc) = item { - struc - } else { - panic!("#[pyclass] can only be on a struct declaration"); + let (ident, attrs) = match item.clone() { + Item::Struct(struc) => (struc.ident, struc.attrs), + Item::Enum(enu) => (enu.ident, enu.attrs), + _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; let rp_path = rustpython_path_attr(&attr); @@ -159,10 +159,10 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { } } } - let class_name = class_name.unwrap_or_else(|| struc.ident.to_string()); + let class_name = class_name.unwrap_or_else(|| ident.to_string()); let mut doc: Option> = None; - for attr in struc.attrs.iter() { + for attr in attrs.iter() { if attr.path.is_ident("doc") { let meta = attr.parse_meta().expect("expected doc attr to be a meta"); if let Meta::NameValue(name_value) = meta { @@ -186,11 +186,9 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { None => quote!(None), }; - let ty = &struc.ident; - quote! { - #struc - impl #rp_path::pyobject::PyClassDef for #ty { + #item + impl #rp_path::pyobject::PyClassDef for #ident { const NAME: &'static str = #class_name; const DOC: Option<&'static str> = #doc; } From b3ea8155a5558b4c7fec3c1775f4fcaf2449442c Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 31 Mar 2019 22:40:57 -0500 Subject: [PATCH 023/223] Don't clone --- derive/src/pyclass.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 7e196a2a1..ad4ebf66e 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -137,9 +137,9 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { } pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { - let (ident, attrs) = match item.clone() { - Item::Struct(struc) => (struc.ident, struc.attrs), - Item::Enum(enu) => (enu.ident, enu.attrs), + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; From 016ecf204dcb8ff9c956038572f03fbb9bd5d186 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 024/223] Move iterator __contains__ to 'in' implementation --- vm/src/frame.rs | 30 ++++-------------------------- vm/src/obj/objiter.rs | 33 +++++---------------------------- vm/src/vm.rs | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 54 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 88a619eb3..b3cf2149b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1075,36 +1075,14 @@ impl Frame { a.get_id() } - // https://docs.python.org/3/reference/expressions.html#membership-test-operations - fn _membership( - &self, - vm: &VirtualMachine, - needle: PyObjectRef, - haystack: &PyObjectRef, - ) -> PyResult { - vm.call_method(&haystack, "__contains__", vec![needle]) - // TODO: implement __iter__ and __getitem__ cases when __contains__ is - // not implemented. - } - fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(found), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(objbool::boolval(vm, found)?)) } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(!objbool::boolval(vm, found)?)) } fn _is(&self, a: PyObjectRef, b: PyObjectRef) -> bool { diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 5c88a6337..68c8b055f 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -6,7 +6,6 @@ use crate::function::PyFuncArgs; use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; -use super::objbool; use super::objbytearray::PyByteArray; use super::objbytes::PyBytes; use super::objrange::PyRange; @@ -24,6 +23,10 @@ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); // return Err(type_error); + + // TODO: special case when iter_target only has __getitem__ + // see: https://docs.python.org/3/library/functions.html#iter + // also https://docs.python.org/3.8/reference/datamodel.html#special-method-names } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -70,33 +73,8 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -fn contains(vm: &VirtualMachine, args: PyFuncArgs, iter_type: PyClassRef) -> PyResult { - arg_check!( - vm, - args, - required = [(iter, Some(iter_type)), (needle, None)] - ); - loop { - if let Some(element) = get_next_object(vm, iter)? { - let equal = vm._eq(needle.clone(), element.clone())?; - if objbool::get_value(&equal) { - return Ok(vm.new_bool(true)); - } else { - continue; - } - } else { - return Ok(vm.new_bool(false)); - } - } -} - -/// Common setup for iter types, adds __iter__ and __contains__ methods +/// Common setup for iter types, adds __iter__ method pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { - let contains_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| contains(vm, args, cloned_iter_type.clone()) - }; - let iter_func = { let cloned_iter_type = iter_type.clone(); move |vm: &VirtualMachine, args: PyFuncArgs| { @@ -111,7 +89,6 @@ pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { }; extend_class!(context, iter_type, { - "__contains__" => context.new_rustfunc(contains_func), "__iter__" => context.new_rustfunc(iter_func) }); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 36f4450e6..2fad29124 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -867,6 +867,31 @@ impl VirtualMachine { Err(vm.new_unsupported_operand_error(a, b, ">=")) }) } + + // https://docs.python.org/3/reference/expressions.html#membership-test-operations + fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + let iter = objiter::get_iter(self, &haystack)?; + loop { + if let Some(element) = objiter::get_next_object(self, &iter)? { + let equal = self._eq(needle.clone(), element.clone())?; + if objbool::get_value(&equal) { + return Ok(self.new_bool(true)); + } else { + continue; + } + } else { + return Ok(self.new_bool(false)); + } + } + } + + pub fn _membership(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + if let Ok(method) = self.get_method(haystack.clone(), "__contains__") { + self.invoke(method, vec![needle]) + } else { + self._membership_iter_search(haystack, needle) + } + } } impl Default for VirtualMachine { From cba8aa9be52bfc6c9e0f4253de28cd476398bde0 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 025/223] Drop iter_type_init, explicitly define __iter__ for iterators --- vm/src/obj/objenumerate.rs | 8 ++++-- vm/src/obj/objfilter.rs | 9 ++++-- vm/src/obj/objiter.rs | 56 +++++++++----------------------------- vm/src/obj/objmap.rs | 6 +++- vm/src/obj/objzip.rs | 8 ++++-- 5 files changed, 36 insertions(+), 51 deletions(-) diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 85781bb7a..c94aeabbd 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -57,13 +57,17 @@ impl PyEnumerateRef { Ok(result) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let enumerate_type = &context.enumerate_type; - objiter::iter_type_init(context, enumerate_type); extend_class!(context, enumerate_type, { "__new__" => context.new_rustfunc(enumerate_new), - "__next__" => context.new_rustfunc(PyEnumerateRef::next) + "__next__" => context.new_rustfunc(PyEnumerateRef::next), + "__iter__" => context.new_rustfunc(PyEnumerateRef::iter), }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index 07c512f6d..858a0531a 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -52,13 +52,15 @@ impl PyFilterRef { } } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let filter_type = &context.filter_type; - objiter::iter_type_init(context, filter_type); - let filter_doc = "filter(function or None, iterable) --> filter object\n\n\ Return an iterator yielding those items of iterable for which function(item)\n\ @@ -67,6 +69,7 @@ pub fn init(context: &PyContext) { extend_class!(context, filter_type, { "__new__" => context.new_rustfunc(filter_new), "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(PyFilterRef::next) + "__next__" => context.new_rustfunc(PyFilterRef::next), + "__iter__" => context.new_rustfunc(PyFilterRef::iter), }); } diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 68c8b055f..67513a255 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,8 +2,7 @@ * Various types to support iteration. */ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; use super::objbytearray::PyByteArray; @@ -11,7 +10,6 @@ use super::objbytes::PyBytes; use super::objrange::PyRange; use super::objsequence; use super::objtype; -use crate::obj::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -73,41 +71,12 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -/// Common setup for iter types, adds __iter__ method -pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { - let iter_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| { - arg_check!( - vm, - args, - required = [(iter, Some(cloned_iter_type.clone()))] - ); - // Return self: - Ok(iter.clone()) - } - }; +type PyIteratorValueRef = PyRef; - extend_class!(context, iter_type, { - "__iter__" => context.new_rustfunc(iter_func) - }); -} - -// Sequence iterator: -fn iter_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter_target, None)]); - - get_iter(vm, iter_target) -} - -fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter, Some(vm.ctx.iter_type()))]); - - if let Some(PyIteratorValue { - ref position, - iterated_obj: ref iterated_obj_ref, - }) = iter.payload() - { +impl PyIteratorValueRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let position = &self.position; + let iterated_obj_ref = &self.iterated_obj; if let Some(range) = iterated_obj_ref.payload::() { if let Some(int) = range.get(position.get()) { position.set(position.get() + 1); @@ -141,8 +110,10 @@ fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(new_stop_iteration(vm)) } } - } else { - panic!("NOT IMPL"); + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self } } @@ -155,10 +126,9 @@ pub fn init(context: &PyContext) { supply its own iterator, or be a sequence.\n\ In the second form, the callable is called until it returns the sentinel."; - iter_type_init(context, iter_type); extend_class!(context, iter_type, { - "__new__" => context.new_rustfunc(iter_new), - "__next__" => context.new_rustfunc(iter_next), - "__doc__" => context.new_str(iter_doc.to_string()) + "__next__" => context.new_rustfunc(PyIteratorValueRef::next), + "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), + "__doc__" => context.new_str(iter_doc.to_string()), }); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index fbf0fa035..061f4179f 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -46,6 +46,10 @@ impl PyMapRef { // the mapper itself can raise StopIteration which does stop the map iteration vm.invoke(self.mapper.clone(), next_objs) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { @@ -55,10 +59,10 @@ pub fn init(context: &PyContext) { Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted."; - objiter::iter_type_init(context, map_type); extend_class!(context, map_type, { "__new__" => context.new_rustfunc(map_new), "__next__" => context.new_rustfunc(PyMapRef::next), + "__iter__" => context.new_rustfunc(PyMapRef::iter), "__doc__" => context.new_str(map_doc.to_string()) }); } diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index a12152eee..81640a724 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -40,13 +40,17 @@ impl PyZipRef { Ok(vm.ctx.new_tuple(next_objs)) } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let zip_type = &context.zip_type; - objiter::iter_type_init(context, zip_type); extend_class!(context, zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(PyZipRef::next) + "__next__" => context.new_rustfunc(PyZipRef::next), + "__iter__" => context.new_rustfunc(PyZipRef::iter), }); } From bbfca26b27cb03f20931f76d415ef95ff4706dd0 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 026/223] Move PyIteratorValue to objiter.rs --- vm/src/obj/objbytes.rs | 3 ++- vm/src/obj/objdict.rs | 3 ++- vm/src/obj/objiter.rs | 20 +++++++++++++++++++- vm/src/obj/objlist.rs | 5 ++--- vm/src/obj/objrange.rs | 3 ++- vm/src/obj/objset.rs | 5 ++--- vm/src/obj/objtuple.rs | 5 ++--- vm/src/pyobject.rs | 16 +--------------- 8 files changed, 32 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 9390e7070..2f5bdc0d9 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,10 +6,11 @@ use std::ops::Deref; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; +use super::objiter::PyIteratorValue; use super::objtype::PyClassRef; #[derive(Debug)] diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 061c3477e..5a06fbf99 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -5,11 +5,12 @@ use std::ops::{Deref, DerefMut}; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objiter; +use super::objiter::PyIteratorValue; use super::objstr::{self, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 67513a255..c1ffa7773 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,7 +2,9 @@ * Various types to support iteration. */ -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult}; +use std::cell::Cell; + +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objbytearray::PyByteArray; @@ -10,6 +12,7 @@ use super::objbytes::PyBytes; use super::objrange::PyRange; use super::objsequence; use super::objtype; +use super::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -71,6 +74,21 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } +// TODO: This is a workaround and shouldn't exist. +// Each iterable type should have its own distinct iterator type. +// (however, this boilerplate can be reused for "generic iterator" for types with only __getiter__) +#[derive(Debug)] +pub struct PyIteratorValue { + pub position: Cell, + pub iterated_obj: PyObjectRef, +} + +impl PyValue for PyIteratorValue { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.iter_type() + } +} + type PyIteratorValueRef = PyRef; impl PyIteratorValueRef { diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index dd5eb055b..444b5ad9d 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -4,13 +4,12 @@ use std::fmt; use num_traits::ToPrimitive; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; +use super::objiter::PyIteratorValue; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, PySliceableSequence, diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 04c9c273b..5af576bfc 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,10 +6,11 @@ use num_integer::Integer; use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{Either, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; +use super::objiter::PyIteratorValue; use super::objslice::PySliceRef; use super::objtype::PyClassRef; diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index ad82c34de..c048686d7 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,14 +8,13 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - PyContext, PyIteratorValue, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; use super::objiter; +use super::objiter::PyIteratorValue; use super::objtype; use super::objtype::PyClassRef; diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index a199e34b1..e7dc5d0ea 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -3,13 +3,12 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, -}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; +use super::objiter::PyIteratorValue; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index aa3348be5..3574d71d9 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,5 +1,5 @@ use std::any::Any; -use std::cell::{Cell, RefCell}; +use std::cell::RefCell; use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; @@ -1127,20 +1127,6 @@ where } } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -#[derive(Debug)] -pub struct PyIteratorValue { - pub position: Cell, - pub iterated_obj: PyObjectRef, -} - -impl PyValue for PyIteratorValue { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.iter_type() - } -} - impl PyObject where T: Sized + PyObjectPayload, From c918e9d5d3ab343e64a607529742690b144f212d Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 027/223] Split iterators into separate types --- vm/src/obj/objbytearray.rs | 47 +++++++++++++++++++++++++++++++++++- vm/src/obj/objbytes.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/obj/objdict.rs | 24 +++++++++++-------- vm/src/obj/objiter.rs | 49 ++------------------------------------ vm/src/obj/objlist.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/obj/objrange.rs | 49 ++++++++++++++++++++++++++++++++------ vm/src/obj/objset.rs | 9 +++---- vm/src/obj/objtuple.rs | 44 ++++++++++++++++++++++++++++++---- vm/src/pyobject.rs | 35 +++++++++++++++++++++++++++ 9 files changed, 264 insertions(+), 81 deletions(-) diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index c609fdb2f..f6244399b 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,6 +1,6 @@ //! Implementation of the python bytearray object. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::fmt::Write; use std::ops::{Deref, DerefMut}; @@ -11,6 +11,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; +use super::objiter; use super::objtype::PyClassRef; #[derive(Debug)] @@ -67,6 +68,7 @@ pub fn init(context: &PyContext) { "__eq__" => context.new_rustfunc(PyByteArrayRef::eq), "__len__" => context.new_rustfunc(PyByteArrayRef::len), "__repr__" => context.new_rustfunc(PyByteArrayRef::repr), + "__iter__" => context.new_rustfunc(PyByteArrayRef::iter), "clear" => context.new_rustfunc(PyByteArrayRef::clear), "isalnum" => context.new_rustfunc(PyByteArrayRef::isalnum), "isalpha" => context.new_rustfunc(PyByteArrayRef::isalpha), @@ -80,6 +82,12 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyByteArrayRef::pop), "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); + + let bytearrayiterator_type = &context.bytearrayiterator_type; + extend_class!(context, bytearrayiterator_type, { + "__next__" => context.new_rustfunc(PyByteArrayIteratorRef::next), + "__iter__" => context.new_rustfunc(PyByteArrayIteratorRef::iter), + }); } fn bytearray_new( @@ -225,6 +233,13 @@ impl PyByteArrayRef { value: RefCell::new(bytes), } } + + fn iter(self, _vm: &VirtualMachine) -> PyByteArrayIterator { + PyByteArrayIterator { + position: Cell::new(0), + bytearray: self, + } + } } // helper function for istitle @@ -266,3 +281,33 @@ mod tests { assert_eq!(&to_hex(&[11u8, 222u8]), "\\x0b\\xde"); } } + +#[derive(Debug)] +pub struct PyByteArrayIterator { + position: Cell, + bytearray: PyByteArrayRef, +} + +impl PyValue for PyByteArrayIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytearrayiterator_type() + } +} + +type PyByteArrayIteratorRef = PyRef; + +impl PyByteArrayIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytearray.value.borrow().len() { + let ret = self.bytearray.value.borrow()[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2f5bdc0d9..4119d688f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -10,7 +10,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objtype::PyClassRef; #[derive(Debug)] @@ -69,6 +69,12 @@ pub fn init(context: &PyContext) { "__iter__" => context.new_rustfunc(PyBytesRef::iter), "__doc__" => context.new_str(bytes_doc.to_string()) }); + + let bytesiterator_type = &context.bytesiterator_type; + extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } fn bytes_new( @@ -150,10 +156,10 @@ impl PyBytesRef { format!("b'{}'", data) } - fn iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { position: Cell::new(0), - iterated_obj: obj.into_object(), + bytes: self, } } } @@ -161,3 +167,33 @@ impl PyBytesRef { pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().value } + +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, +} + +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } +} + +type PyBytesIteratorRef = PyRef; + +impl PyBytesIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.value.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 5a06fbf99..79c98c2dd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -10,7 +10,7 @@ use crate::pyobject::{ use crate::vm::{ReprGuard, VirtualMachine}; use super::objiter; -use super::objiter::PyIteratorValue; +use super::objlist::PyListIterator; use super::objstr::{self, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -211,7 +211,8 @@ impl PyDictRef { } /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyIteratorValue { + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type, not a list iterator let keys = self .entries .borrow() @@ -220,13 +221,14 @@ impl PyDictRef { .collect(); let key_list = vm.ctx.new_list(keys); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: key_list, + list: key_list.downcast().unwrap(), } } - fn values(self, vm: &VirtualMachine) -> PyIteratorValue { + fn values(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type. `values` should be a live view over the collection, not an iterator. let values = self .entries .borrow() @@ -235,13 +237,14 @@ impl PyDictRef { .collect(); let values_list = vm.ctx.new_list(values); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: values_list, + list: values_list.downcast().unwrap(), } } - fn items(self, vm: &VirtualMachine) -> PyIteratorValue { + fn items(self, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type. `items` should be a live view over the collection, not an iterator. let items = self .entries .borrow() @@ -250,9 +253,9 @@ impl PyDictRef { .collect(); let items_list = vm.ctx.new_list(items); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: items_list, + list: items_list.downcast().unwrap(), } } @@ -333,6 +336,7 @@ pub fn init(context: &PyContext) { "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), + // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::iter), "get" => context.new_rustfunc(PyDictRef::get), }); diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index c1ffa7773..a99fb0a87 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -7,10 +7,6 @@ use std::cell::Cell; use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objbytearray::PyByteArray; -use super::objbytes::PyBytes; -use super::objrange::PyRange; -use super::objsequence; use super::objtype; use super::objtype::PyClassRef; @@ -92,42 +88,8 @@ impl PyValue for PyIteratorValue { type PyIteratorValueRef = PyRef; impl PyIteratorValueRef { - fn next(self, vm: &VirtualMachine) -> PyResult { - let position = &self.position; - let iterated_obj_ref = &self.iterated_obj; - if let Some(range) = iterated_obj_ref.payload::() { - if let Some(int) = range.get(position.get()) { - position.set(position.get() + 1); - Ok(vm.ctx.new_int(int)) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.len() { - let obj_ref = vm.ctx.new_int(bytes[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.value.borrow().len() { - let obj_ref = vm.ctx.new_int(bytes.value.borrow()[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else { - let elements = objsequence::get_elements(iterated_obj_ref); - if position.get() < elements.len() { - let obj_ref = elements[position.get()].clone(); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } + fn next(self, _vm: &VirtualMachine) -> PyResult { + unimplemented!() } fn iter(self, _vm: &VirtualMachine) -> Self { @@ -138,15 +100,8 @@ impl PyIteratorValueRef { pub fn init(context: &PyContext) { let iter_type = &context.iter_type; - let iter_doc = "iter(iterable) -> iterator\n\ - iter(callable, sentinel) -> iterator\n\n\ - Get an iterator from an object. In the first form, the argument must\n\ - supply its own iterator, or be a sequence.\n\ - In the second form, the callable is called until it returns the sentinel."; - extend_class!(context, iter_type, { "__next__" => context.new_rustfunc(PyIteratorValueRef::next), "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), - "__doc__" => context.new_str(iter_doc.to_string()), }); } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 444b5ad9d..865707201 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -9,7 +9,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, PySliceableSequence, @@ -122,10 +122,10 @@ impl PyListRef { ) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyListIterator { + PyListIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + list: self, } } @@ -411,6 +411,36 @@ fn list_sort(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[derive(Debug)] +pub struct PyListIterator { + pub position: Cell, + pub list: PyListRef, +} + +impl PyValue for PyListIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.listiterator_type() + } +} + +type PyListIteratorRef = PyRef; + +impl PyListIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.list.elements.borrow().len() { + let ret = self.list.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let list_type = &context.list_type; @@ -449,4 +479,10 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyListRef::pop), "remove" => context.new_rustfunc(PyListRef::remove) }); + + let listiterator_type = &context.listiterator_type; + extend_class!(context, listiterator_type, { + "__next__" => context.new_rustfunc(PyListIteratorRef::next), + "__iter__" => context.new_rustfunc(PyListIteratorRef::iter), + }); } diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 5af576bfc..ae946d05e 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -10,7 +10,7 @@ use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objslice::PySliceRef; use super::objtype::PyClassRef; @@ -114,6 +114,12 @@ pub fn init(context: &PyContext) { "stop" => context.new_property(PyRangeRef::stop), "step" => context.new_property(PyRangeRef::step), }); + + let rangeiterator_type = &context.rangeiterator_type; + extend_class!(context, rangeiterator_type, { + "__next__" => context.new_rustfunc(PyRangeIteratorRef::next), + "__iter__" => context.new_rustfunc(PyRangeIteratorRef::iter), + }); } type PyRangeRef = PyRef; @@ -157,14 +163,14 @@ impl PyRangeRef { self.step.clone() } - fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyRangeIterator { + PyRangeIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + range: self, } } - fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { + fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -190,9 +196,9 @@ impl PyRangeRef { step: PyInt::new(-step).into_ref(vm), }; - PyIteratorValue { + PyRangeIterator { position: Cell::new(0), - iterated_obj: reversed.into_ref(vm).into_object(), + range: reversed.into_ref(vm), } } @@ -314,3 +320,32 @@ fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(range.into_object()) } + +#[derive(Debug)] +pub struct PyRangeIterator { + position: Cell, + range: PyRangeRef, +} + +impl PyValue for PyRangeIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.rangeiterator_type() + } +} + +type PyRangeIteratorRef = PyRef; + +impl PyRangeIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if let Some(int) = self.range.get(self.position.get()) { + self.position.set(self.position.get() + 1); + Ok(int) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index c048686d7..fb558c990 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -14,7 +14,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; use super::objiter; -use super::objiter::PyIteratorValue; +use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; @@ -560,12 +560,13 @@ fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(zelf.clone()) } -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyIteratorValue { +fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyListIterator { + // TODO: separate type let items = zelf.elements.borrow().values().cloned().collect(); let set_list = vm.ctx.new_list(items); - PyIteratorValue { + PyListIterator { position: Cell::new(0), - iterated_obj: set_list, + list: set_list.downcast().unwrap(), } } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index e7dc5d0ea..216945284 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -8,7 +8,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter::PyIteratorValue; +use super::objiter; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; @@ -139,10 +139,10 @@ impl PyTupleRef { Ok(hasher.finish()) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { + PyTupleIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + tuple: self, } } @@ -224,6 +224,36 @@ fn tuple_new( PyTuple::from(elements).into_ref_with_type(vm, cls) } +#[derive(Debug)] +pub struct PyTupleIterator { + position: Cell, + tuple: PyTupleRef, +} + +impl PyValue for PyTupleIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.tupleiterator_type() + } +} + +type PyTupleIteratorRef = PyRef; + +impl PyTupleIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.tuple.elements.borrow().len() { + let ret = self.tuple.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let tuple_type = &context.tuple_type; @@ -251,4 +281,10 @@ If the argument is a tuple, the return value is the same object."; "__doc__" => context.new_str(tuple_doc.to_string()), "index" => context.new_rustfunc(PyTupleRef::index) }); + + let tupleiterator_type = &context.tupleiterator_type; + extend_class!(context, tupleiterator_type, { + "__next__" => context.new_rustfunc(PyTupleIteratorRef::next), + "__iter__" => context.new_rustfunc(PyTupleIteratorRef::iter), + }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 3574d71d9..291d92e25 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -108,7 +108,9 @@ impl fmt::Display for PyObject { #[derive(Debug)] pub struct PyContext { pub bytes_type: PyClassRef, + pub bytesiterator_type: PyClassRef, pub bytearray_type: PyClassRef, + pub bytearrayiterator_type: PyClassRef, pub bool_type: PyClassRef, pub classmethod_type: PyClassRef, pub code_type: PyClassRef, @@ -126,17 +128,20 @@ pub struct PyContext { pub true_value: PyIntRef, pub false_value: PyIntRef, pub list_type: PyClassRef, + pub listiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, pub ellipsis: PyEllipsisRef, pub not_implemented: PyNotImplementedRef, pub tuple_type: PyClassRef, + pub tupleiterator_type: PyClassRef, pub set_type: PyClassRef, pub staticmethod_type: PyClassRef, pub super_type: PyClassRef, pub str_type: PyClassRef, pub range_type: PyClassRef, + pub rangeiterator_type: PyClassRef, pub slice_type: PyClassRef, pub type_type: PyClassRef, pub zip_type: PyClassRef, @@ -248,6 +253,7 @@ impl PyContext { let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); + let listiterator_type = create_type("list_iterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -255,8 +261,11 @@ impl PyContext { let frame_type = create_type("frame", &type_type, &object_type); let complex_type = create_type("complex", &type_type, &object_type); let bytes_type = create_type("bytes", &type_type, &object_type); + let bytesiterator_type = create_type("bytes_iterator", &type_type, &object_type); let bytearray_type = create_type("bytearray", &type_type, &object_type); + let bytearrayiterator_type = create_type("bytearray_iterator", &type_type, &object_type); let tuple_type = create_type("tuple", &type_type, &object_type); + let tupleiterator_type = create_type("tuple_iterator", &type_type, &object_type); let iter_type = create_type("iter", &type_type, &object_type); let enumerate_type = create_type("enumerate", &type_type, &object_type); let filter_type = create_type("filter", &type_type, &object_type); @@ -266,6 +275,7 @@ impl PyContext { let memoryview_type = create_type("memoryview", &type_type, &object_type); let code_type = create_type("code", &type_type, &int_type); let range_type = create_type("range", &type_type, &object_type); + let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); let exceptions = exceptions::ExceptionZoo::new(&type_type, &object_type); @@ -291,7 +301,9 @@ impl PyContext { bool_type, memoryview_type, bytearray_type, + bytearrayiterator_type, bytes_type, + bytesiterator_type, code_type, complex_type, classmethod_type, @@ -300,11 +312,13 @@ impl PyContext { frame_type, staticmethod_type, list_type, + listiterator_type, set_type, frozenset_type, true_value, false_value, tuple_type, + tupleiterator_type, iter_type, ellipsis_type, enumerate_type, @@ -317,6 +331,7 @@ impl PyContext { not_implemented, str_type, range_type, + rangeiterator_type, slice_type, object: object_type, function_type, @@ -373,10 +388,18 @@ impl PyContext { self.bytearray_type.clone() } + pub fn bytearrayiterator_type(&self) -> PyClassRef { + self.bytearrayiterator_type.clone() + } + pub fn bytes_type(&self) -> PyClassRef { self.bytes_type.clone() } + pub fn bytesiterator_type(&self) -> PyClassRef { + self.bytesiterator_type.clone() + } + pub fn code_type(&self) -> PyClassRef { self.code_type.clone() } @@ -405,6 +428,10 @@ impl PyContext { self.list_type.clone() } + pub fn listiterator_type(&self) -> PyClassRef { + self.listiterator_type.clone() + } + pub fn module_type(&self) -> PyClassRef { self.module_type.clone() } @@ -417,6 +444,10 @@ impl PyContext { self.range_type.clone() } + pub fn rangeiterator_type(&self) -> PyClassRef { + self.rangeiterator_type.clone() + } + pub fn slice_type(&self) -> PyClassRef { self.slice_type.clone() } @@ -437,6 +468,10 @@ impl PyContext { self.tuple_type.clone() } + pub fn tupleiterator_type(&self) -> PyClassRef { + self.tupleiterator_type.clone() + } + pub fn iter_type(&self) -> PyClassRef { self.iter_type.clone() } From de5b1c405573e8fa6ef3611ee53a08c234466ee4 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 028/223] Support iter() for types with only __getitem__ --- vm/src/obj/objiter.rs | 55 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index a99fb0a87..6e1dc1d06 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -4,7 +4,7 @@ use std::cell::Cell; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; use super::objtype; @@ -16,14 +16,19 @@ use super::objtype::PyClassRef; * function 'iter' is called. */ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { - vm.call_method(iter_target, "__iter__", vec![]) - // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); - // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); - // return Err(type_error); - - // TODO: special case when iter_target only has __getitem__ - // see: https://docs.python.org/3/library/functions.html#iter - // also https://docs.python.org/3.8/reference/datamodel.html#special-method-names + if let Ok(method) = vm.get_method(iter_target.clone(), "__iter__") { + vm.invoke(method, vec![]) + } else if vm.get_method(iter_target.clone(), "__getitem__").is_ok() { + Ok(PySequenceIterator { + position: Cell::new(0), + obj: iter_target.clone(), + } + .into_ref(vm) + .into_object()) + } else { + let message = format!("Cannot iterate over {}", iter_target.class().name); + return Err(vm.new_type_error(message)); + } } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -70,26 +75,34 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -// (however, this boilerplate can be reused for "generic iterator" for types with only __getiter__) #[derive(Debug)] -pub struct PyIteratorValue { +pub struct PySequenceIterator { pub position: Cell, - pub iterated_obj: PyObjectRef, + pub obj: PyObjectRef, } -impl PyValue for PyIteratorValue { +impl PyValue for PySequenceIterator { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.iter_type() } } -type PyIteratorValueRef = PyRef; +type PySequenceIteratorRef = PyRef; -impl PyIteratorValueRef { - fn next(self, _vm: &VirtualMachine) -> PyResult { - unimplemented!() +impl PySequenceIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let number = vm.ctx.new_int(self.position.get()); + match vm.call_method(&self.obj, "__getitem__", vec![number]) { + Ok(val) => { + self.position.set(self.position.get() + 1); + Ok(val) + } + Err(ref e) if objtype::isinstance(&e, &vm.ctx.exceptions.index_error) => { + Err(new_stop_iteration(vm)) + } + // also catches stop_iteration => stop_iteration + Err(e) => Err(e), + } } fn iter(self, _vm: &VirtualMachine) -> Self { @@ -101,7 +114,7 @@ pub fn init(context: &PyContext) { let iter_type = &context.iter_type; extend_class!(context, iter_type, { - "__next__" => context.new_rustfunc(PyIteratorValueRef::next), - "__iter__" => context.new_rustfunc(PyIteratorValueRef::iter), + "__next__" => context.new_rustfunc(PySequenceIteratorRef::next), + "__iter__" => context.new_rustfunc(PySequenceIteratorRef::iter), }); } From 2f2a8438b55acba2cc98cbc9de8eb9cfb79a583e Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Mon, 1 Apr 2019 19:42:30 +0200 Subject: [PATCH 029/223] Add iterable tests --- tests/snippets/iterable.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/snippets/iterable.py diff --git a/tests/snippets/iterable.py b/tests/snippets/iterable.py new file mode 100644 index 000000000..45bb1cf4d --- /dev/null +++ b/tests/snippets/iterable.py @@ -0,0 +1,32 @@ +from testutils import assert_raises + +def test_container(x): + assert 3 in x + assert 4 not in x + assert list(x) == list(iter(x)) + assert list(x) == [0, 1, 2, 3] + assert [*x] == [0, 1, 2, 3] + lst = [] + lst.extend(x) + assert lst == [0, 1, 2, 3] + +class C: + def __iter__(self): + return iter([0, 1, 2, 3]) +test_container(C()) + +class C: + def __getitem__(self, x): + return (0, 1, 2, 3)[x] # raises IndexError on x==4 +test_container(C()) + +class C: + def __getitem__(self, x): + if x > 3: + raise StopIteration + return x +test_container(C()) + +class C: pass +assert_raises(TypeError, lambda: 5 in C()) +assert_raises(TypeError, lambda: iter(C)) From 6276241d59f085965a6ab02ff72171be8c4bd109 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 1 Apr 2019 18:11:37 -0500 Subject: [PATCH 030/223] Redo much of whats_left_to_implement.py --- tests/snippets/whats_left_to_implement.py | 1743 +++++++++++---------- whats_left.sh | 5 + 2 files changed, 879 insertions(+), 869 deletions(-) create mode 100755 whats_left.sh diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index b6c8cbe42..41b092506 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,878 +1,883 @@ -expected_methods = [] +import platform -# TODO: using tuple could have been better -expected_methods.append({'name': 'bool', 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -], 'type': bool}) -expected_methods.append({'name': 'bytearray', 'type': bytearray, 'methods': [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'bytes', 'type': bytes, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'complex', 'type': complex, 'methods': [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', -]}) -expected_methods.append({'name': 'dict', 'type': dict, 'methods': [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', -]}) -expected_methods.append({'name': 'float','type':float,'methods':[ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__set_format__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', -]}) -expected_methods.append({'name': 'frozenset','type':frozenset, 'methods': [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', -]}) -expected_methods.append({'name': 'int', 'type':int, 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -]}) -expected_methods.append({'name': 'iter','type':iter,'methods':[ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', -]}) -expected_methods.append({'name': 'list','type':list,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', -]}) -expected_methods.append({'name': 'memoryview','type':memoryview,'methods':[ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', -]}) -expected_methods.append({'name': 'range','type':range,'methods':[ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', -]}) -expected_methods.append({'name': 'set','type':set,'methods':[ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', -]}) -expected_methods.append({'name': 'string','type':str,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' -]}) -expected_methods.append({'name': 'tuple','type':tuple, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', -]}) -expected_methods.append({'name': 'object', 'type':object, 'methods':[ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' -]}) +expected_methods = { + 'bool': (bool, [ + '__abs__', + '__add__', + '__and__', + '__bool__', + '__ceil__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floor__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__index__', + '__init__', + '__init_subclass__', + '__int__', + '__invert__', + '__le__', + '__lshift__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__or__', + '__pos__', + '__pow__', + '__radd__', + '__rand__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__ror__', + '__round__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + '__xor__', + 'bit_length', + 'conjugate', + 'denominator', + 'from_bytes', + 'imag', + 'numerator', + 'real', + 'to_bytes', + ]), + 'bytearray': (bytearray, [ + '__add__', + '__alloc__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'capitalize', + 'center', + 'clear', + 'copy', + 'count', + 'decode', + 'endswith', + 'expandtabs', + 'extend', + 'find', + 'fromhex', + 'hex', + 'index', + 'insert', + 'isalnum', + 'isalpha', + 'isascii', + 'isdigit', + 'islower', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'pop', + 'remove', + 'replace', + 'reverse', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill', + ]), + 'bytes': (bytes, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'capitalize', + 'center', + 'count', + 'decode', + 'endswith', + 'expandtabs', + 'find', + 'fromhex', + 'hex', + 'index', + 'isalnum', + 'isalpha', + 'isascii', + 'isdigit', + 'islower', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'replace', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill', + ]), + 'complex': (complex, [ + '__abs__', + '__add__', + '__bool__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__int__', + '__le__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rsub__', + '__rtruediv__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + 'conjugate', + 'imag', + 'real', + ]), + 'dict': (dict, [ + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'clear', + 'copy', + 'fromkeys', + 'get', + 'items', + 'keys', + 'pop', + 'popitem', + 'setdefault', + 'update', + 'values', + ]), + 'float': (float, [ + '__abs__', + '__add__', + '__bool__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getformat__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__int__', + '__le__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rmod__', + '__rmul__', + '__round__', + '__rpow__', + '__rsub__', + '__rtruediv__', + '__set_format__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + 'as_integer_ratio', + 'conjugate', + 'fromhex', + 'hex', + 'imag', + 'is_integer', + 'real', + ]), + 'frozenset': (frozenset, [ + '__and__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__or__', + '__rand__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__ror__', + '__rsub__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__xor__', + 'copy', + 'difference', + 'intersection', + 'isdisjoint', + 'issubset', + 'issuperset', + 'symmetric_difference', + 'union', + ]), + 'int': (int, [ + '__abs__', + '__add__', + '__and__', + '__bool__', + '__ceil__', + '__class__', + '__delattr__', + '__dir__', + '__divmod__', + '__doc__', + '__eq__', + '__float__', + '__floor__', + '__floordiv__', + '__format__', + '__ge__', + '__getattribute__', + '__getnewargs__', + '__gt__', + '__hash__', + '__index__', + '__init__', + '__init_subclass__', + '__int__', + '__invert__', + '__le__', + '__lshift__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__neg__', + '__new__', + '__or__', + '__pos__', + '__pow__', + '__radd__', + '__rand__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__ror__', + '__round__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__truediv__', + '__trunc__', + '__xor__', + 'bit_length', + 'conjugate', + 'denominator', + 'from_bytes', + 'imag', + 'numerator', + 'real', + 'to_bytes', + ]), + 'iter': (iter, [ + '__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__length_hint__', + '__lt__', + '__ne__', + '__new__', + '__next__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setstate__', + '__sizeof__', + '__str__', + '__subclasshook__', + ]), + 'list': (list, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'pop', + 'remove', + 'reverse', + 'sort', + ]), + 'memoryview': (memoryview, [ + '__class__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__enter__', + '__eq__', + '__exit__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'c_contiguous', + 'cast', + 'contiguous', + 'f_contiguous', + 'format', + 'hex', + 'itemsize', + 'nbytes', + 'ndim', + 'obj', + 'readonly', + 'release', + 'shape', + 'strides', + 'suboffsets', + 'tobytes', + 'tolist', + ]), + 'range': (range, [ + '__bool__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'index', + 'start', + 'step', + 'stop', + ]), + 'set': (set, [ + '__and__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__iand__', + '__init__', + '__init_subclass__', + '__ior__', + '__isub__', + '__iter__', + '__ixor__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__or__', + '__rand__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__ror__', + '__rsub__', + '__rxor__', + '__setattr__', + '__sizeof__', + '__str__', + '__sub__', + '__subclasshook__', + '__xor__', + 'add', + 'clear', + 'copy', + 'difference', + 'difference_update', + 'discard', + 'intersection', + 'intersection_update', + 'isdisjoint', + 'issubset', + 'issuperset', + 'pop', + 'remove', + 'symmetric_difference', + 'symmetric_difference_update', + 'union', + 'update', + ]), + 'str': (str, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mod__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmod__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'capitalize', + 'casefold', + 'center', + 'count', + 'encode', + 'endswith', + 'expandtabs', + 'find', + 'format', + 'format_map', + 'index', + 'isalnum', + 'isalpha', + 'isascii', + 'isdecimal', + 'isdigit', + 'isidentifier', + 'islower', + 'isnumeric', + 'isprintable', + 'isspace', + 'istitle', + 'isupper', + 'join', + 'ljust', + 'lower', + 'lstrip', + 'maketrans', + 'partition', + 'replace', + 'rfind', + 'rindex', + 'rjust', + 'rpartition', + 'rsplit', + 'rstrip', + 'split', + 'splitlines', + 'startswith', + 'strip', + 'swapcase', + 'title', + 'translate', + 'upper', + 'zfill' + ]), + 'tuple': (tuple, [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'index', + ]), + 'object': (object, [ + '__repr__', + '__hash__', + '__str__', + '__getattribute__', + '__setattr__', + '__delattr__', + '__lt__', + '__le__', + '__eq__', + '__ne__', + '__gt__', + '__ge__', + '__init__', + '__new__', + '__reduce_ex__', + '__reduce__', + '__subclasshook__', + '__init_subclass__', + '__format__', + '__sizeof__', + '__dir__', + '__class__', + '__doc__' + ]), +} not_implemented = [] -for item in expected_methods: - for method in item['methods']: +for name, (ty, methods) in expected_methods.items(): + for method in methods: try: - if not hasattr(item['type'], method): - not_implemented.append((item['name'], method)) + if not hasattr(ty, method): + not_implemented.append((name, method)) except NameError: - not_implemented.append((item['name'], method)) + not_implemented.append((name, method)) -for r in not_implemented: - print(r[0], ".", r[1]) +if not_implemented: + for r in not_implemented: + print(r[0], ".", r[1], sep="") else: print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/whats_left.sh b/whats_left.sh new file mode 100755 index 000000000..c6f191bd5 --- /dev/null +++ b/whats_left.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +cd "$(dirname "$0")" || exit + +cargo run -- tests/snippets/whats_left_to_implement.py From 9ff1e41422578fd7566fba4b15d8fdec0a43b9dc Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 1 Apr 2019 18:28:15 -0500 Subject: [PATCH 031/223] Give values instead of types, list comprehension, remove methods not in CPython --- tests/snippets/whats_left_to_implement.py | 49 +++++++++-------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index 41b092506..a9a2d4de9 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,7 +1,7 @@ import platform expected_methods = { - 'bool': (bool, [ + 'bool': (True, [ '__abs__', '__add__', '__and__', @@ -73,7 +73,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'bytearray': (bytearray, [ + 'bytearray': (bytearray(), [ '__add__', '__alloc__', '__class__', @@ -128,7 +128,6 @@ expected_methods = { 'insert', 'isalnum', 'isalpha', - 'isascii', 'isdigit', 'islower', 'isspace', @@ -160,7 +159,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'bytes': (bytes, [ + 'bytes': (b'', [ '__add__', '__class__', '__contains__', @@ -206,7 +205,6 @@ expected_methods = { 'index', 'isalnum', 'isalpha', - 'isascii', 'isdigit', 'islower', 'isspace', @@ -235,7 +233,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'complex': (complex, [ + 'complex': (0j, [ '__abs__', '__add__', '__bool__', @@ -286,7 +284,7 @@ expected_methods = { 'imag', 'real', ]), - 'dict': (dict, [ + 'dict': ({}, [ '__class__', '__contains__', '__delattr__', @@ -328,7 +326,7 @@ expected_methods = { 'update', 'values', ]), - 'float': (float, [ + 'float': (0.0, [ '__abs__', '__add__', '__bool__', @@ -371,7 +369,6 @@ expected_methods = { '__rpow__', '__rsub__', '__rtruediv__', - '__set_format__', '__setattr__', '__sizeof__', '__str__', @@ -387,7 +384,7 @@ expected_methods = { 'is_integer', 'real', ]), - 'frozenset': (frozenset, [ + 'frozenset': (frozenset(), [ '__and__', '__class__', '__contains__', @@ -431,7 +428,7 @@ expected_methods = { 'symmetric_difference', 'union', ]), - 'int': (int, [ + 'int': (0, [ '__abs__', '__add__', '__and__', @@ -503,7 +500,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'iter': (iter, [ + 'iter': ([].__iter__(), [ '__class__', '__delattr__', '__dir__', @@ -532,7 +529,7 @@ expected_methods = { '__str__', '__subclasshook__', ]), - 'list': (list, [ + 'list': ([], [ '__add__', '__class__', '__contains__', @@ -580,7 +577,7 @@ expected_methods = { 'reverse', 'sort', ]), - 'memoryview': (memoryview, [ + 'memoryview': (memoryview(b''), [ '__class__', '__delattr__', '__delitem__', @@ -628,7 +625,7 @@ expected_methods = { 'tobytes', 'tolist', ]), - 'range': (range, [ + 'range': (range(0), [ '__bool__', '__class__', '__contains__', @@ -664,7 +661,7 @@ expected_methods = { 'step', 'stop', ]), - 'set': (set, [ + 'set': (set(), [ '__and__', '__class__', '__contains__', @@ -721,7 +718,7 @@ expected_methods = { 'union', 'update', ]), - 'str': (str, [ + 'str': ('', [ '__add__', '__class__', '__contains__', @@ -768,7 +765,6 @@ expected_methods = { 'index', 'isalnum', 'isalpha', - 'isascii', 'isdecimal', 'isdigit', 'isidentifier', @@ -801,7 +797,7 @@ expected_methods = { 'upper', 'zfill' ]), - 'tuple': (tuple, [ + 'tuple': ((), [ '__add__', '__class__', '__contains__', @@ -836,7 +832,7 @@ expected_methods = { 'count', 'index', ]), - 'object': (object, [ + 'object': (object(), [ '__repr__', '__hash__', '__str__', @@ -863,15 +859,10 @@ expected_methods = { ]), } -not_implemented = [] - -for name, (ty, methods) in expected_methods.items(): - for method in methods: - try: - if not hasattr(ty, method): - not_implemented.append((name, method)) - except NameError: - not_implemented.append((name, method)) +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] if not_implemented: for r in not_implemented: From 98889d5339f8ec2e5c17ca58becf6a66dfbcd6cb Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 1 Apr 2019 13:38:06 -0700 Subject: [PATCH 032/223] match_class! macro --- vm/src/macros.rs | 87 ++++++++++++++++++++++++++++++++++++++++++ vm/src/obj/objfloat.rs | 6 +++ 2 files changed, 93 insertions(+) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index a63186797..601151a05 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -147,3 +147,90 @@ macro_rules! extend_class { )* } } + +/// Macro to match on the built-in class of a Python object. +/// +/// Like `match`, `match_object!` is an expression, and so must have a default +/// arm that will be evaluated when the object was not an instance of any of +/// the classes specified in other arms. +/// +/// # Examples +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// assert_eq!( +/// "int", +/// match_class!(obj.clone(), +/// PyInt => "int", +/// PyFloat => "float", +/// _ => "neither" +/// ) +/// ); +/// +/// ``` +/// +/// With a binding to the downcasted type: +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// +/// let int_value = match_class!(obj, +/// i @ PyInt => i.as_bigint().clone(), +/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), +/// obj => panic!("non-numeric object {}", obj) +/// ); +/// +/// assert!(int_value.is_zero()); +/// ``` +#[macro_export] +macro_rules! match_class { + // The default arm. + ($obj:expr, _ => $default:expr) => { + $default + }; + + // The default arm, binding the original object to the specified identifier. + ($obj:expr, $binding:ident => $default:expr) => {{ + let $binding = $obj; + $default + }}; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier. + ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { + match $obj.downcast::<$class>() { + Ok($binding) => $expr, + Err(_obj) => match_class!(_obj, $($rest)*), + } + }; + + // An arm taken when the object is an instance of the specified built-in + // class. + ($obj:expr, $class:ty => $expr:expr, $($rest:tt)*) => { + if $obj.payload_is::<$class>() { + $expr + } else { + match_class!($obj, $($rest)*) + } + }; +} diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index ca6bad9d2..4ef7123ee 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -16,6 +16,12 @@ pub struct PyFloat { value: f64, } +impl PyFloat { + pub fn to_f64(&self) -> f64 { + self.value + } +} + impl PyValue for PyFloat { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.float_type() From 8549ea22a3d280498d72b10b0fbfa8378f3dd146 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Mon, 1 Apr 2019 20:22:11 -0700 Subject: [PATCH 033/223] Use match_class! in objint::to_int --- vm/src/obj/objint.rs | 37 +++++++++++++++---------------------- vm/src/obj/objstr.rs | 7 +++++++ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 08175dc07..c26cdf4c0 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -12,8 +12,8 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objfloat; -use super::objstr; +use super::objfloat::{self, PyFloat}; +use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -351,7 +351,7 @@ impl PyIntRef { self.value.to_string() } - fn format(self, spec: PyRef, vm: &VirtualMachine) -> PyResult { + fn format(self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -408,29 +408,22 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul // Casting function: pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { - let val = if objtype::isinstance(obj, &vm.ctx.int_type()) { - get_value(obj).clone() - } else if objtype::isinstance(obj, &vm.ctx.float_type()) { - objfloat::get_value(obj).to_bigint().unwrap() - } else if objtype::isinstance(obj, &vm.ctx.str_type()) { - let s = objstr::get_value(obj); - match i32::from_str_radix(&s, base) { - Ok(v) => v.to_bigint().unwrap(), - Err(err) => { - trace!("Error occurred during int conversion {:?}", err); - return Err(vm.new_value_error(format!( + match_class!(obj.clone(), + i @ PyInt => Ok(i.as_bigint().clone()), + f @ PyFloat => Ok(f.to_f64().to_bigint().unwrap()), + s @ PyString => { + i32::from_str_radix(s.as_str(), base) + .map(|i| BigInt::from(i)) + .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s - ))); - } - } - } else { - return Err(vm.new_type_error(format!( + ))) + }, + obj => Err(vm.new_type_error(format!( "int() argument must be a string or a number, not '{}'", obj.class().name - ))); - }; - Ok(val) + ))) + ) } // Retrieve inner int value: diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index aa62812a2..8ac23c160 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -25,6 +25,13 @@ pub struct PyString { // TODO: shouldn't be public pub value: String, } + +impl PyString { + pub fn as_str(&self) -> &str { + &self.value + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { From 75d02a1725ff3b050b395ec79287ca0d85e6948b Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 2 Apr 2019 09:30:18 -0700 Subject: [PATCH 034/223] complex: move to impl block --- vm/src/obj/objcomplex.rs | 242 ++++++++++++++++++++------------------- 1 file changed, 122 insertions(+), 120 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 296830e43..9501ee7f3 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -35,17 +35,17 @@ pub fn init(context: &PyContext) { This is equivalent to (real + imag*1j) where imag defaults to 0."; extend_class!(context, complex_type, { - "__abs__" => context.new_rustfunc(complex_abs), - "__add__" => context.new_rustfunc(complex_add), "__doc__" => context.new_str(complex_doc.to_string()), - "__eq__" => context.new_rustfunc(complex_eq), - "__neg__" => context.new_rustfunc(complex_neg), - "__new__" => context.new_rustfunc(complex_new), - "__radd__" => context.new_rustfunc(complex_radd), - "__repr__" => context.new_rustfunc(complex_repr), - "conjugate" => context.new_rustfunc(complex_conjugate), - "imag" => context.new_property(complex_imag), - "real" => context.new_property(complex_real) + "__abs__" => context.new_rustfunc(PyComplexRef::abs), + "__add__" => context.new_rustfunc(PyComplexRef::add), + "__eq__" => context.new_rustfunc(PyComplexRef::eq), + "__neg__" => context.new_rustfunc(PyComplexRef::neg), + "__new__" => context.new_rustfunc(PyComplexRef::new), + "__radd__" => context.new_rustfunc(PyComplexRef::radd), + "__repr__" => context.new_rustfunc(PyComplexRef::repr), + "conjugate" => context.new_rustfunc(PyComplexRef::conjugate), + "imag" => context.new_property(PyComplexRef::imag), + "real" => context.new_property(PyComplexRef::real) }); } @@ -53,128 +53,130 @@ pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -fn complex_new( - cls: PyClassRef, - real: OptionalArg, - imag: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let real = match real { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; +impl PyComplexRef { + fn new( + cls: PyClassRef, + real: OptionalArg, + imag: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let real = match real { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let imag = match imag { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; + let imag = match imag { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let value = Complex64::new(real, imag); - PyComplex { value }.into_ref_with_type(vm, cls) -} - -fn complex_real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) -} - -fn complex_imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) -} - -fn complex_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) -} - -fn complex_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + let value = Complex64::new(real, imag); + PyComplex { value }.into_ref_with_type(vm, cls) } -} -fn complex_radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + fn real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + let Complex64 { re, .. } = get_value(zelf); + Ok(vm.ctx.new_float(re)) } -} -fn complex_conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + fn imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + let Complex64 { im, .. } = get_value(zelf); + Ok(vm.ctx.new_float(im)) + } - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) -} + fn abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); -fn complex_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); + let Complex64 { re, im } = get_value(zelf); + Ok(vm.ctx.new_float(re.hypot(im))) + } - let z = get_value(zelf); + fn add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(i, Some(vm.ctx.complex_type())), (i2, None)] + ); - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, - None => false, + let v1 = get_value(i); + if objtype::isinstance(i2, &vm.ctx.complex_type()) { + Ok(vm.ctx.new_complex(v1 + get_value(i2))) + } else if objtype::isinstance(i2, &vm.ctx.int_type()) { + Ok(vm.ctx.new_complex(Complex64::new( + v1.re + objint::get_value(i2).to_f64().unwrap(), + v1.im, + ))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) - } else { - return Ok(vm.ctx.not_implemented()); - }; + } - Ok(vm.ctx.new_bool(result)) -} + fn radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(i, Some(vm.ctx.complex_type())), (i2, None)] + ); -fn complex_neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) -} + let v1 = get_value(i); -fn complex_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) - } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + if objtype::isinstance(i2, &vm.ctx.int_type()) { + Ok(vm.ctx.new_complex(Complex64::new( + v1.re + objint::get_value(i2).to_f64().unwrap(), + v1.im, + ))) + } else { + Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + } + } + + fn conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + + let v1 = get_value(i); + Ok(vm.ctx.new_complex(v1.conj())) + } + + fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] + ); + + let z = get_value(zelf); + + let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { + z == get_value(other) + } else if objtype::isinstance(other, &vm.ctx.int_type()) { + match objint::get_value(other).to_f64() { + Some(f) => z.im == 0.0f64 && z.re == f, + None => false, + } + } else if objtype::isinstance(other, &vm.ctx.float_type()) { + z.im == 0.0 && z.re == objfloat::get_value(other) + } else { + return Ok(vm.ctx.not_implemented()); + }; + + Ok(vm.ctx.new_bool(result)) + } + + fn neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + Ok(vm.ctx.new_complex(-get_value(zelf))) + } + + fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); + let v = get_value(obj); + let repr = if v.re == 0. { + format!("{}j", v.im) + } else { + format!("({}+{}j)", v.re, v.im) + }; + Ok(vm.new_str(repr)) + } } From 6863c19fb7aa8a87eae9e022080b586f092fcfcb Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 2 Apr 2019 09:51:01 -0700 Subject: [PATCH 035/223] complex: convert to new args style --- vm/src/obj/objcomplex.rs | 127 ++++++++++++++------------------------- 1 file changed, 46 insertions(+), 81 deletions(-) diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 9501ee7f3..9f88dd6a4 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,11 +1,11 @@ use num_complex::Complex64; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objfloat; +use super::objfloat::{self, PyFloat}; use super::objint; use super::objtype::{self, PyClassRef}; @@ -74,109 +74,74 @@ impl PyComplexRef { PyComplex { value }.into_ref_with_type(vm, cls) } - fn real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) + fn real(self, _vm: &VirtualMachine) -> PyFloat { + self.value.re.into() } - fn imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) + fn imag(self, _vm: &VirtualMachine) -> PyFloat { + self.value.im.into() } - fn abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) + fn abs(self, _vm: &VirtualMachine) -> PyFloat { + let Complex64 { im, re } = self.value; + re.hypot(im).into() } - fn add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.complex_type()) { + vm.ctx.new_complex(self.value + get_value(&other)) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + vm.ctx.not_implemented() } } - fn radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) + fn radd(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + vm.ctx.not_implemented() } } - fn conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); - - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) + fn conjugate(self, _vm: &VirtualMachine) -> PyComplex { + self.value.conj().into() } - fn eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); - - let z = get_value(zelf); - - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { + self.value == get_value(&other) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + match objint::get_value(&other).to_f64() { + Some(f) => self.value.im == 0.0f64 && self.value.re == f, None => false, } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) + } else if objtype::isinstance(&other, &vm.ctx.float_type()) { + self.value.im == 0.0 && self.value.re == objfloat::get_value(&other) } else { - return Ok(vm.ctx.not_implemented()); + return vm.ctx.not_implemented(); }; - Ok(vm.ctx.new_bool(result)) + vm.ctx.new_bool(result) } - fn neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) + fn neg(self, _vm: &VirtualMachine) -> PyComplex { + PyComplex::from(-self.value) } - fn repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) + fn repr(self, _vm: &VirtualMachine) -> String { + let Complex64 { re, im } = self.value; + if re == 0.0 { + format!("{}j", im) } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + format!("({}+{}j)", re, im) + } } } From 2fb3fc92ecaa13d031bdd3dd2e83ccf778660b46 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Tue, 2 Apr 2019 20:50:14 +0200 Subject: [PATCH 036/223] Support chained comparisons --- parser/src/ast.rs | 5 +- parser/src/parser.rs | 36 ++++++++------ parser/src/python.lalrpop | 10 +++- tests/snippets/ast_snippet.py | 7 +++ tests/snippets/comparisons.py | 13 +++++ vm/src/compile.rs | 93 ++++++++++++++++++++++++++++------- vm/src/stdlib/ast.rs | 20 ++++++-- 7 files changed, 141 insertions(+), 43 deletions(-) create mode 100644 tests/snippets/comparisons.py diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 257ceadf2..ece0e9798 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -159,9 +159,8 @@ pub enum Expression { value: Box, }, Compare { - a: Box, - op: Comparison, - b: Box, + vals: Vec, + ops: Vec, }, Attribute { value: Box, diff --git a/parser/src/parser.rs b/parser/src/parser.rs index a119740ec..4c37f2842 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -460,26 +460,30 @@ mod tests { }, ifs: vec![ ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Less, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(5) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(5) + } } - }), + ], + ops: vec![ast::Comparison::Less], }, ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Greater, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(10) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(10) + } } - }), + ], + ops: vec![ast::Comparison::Greater], }, ], } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 83020e926..f4c81f1d8 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -680,7 +680,15 @@ NotTest: ast::Expression = { }; Comparison: ast::Expression = { - => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) }, + => { + let mut vals = vec![e]; + let mut ops = vec![]; + for x in comparisons { + ops.push(x.0); + vals.push(x.1); + } + ast::Expression::Compare { vals, ops } + }, => e, }; diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index 43bf74756..dad767b45 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -21,3 +21,10 @@ print(foo.body[0].value.func.id) assert foo.body[0].value.func.id == 'print' assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 + +n = ast.parse("3 < 4 > 5\n") +assert n.body[0].value.left.n == 3 +assert 'Lt' in str(n.body[0].value.ops[0]) +assert 'Gt' in str(n.body[0].value.ops[1]) +assert n.body[0].value.comparators[0].n == 4 +assert n.body[0].value.comparators[1].n == 5 diff --git a/tests/snippets/comparisons.py b/tests/snippets/comparisons.py new file mode 100644 index 000000000..1a4597ef6 --- /dev/null +++ b/tests/snippets/comparisons.py @@ -0,0 +1,13 @@ + +assert 1 < 2 +assert 1 < 2 < 3 +assert 5 == 5 == 5 +assert (5 == 5) == True +assert 5 == 5 != 4 == 4 > 3 > 2 < 3 <= 3 != 0 == 0 + +assert not 1 > 2 +assert not 5 == 5 == True +assert not 5 == 5 != 5 == 5 +assert not 1 < 2 < 3 > 4 +assert not 1 < 2 > 3 < 4 +assert not 1 > 2 < 3 < 4 diff --git a/vm/src/compile.rs b/vm/src/compile.rs index f79980af9..fe295f424 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -840,6 +840,79 @@ impl Compiler { Ok(()) } + fn compile_chained_comparison( + &mut self, + vals: &[ast::Expression], + ops: &[ast::Comparison], + ) -> Result<(), CompileError> { + assert!(ops.len() > 0); + assert!(vals.len() == ops.len() + 1); + + let to_operator = |op: &ast::Comparison| match op { + ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, + ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, + ast::Comparison::Less => bytecode::ComparisonOperator::Less, + ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, + ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, + ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, + ast::Comparison::In => bytecode::ComparisonOperator::In, + ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, + ast::Comparison::Is => bytecode::ComparisonOperator::Is, + ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, + }; + + // a == b == c == d + // compile into (pseudocode): + // result = a == b + // if result: + // result = b == c + // if result: + // result = c == d + + // initialize lhs outside of loop + self.compile_expression(&vals[0])?; + + let break_label = self.new_label(); + let last_label = self.new_label(); + + // for all comparisons except the last (as the last one doesn't need a conditional jump) + let ops_slice = &ops[0..ops.len()]; + let vals_slice = &vals[1..ops.len()]; + for (op, val) in ops_slice.iter().zip(vals_slice.iter()) { + self.compile_expression(val)?; + // store rhs for the next comparison in chain + self.emit(Instruction::Duplicate); + self.emit(Instruction::Rotate { amount: 3 }); + + self.emit(Instruction::CompareOperation { + op: to_operator(op), + }); + + // if comparison result is false, we break with this value; if true, try the next one. + // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP) + self.emit(Instruction::Duplicate); + self.emit(Instruction::JumpIfFalse { + target: break_label, + }); + self.emit(Instruction::Pop); + } + + // handle the last comparison + self.compile_expression(vals.last().unwrap())?; + self.emit(Instruction::CompareOperation { + op: to_operator(ops.last().unwrap()), + }); + self.emit(Instruction::Jump { target: last_label }); + + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.set_label(break_label); + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::Pop); + + self.set_label(last_label); + Ok(()) + } + fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { @@ -1022,24 +1095,8 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Compare { a, op, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - - let i = match op { - ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, - ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, - ast::Comparison::Less => bytecode::ComparisonOperator::Less, - ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, - ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, - ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, - ast::Comparison::In => bytecode::ComparisonOperator::In, - ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, - ast::Comparison::Is => bytecode::ComparisonOperator::Is, - ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, - }; - let i = Instruction::CompareOperation { op: i }; - self.emit(i); + ast::Expression::Compare { vals, ops } => { + self.compile_chained_comparison(vals, ops)?; } ast::Expression::Number { value } => { let const_value = match value { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 0011407ab..91631b0c9 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -327,14 +327,14 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj node } - ast::Expression::Compare { a, op, b } => { + ast::Expression::Compare { vals, ops } => { let node = create_node(vm, "Compare"); - let py_a = expression_to_ast(vm, a); + let py_a = expression_to_ast(vm, &vals[0]); vm.ctx.set_attr(&node, "left", py_a); // Operator: - let str_op = match op { + let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => "Eq", ast::Comparison::NotEqual => "NotEq", ast::Comparison::Less => "Lt", @@ -346,10 +346,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = vm.ctx.new_list(vec![vm.ctx.new_str(str_op.to_string())]); + let py_ops = vm.ctx.new_list( + ops.iter() + .map(|x| vm.ctx.new_str(to_operator(x).to_string())) + .collect(), + ); + vm.ctx.set_attr(&node, "ops", py_ops); - let py_b = vm.ctx.new_list(vec![expression_to_ast(vm, b)]); + let py_b = vm.ctx.new_list( + vals.iter() + .skip(1) + .map(|x| expression_to_ast(vm, x)) + .collect(), + ); vm.ctx.set_attr(&node, "comparators", py_b); node } From 23f57f757ccb5074e4c1e0c557fcf48bffaf9870 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 3 Apr 2019 14:08:12 +0200 Subject: [PATCH 037/223] Improve trailing comma situation --- parser/src/ast.rs | 6 ++-- parser/src/python.lalrpop | 61 +++++++++++++++--------------------- tests/snippets/function.py | 4 +++ tests/snippets/generators.py | 7 +++++ vm/src/compile.rs | 21 +++---------- vm/src/stdlib/ast.rs | 5 ++- 6 files changed, 45 insertions(+), 59 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index ece0e9798..21c8dd736 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -47,7 +47,7 @@ pub enum Statement { Break, Continue, Return { - value: Option>, + value: Option>, }, Import { import_parts: Vec, @@ -94,7 +94,7 @@ pub enum Statement { }, For { target: Expression, - iter: Vec, + iter: Expression, body: Vec, orelse: Option>, }, @@ -114,12 +114,10 @@ pub enum Statement { bases: Vec, keywords: Vec, decorator_list: Vec, - // TODO: docstring: String, }, FunctionDef { name: String, args: Parameters, - // docstring: String, body: Vec, decorator_list: Vec, returns: Option, diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index f4c81f1d8..151e06de2 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -102,7 +102,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { + => { // TODO: this works in most cases: let rhs = e2.into_iter().next().unwrap(); ast::LocatedStatement { @@ -117,15 +117,7 @@ ExpressionStatement: ast::LocatedStatement = { }; AssignSuffix: ast::Expression = { - "=" => { - if e.len() > 1 { - ast::Expression::Tuple { - elements: e - } - } else { - e.into_iter().next().unwrap() - } - }, + "=" => e, "=" => e, }; @@ -182,7 +174,7 @@ FlowStatement: ast::LocatedStatement = { "return" => { ast::LocatedStatement { location: loc, - node: ast::Statement::Return { value: t }, + node: ast::Statement::Return { value: t.map(Box::new) }, } }, => { @@ -622,22 +614,8 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => { - ast::Expression::Yield { - value: ex.map(|expr| Box::new( - if expr.len() > 1 { - ast::Expression::Tuple { elements: expr } - } else { - expr.into_iter().next().unwrap() - }) - ) - } - }, - "yield" "from" => { - ast::Expression::YieldFrom { - value: Box::new(e), - } - }, + "yield" => ast::Expression::Yield { value: value.map(Box::new) }, + "yield" "from" => ast::Expression::YieldFrom { value: Box::new(e) }, }; Test: ast::Expression = { @@ -660,7 +638,7 @@ LambdaDef: ast::Expression = { "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body:Box::new(b) + body: Box::new(b) } } @@ -817,7 +795,7 @@ Atom: ast::Expression = { // List comprehension: e }, - "(" ")" => { + "(" ")" => { match e { None => ast::Expression::Tuple { elements: Vec::new() }, Some(elements) => { @@ -913,13 +891,26 @@ ExpressionList2: Vec = { }, }; +// TODO: this TestList2 should be renamed or removed. #[inline] -TestList: Vec = { - => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - } +TestList2: Vec = { + > => elements, +}; + +// A test list is one of: +// - a list of expressions +// - a single expression +// - a single expression followed by a trailing comma +TestList: ast::Expression = { + > => { + if e.len() == 1 && trailing_comma.is_none() { + e.into_iter().next().unwrap() + } else { + ast::Expression::Tuple { + elements: e + } + } + } }; // Test diff --git a/tests/snippets/function.py b/tests/snippets/function.py index 7d3f25709..aafc9b53c 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -10,6 +10,10 @@ def my_func(a,): assert my_func(2) == 4 +def fubar(): + return 42, + +assert fubar() == (42,) def f1(): diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 5f64c4905..65044c0da 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -35,3 +35,10 @@ r = list(g3()) # print(r) assert r == [23, 1, 2, 3, 44] +def g4(): + yield + yield 2, + +r = list(g4()) +assert r == [None, (2,)] + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index fe295f424..88818f52f 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -353,19 +353,8 @@ impl Compiler { return Err(CompileError::InvalidReturn); } match value { - Some(e) => { - let size = e.len(); - for v in e { - self.compile_expression(v)?; - } - - // If we have more than 1 return value, make it a tuple: - if size > 1 { - self.emit(Instruction::BuildTuple { - size, - unpack: false, - }); - } + Some(v) => { + self.compile_expression(v)?; } None => { self.emit(Instruction::LoadConst { @@ -796,7 +785,7 @@ impl Compiler { fn compile_for( &mut self, target: &ast::Expression, - iter: &[ast::Expression], + iter: &ast::Expression, body: &[ast::LocatedStatement], orelse: &Option>, ) -> Result<(), CompileError> { @@ -810,9 +799,7 @@ impl Compiler { }); // The thing iterated: - for i in iter { - self.compile_expression(i)?; - } + self.compile_expression(iter)?; // Retrieve Iterator self.emit(Instruction::GetIter); diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 91631b0c9..65fcdf660 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -151,8 +151,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "Return"); let py_value = if let Some(value) = value { - vm.ctx - .new_tuple(value.iter().map(|v| expression_to_ast(vm, v)).collect()) + expression_to_ast(vm, value) } else { vm.ctx.none() }; @@ -189,7 +188,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let py_target = expression_to_ast(vm, target); vm.ctx.set_attr(&node, "target", py_target); - let py_iter = expressions_to_ast(vm, iter); + let py_iter = expression_to_ast(vm, iter); vm.ctx.set_attr(&node, "iter", py_iter); let py_body = statements_to_ast(vm, body); From e078614e05b25b2a6dc08e1c81d1f0804512c6cb Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 3 Apr 2019 08:50:55 -0500 Subject: [PATCH 038/223] Change to classes and remove unnecesary if --- tests/snippets/whats_left_to_implement.py | 37 +++++++++++------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py index a9a2d4de9..6abade9df 100644 --- a/tests/snippets/whats_left_to_implement.py +++ b/tests/snippets/whats_left_to_implement.py @@ -1,7 +1,7 @@ import platform expected_methods = { - 'bool': (True, [ + 'bool': (bool, [ '__abs__', '__add__', '__and__', @@ -73,7 +73,7 @@ expected_methods = { 'real', 'to_bytes', ]), - 'bytearray': (bytearray(), [ + 'bytearray': (bytearray, [ '__add__', '__alloc__', '__class__', @@ -159,7 +159,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'bytes': (b'', [ + 'bytes': (bytes, [ '__add__', '__class__', '__contains__', @@ -233,7 +233,7 @@ expected_methods = { 'upper', 'zfill', ]), - 'complex': (0j, [ + 'complex': (complex, [ '__abs__', '__add__', '__bool__', @@ -284,7 +284,7 @@ expected_methods = { 'imag', 'real', ]), - 'dict': ({}, [ + 'dict': (dict, [ '__class__', '__contains__', '__delattr__', @@ -326,7 +326,7 @@ expected_methods = { 'update', 'values', ]), - 'float': (0.0, [ + 'float': (float, [ '__abs__', '__add__', '__bool__', @@ -384,7 +384,7 @@ expected_methods = { 'is_integer', 'real', ]), - 'frozenset': (frozenset(), [ + 'frozenset': (frozenset, [ '__and__', '__class__', '__contains__', @@ -428,7 +428,7 @@ expected_methods = { 'symmetric_difference', 'union', ]), - 'int': (0, [ + 'int': (int, [ '__abs__', '__add__', '__and__', @@ -529,7 +529,7 @@ expected_methods = { '__str__', '__subclasshook__', ]), - 'list': ([], [ + 'list': (list, [ '__add__', '__class__', '__contains__', @@ -577,7 +577,7 @@ expected_methods = { 'reverse', 'sort', ]), - 'memoryview': (memoryview(b''), [ + 'memoryview': (memoryview, [ '__class__', '__delattr__', '__delitem__', @@ -625,7 +625,7 @@ expected_methods = { 'tobytes', 'tolist', ]), - 'range': (range(0), [ + 'range': (range, [ '__bool__', '__class__', '__contains__', @@ -661,7 +661,7 @@ expected_methods = { 'step', 'stop', ]), - 'set': (set(), [ + 'set': (set, [ '__and__', '__class__', '__contains__', @@ -718,7 +718,7 @@ expected_methods = { 'union', 'update', ]), - 'str': ('', [ + 'str': (str, [ '__add__', '__class__', '__contains__', @@ -797,7 +797,7 @@ expected_methods = { 'upper', 'zfill' ]), - 'tuple': ((), [ + 'tuple': (tuple, [ '__add__', '__class__', '__contains__', @@ -832,7 +832,7 @@ expected_methods = { 'count', 'index', ]), - 'object': (object(), [ + 'object': (object, [ '__repr__', '__hash__', '__str__', @@ -864,10 +864,9 @@ not_implemented = [(name, method) for method in methods if not hasattr(val, method)] -if not_implemented: - for r in not_implemented: - print(r[0], ".", r[1], sep="") -else: +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: print("Not much \\o/") if platform.python_implementation() == "CPython": From 29a137bf0f8722fd449e18997ced1dc964c088ea Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 3 Apr 2019 22:24:19 +0200 Subject: [PATCH 039/223] Drop testlist2. Also add test cases for trailing comma in for statement and tuple addition example. --- parser/src/python.lalrpop | 182 ++++++++++++++------------------------ tests/snippets/for.py | 12 +++ tests/snippets/tuple.py | 4 + 3 files changed, 81 insertions(+), 117 deletions(-) diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 151e06de2..305f97c10 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -102,9 +102,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { - // TODO: this works in most cases: - let rhs = e2.into_iter().next().unwrap(); + => { ast::LocatedStatement { location: loc, node: ast::Statement::AugAssign { @@ -122,18 +120,12 @@ AssignSuffix: ast::Expression = { }; TestOrStarExprList: ast::Expression = { - => { - let mut res = vec![e]; - res.extend(e2.into_iter().map(|x| x.1)); - - // First build tuple from first item: - let expr = if (res.len() > 1) || comma.is_some() { - ast::Expression::Tuple { elements: res } + > => { + if elements.len() == 1 && comma.is_none() { + elements.into_iter().next().unwrap() } else { - res.into_iter().next().unwrap() - }; - - expr + ast::Expression::Tuple { elements } + } } }; @@ -301,15 +293,11 @@ NonlocalStatement: ast::LocatedStatement = { }; AssertStatement: ast::LocatedStatement = { - "assert" => { + "assert" => { ast::LocatedStatement { location: loc, node: ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, - } + test, msg: msg.map(|e| e.1) } } }, @@ -326,7 +314,7 @@ CompoundStatement: ast::LocatedStatement = { }; IfStatement: ast::LocatedStatement = { - "if" ":" => { + "if" ":" => { // Determine last else: let mut last = s3.map(|s| s.2); @@ -341,17 +329,17 @@ IfStatement: ast::LocatedStatement = { ast::LocatedStatement { location: loc, - node: ast::Statement::If { test: t, body: s1, orelse: last } + node: ast::Statement::If { test, body: s1, orelse: last } } }, }; WhileStatement: ast::LocatedStatement = { - "while" ":" => { + "while" ":" => { let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::While { test: e, body: s, orelse: or_else }, + node: ast::Statement::While { test, body, orelse: or_else }, } }, }; @@ -434,9 +422,7 @@ FuncDef: ast::LocatedStatement = { }; Parameters: ast::Parameters = { - "(" )?> ")" => { - a.unwrap_or_else(Default::default) - }, + "(" )?> ")" => a.unwrap_or_else(Default::default), }; // Note that this is a macro which is used once for function defs, and @@ -619,42 +605,41 @@ YieldExpr: ast::Expression = { }; Test: ast::Expression = { - => { - match c { - Some(c) => { - ast::Expression::IfExpression { - test: Box::new(c.1), - body: Box::new(e), - orelse: Box::new(c.3), - } - }, - None => e, + => { + if let Some(c) = condition { + ast::Expression::IfExpression { + test: Box::new(c.1), + body: Box::new(expr), + orelse: Box::new(c.3), + } + } else { + expr } }, - => e, + LambdaDef, }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body: Box::new(b) + body: Box::new(body) } } OrTest: ast::Expression = { - => e, + AndTest, "or" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }, }; AndTest: ast::Expression = { - => e, + NotTest, "and" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }, }; NotTest: ast::Expression = { "not" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }, - => e, + Comparison, }; Comparison: ast::Expression = { @@ -667,7 +652,7 @@ Comparison: ast::Expression = { } ast::Expression::Compare { vals, ops } }, - => e, + Expression, }; CompOp: ast::Comparison = { @@ -685,7 +670,7 @@ CompOp: ast::Comparison = { Expression: ast::Expression = { "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, - => e, + XorExpression, }; XorExpression: ast::Expression = { @@ -735,7 +720,7 @@ Factor: ast::Expression = { "+" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos }, "-" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg }, "~" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv }, - => e, + Power, }; Power: ast::Expression = { @@ -748,7 +733,7 @@ Power: ast::Expression = { }; AtomExpr: ast::Expression = { - => e, + Atom, "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, "." => ast::Expression::Attribute { value: Box::new(e), name: n }, @@ -769,7 +754,7 @@ SubscriptList: ast::Expression = { }; Subscript: ast::Expression = { - => e, + Test, ":" => { let s1 = e1.unwrap_or(ast::Expression::None); let s2 = e2.unwrap_or(ast::Expression::None); @@ -783,30 +768,17 @@ SliceOp: ast::Expression = { } Atom: ast::Expression = { - => ast::Expression::String { value: s }, - => ast::Expression::Bytes { value: b }, - => ast::Expression::Number { value: n }, - => ast::Expression::Identifier { name: i }, + => ast::Expression::String { value }, + => ast::Expression::Bytes { value }, + => ast::Expression::Number { value }, + => ast::Expression::Identifier { name }, "[" "]" => { let elements = e.unwrap_or(Vec::new()); ast::Expression::List { elements } }, - "[" "]" => { - // List comprehension: - e - }, - "(" ")" => { - match e { - None => ast::Expression::Tuple { elements: Vec::new() }, - Some(elements) => { - if elements.len() == 1 && trailing_comma.is_none() { - // This is "(e)", which is equivalent to "e" - elements.into_iter().next().unwrap() - } else { - ast::Expression::Tuple { elements } - } - } - } + "[" "]" => e, + "(" ")" => { + elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() }) }, "(" ")" => { ast::Expression::Comprehension { @@ -825,9 +797,7 @@ Atom: ast::Expression = { }; TestListComp: Vec = { - > <_trailing_comma:","?> => { - e - }, + > <_trailing_comma:","?> => e, }; TestListComp2: ast::Expression = { @@ -840,9 +810,7 @@ TestListComp2: ast::Expression = { }; TestDict: Vec<(ast::Expression, ast::Expression)> = { - > <_trailing_comma:","?> => { - e1 - } + > <_trailing_comma:","?> => elements, }; TestDictComp: ast::Expression = { @@ -859,9 +827,7 @@ DictEntry: (ast::Expression, ast::Expression) = { }; TestSet: Vec = { - > ","? => { - e1 - } + > ","? => e1 }; TestSetComp: ast::Expression = { @@ -873,28 +839,23 @@ TestSetComp: ast::Expression = { } }; +ExpressionOrStarExpression = { + Expression, + StarExpr +}; + ExpressionList: ast::Expression = { - => { - if e.len() == 1 { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements: e } + ast::Expression::Tuple { elements } } }, }; ExpressionList2: Vec = { - ","? => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - }, -}; - -// TODO: this TestList2 should be renamed or removed. -#[inline] -TestList2: Vec = { - > => elements, + > ","? => elements, }; // A test list is one of: @@ -902,13 +863,11 @@ TestList2: Vec = { // - a single expression // - a single expression followed by a trailing comma TestList: ast::Expression = { - > => { - if e.len() == 1 && trailing_comma.is_none() { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { - elements: e - } + ast::Expression::Tuple { elements } } } }; @@ -919,27 +878,16 @@ StarExpr: ast::Expression = { }; // Comprehensions: -CompFor: Vec = { - => c, -}; +CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { - ast::Comprehension { - target: e, - iter: i, - ifs: c2, - } + "for" "in" => { + ast::Comprehension { target, iter, ifs: c2 } } }; -ExpressionNoCond: ast::Expression = { - OrTest, -}; - -ComprehensionIf: ast::Expression = { - "if" => c, -}; +ExpressionNoCond: ast::Expression = OrTest; +ComprehensionIf: ast::Expression = "if" => c; ArgumentList: (Vec, Vec) = { > => { @@ -952,7 +900,7 @@ ArgumentList: (Vec, Vec) = { }, None => { if keywords.len() > 0 { - panic!("positional argument follows keyword argument"); + panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); }, @@ -996,8 +944,8 @@ OneOrMore: Vec = { }; Number: ast::Number = { - => { ast::Number::Integer { value: s } }, - => { ast::Number::Float { value: s } }, + => { ast::Number::Integer { value } }, + => { ast::Number::Float { value } }, => { ast::Number::Complex { real: s.0, imag: s.1 } }, }; diff --git a/tests/snippets/for.py b/tests/snippets/for.py index 07e138e3c..e913ac907 100644 --- a/tests/snippets/for.py +++ b/tests/snippets/for.py @@ -10,3 +10,15 @@ else: x = 3 assert x == 3 + +y = [] +for x, in [(9,), [2]]: + y.append(x) + +assert y == [9, 2], str(y) + +y = [] +for x, *z in [(9,88,'b'), [2, 'bla'], [None]*4]: + y.append(z) + +assert y == [[88, 'b'], ['bla'], [None]*3], str(y) diff --git a/tests/snippets/tuple.py b/tests/snippets/tuple.py index d50f552dc..b7393fe00 100644 --- a/tests/snippets/tuple.py +++ b/tests/snippets/tuple.py @@ -34,3 +34,7 @@ class Foo(object): foo = Foo() assert (foo,) == (foo,) + +a = (1, 2, 3) +a += 1, +assert a == (1, 2, 3, 1) From 88bf2a32c4452a92bb09c4a9b5513701c55f9aa3 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Wed, 3 Apr 2019 17:30:04 -0700 Subject: [PATCH 040/223] address feedback --- vm/src/macros.rs | 13 ++++++------- vm/src/obj/objint.rs | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 601151a05..47d3c531a 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -150,9 +150,8 @@ macro_rules! extend_class { /// Macro to match on the built-in class of a Python object. /// -/// Like `match`, `match_object!` is an expression, and so must have a default -/// arm that will be evaluated when the object was not an instance of any of -/// the classes specified in other arms. +/// Like `match`, `match_class!` must be exhaustive, so a default arm with +/// the uncasted object is required. /// /// # Examples /// @@ -173,7 +172,7 @@ macro_rules! extend_class { /// match_class!(obj.clone(), /// PyInt => "int", /// PyFloat => "float", -/// _ => "neither" +/// _ => "neither", /// ) /// ); /// @@ -197,7 +196,7 @@ macro_rules! extend_class { /// let int_value = match_class!(obj, /// i @ PyInt => i.as_bigint().clone(), /// f @ PyFloat => f.to_f64().to_bigint().unwrap(), -/// obj => panic!("non-numeric object {}", obj) +/// obj => panic!("non-numeric object {}", obj), /// ); /// /// assert!(int_value.is_zero()); @@ -205,12 +204,12 @@ macro_rules! extend_class { #[macro_export] macro_rules! match_class { // The default arm. - ($obj:expr, _ => $default:expr) => { + ($obj:expr, _ => $default:expr $(,)?) => { $default }; // The default arm, binding the original object to the specified identifier. - ($obj:expr, $binding:ident => $default:expr) => {{ + ($obj:expr, $binding:ident => $default:expr $(,)?) => {{ let $binding = $obj; $default }}; diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index c26cdf4c0..d7b6a0420 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -407,6 +407,7 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul } // Casting function: +// TODO: this should just call `__int__` on the object pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { match_class!(obj.clone(), i @ PyInt => Ok(i.as_bigint().clone()), From fa53d5cd655be6956fa0086d9441a866938e19f6 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Wed, 3 Apr 2019 21:11:03 -0600 Subject: [PATCH 041/223] impl list.__delitem__ using list.__setitem__ as a guide while it works as well as __setitem__ currently does this impliments no support for slice indexing or negative indexing, adtionaly a bad index panics insted of giving a vm type error! --- vm/src/obj/objlist.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 865707201..1558db6aa 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -306,6 +306,25 @@ impl PyListRef { Ok(vm.ctx.not_implemented()) } } + + fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.borrow_mut(); + if objtype::isinstance(&key, &vm.ctx.int_type()) { + let idx = objint::get_value(&key).to_i32().unwrap(); + if let Some(_) = elements.get_pos(idx) { + elements.remove(idx as usize); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list index out of range".to_string())) + } + + } else { + panic!( + "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + elements, key + ) + } + } } fn list_new( @@ -454,6 +473,7 @@ pub fn init(context: &PyContext) { "__iadd__" => context.new_rustfunc(PyListRef::iadd), "__bool__" => context.new_rustfunc(PyListRef::bool), "__contains__" => context.new_rustfunc(PyListRef::contains), + "__delitem__" => context.new_rustfunc(PyListRef::delitem), "__eq__" => context.new_rustfunc(PyListRef::eq), "__lt__" => context.new_rustfunc(PyListRef::lt), "__gt__" => context.new_rustfunc(PyListRef::gt), From 39d4d64a4cb1f15af9a5900df6af76c98bbffbb4 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 3 Apr 2019 17:11:26 +1300 Subject: [PATCH 042/223] Copy abc and dependent python files over from CPython 3.7 branch --- Lib/_py_abc.py | 147 +++++++++++++++++++++ Lib/_weakrefset.py | 196 ++++++++++++++++++++++++++++ Lib/abc.py | 170 +++++++++++++++++++++++++ PSF-LICENSE | 254 +++++++++++++++++++++++++++++++++++++ tests/snippets/abc_test.py | 4 + 5 files changed, 771 insertions(+) create mode 100644 Lib/_py_abc.py create mode 100644 Lib/_weakrefset.py create mode 100644 Lib/abc.py create mode 100644 PSF-LICENSE create mode 100644 tests/snippets/abc_test.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py new file mode 100644 index 000000000..3c3aa8e3d --- /dev/null +++ b/Lib/_py_abc.py @@ -0,0 +1,147 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py new file mode 100644 index 000000000..304c66f59 --- /dev/null +++ b/Lib/_weakrefset.py @@ -0,0 +1,196 @@ +# Access WeakSet through the weakref module. +# This code is separated-out because it is needed +# by abc.py to load everything else at startup. + +from _weakref import ref + +__all__ = ['WeakSet'] + + +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + l = self._pending_removals + discard = self.data.discard + while l: + discard(l.pop()) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return (self.__class__, (list(self),), + getattr(self, '__dict__', None)) + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + newset = self.copy() + newset.difference_update(other) + return newset + __sub__ = difference + + def difference_update(self, other): + self.__isub__(other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + __and__ = intersection + + def intersection_update(self, other): + self.__iand__(other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __le__ = issubset + + def __lt__(self, other): + return self.data < set(map(ref, other)) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __ge__ = issuperset + + def __gt__(self, other): + return self.data > set(map(ref, other)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(map(ref, other)) + + def symmetric_difference(self, other): + newset = self.copy() + newset.symmetric_difference_update(other) + return newset + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + self.__ixor__(other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) + return self + + def union(self, other): + return self.__class__(e for s in (self, other) for e in s) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 diff --git a/Lib/abc.py b/Lib/abc.py new file mode 100644 index 000000000..709414127 --- /dev/null +++ b/Lib/abc.py @@ -0,0 +1,170 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) according to PEP 3119.""" + + +def abstractmethod(funcobj): + """A decorator indicating abstract methods. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractmethod + def my_abstract_method(self, ...): + ... + """ + funcobj.__isabstractmethod__ = True + return funcobj + + +class abstractclassmethod(classmethod): + """A decorator indicating abstract classmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractclassmethod + def my_abstract_classmethod(cls, ...): + ... + + 'abstractclassmethod' is deprecated. Use 'classmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractstaticmethod(staticmethod): + """A decorator indicating abstract staticmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractstaticmethod + def my_abstract_staticmethod(...): + ... + + 'abstractstaticmethod' is deprecated. Use 'staticmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractproperty(property): + """A decorator indicating abstract properties. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract properties are overridden. + The abstract properties can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractproperty + def my_abstract_property(self): + ... + + This defines a read-only property; you can also define a read-write + abstract property using the 'long' form of property declaration: + + class C(metaclass=ABCMeta): + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) + + 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' + instead. + """ + + __isabstractmethod__ = True + + +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + from _py_abc import ABCMeta, get_cache_token + ABCMeta.__module__ = 'abc' +else: + class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () diff --git a/PSF-LICENSE b/PSF-LICENSE new file mode 100644 index 000000000..1afbedba9 --- /dev/null +++ b/PSF-LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py new file mode 100644 index 000000000..a5b53ea61 --- /dev/null +++ b/tests/snippets/abc_test.py @@ -0,0 +1,4 @@ +import sys +sys.path.append('Lib') + +import abc From 0bd08618afc87e5b787ca3a0b1efe3622bad2e7e Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 3 Apr 2019 20:36:53 +1300 Subject: [PATCH 043/223] Change _py_abc to workaround current issue, and run test with PYTHONPATH linking to Lib directory. --- Lib/_py_abc.py | 2 +- tests/snippets/abc_test.py | 3 --- tests/test_snippets.py | 3 ++- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 3c3aa8e3d..60adace9b 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -33,7 +33,7 @@ class ABCMeta(type): _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, **kwargs): - cls = super().__new__(mcls, name, bases, namespace, **kwargs) + cls = type.__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py index a5b53ea61..494f967be 100644 --- a/tests/snippets/abc_test.py +++ b/tests/snippets/abc_test.py @@ -1,4 +1 @@ -import sys -sys.path.append('Lib') - import abc diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7edbd7b7e..b3bc79ee7 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -27,7 +27,7 @@ TEST_DIRS = { } CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) - +RUSTPYTHON_LIB_DIR = os.path.abspath(os.path.join(ROOT_DIR, "Lib")) @contextlib.contextmanager def pushd(path): @@ -73,6 +73,7 @@ def run_via_rustpython(filename, test_type): env = os.environ.copy() env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' + env['PYTHONPATH'] = RUSTPYTHON_LIB_DIR target = 'release' if env.get('CODE_COVERAGE', 'false') == 'true': From ea872521d6e114cc99cbd7dc0f08ac2f4b2467bf Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 4 Apr 2019 19:45:20 +1300 Subject: [PATCH 044/223] Make object by default hashable --- Lib/_py_abc.py | 2 +- tests/snippets/abc_test.py | 1 - tests/snippets/hash.py | 23 +++++++++++++++++++++++ tests/snippets/test_abc.py | 34 ++++++++++++++++++++++++++++++++++ vm/src/obj/objdict.rs | 5 +++++ vm/src/obj/objlist.rs | 5 +++++ vm/src/obj/objobject.rs | 4 ++-- vm/src/obj/objset.rs | 5 +++++ 8 files changed, 75 insertions(+), 4 deletions(-) delete mode 100644 tests/snippets/abc_test.py create mode 100644 tests/snippets/hash.py create mode 100644 tests/snippets/test_abc.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 60adace9b..07044f2f7 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -43,7 +43,7 @@ class ABCMeta(type): value = getattr(cls, name, None) if getattr(value, "__isabstractmethod__", False): abstracts.add(name) - cls.__abstractmethods__ = frozenset(abstracts) + cls.__abstractmethods__ = set(abstracts) # Set up inheritance registry cls._abc_registry = WeakSet() cls._abc_cache = WeakSet() diff --git a/tests/snippets/abc_test.py b/tests/snippets/abc_test.py deleted file mode 100644 index 494f967be..000000000 --- a/tests/snippets/abc_test.py +++ /dev/null @@ -1 +0,0 @@ -import abc diff --git a/tests/snippets/hash.py b/tests/snippets/hash.py new file mode 100644 index 000000000..b108db705 --- /dev/null +++ b/tests/snippets/hash.py @@ -0,0 +1,23 @@ + +from testutils import assertRaises + + +class A: + pass + + +assert type(hash(None)) is int +assert type(hash(object())) is int +assert type(hash(A())) is int +assert type(hash(1)) is int +assert type(hash(1.1)) is int +assert type(hash("")) is int + +with assertRaises(TypeError): + hash({}) + +with assertRaises(TypeError): + hash(set()) + +with assertRaises(TypeError): + hash([]) diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py new file mode 100644 index 000000000..6f0243fcd --- /dev/null +++ b/tests/snippets/test_abc.py @@ -0,0 +1,34 @@ +import abc + +from testutils import assertRaises + + +class CustomInterface(abc.ABC): + @abc.abstractmethod + def a(self): + pass + + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + +# with assertRaises(TypeError): +# CustomInterface() + + +class Concrete: + def a(self): + pass + + +CustomInterface.register(Concrete) + + +class SubConcrete(Concrete): + pass + + +assert issubclass(Concrete, CustomInterface) +assert issubclass(SubConcrete, CustomInterface) + diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 79c98c2dd..28b3f4210 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -295,6 +295,10 @@ impl PyDictRef { } } } + + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } } impl DictProtocol for PyDictRef { @@ -333,6 +337,7 @@ pub fn init(context: &PyContext) { "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 865707201..6f059b592 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -162,6 +162,10 @@ impl PyListRef { Ok(s) } + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } + fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { let new_elements = seq_mul(&self.elements.borrow(), counter); vm.ctx.new_list(new_elements) @@ -466,6 +470,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(PyListRef::len), "__new__" => context.new_rustfunc(list_new), "__repr__" => context.new_rustfunc(PyListRef::repr), + "__hash__" => context.new_rustfunc(PyListRef::hash), "__doc__" => context.new_str(list_doc.to_string()), "append" => context.new_rustfunc(PyListRef::append), "clear" => context.new_rustfunc(PyListRef::clear), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c43e31a96..df164f49b 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -55,8 +55,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py vm.ctx.not_implemented() } -fn object_hash(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("unhashable type".to_string())) +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 { + zelf.get_id() as u64 } fn object_setattr( diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index fb558c990..34d487fbb 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -588,6 +588,10 @@ fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.new_str(s)) } +fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) +} + pub fn init(context: &PyContext) { let set_type = &context.set_type; @@ -600,6 +604,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(set_len), "__new__" => context.new_rustfunc(set_new), "__repr__" => context.new_rustfunc(set_repr), + "__hash__" => context.new_rustfunc(set_hash), "__eq__" => context.new_rustfunc(set_eq), "__ge__" => context.new_rustfunc(set_ge), "__gt__" => context.new_rustfunc(set_gt), From c01dc5310a45fee5b43b98455be66d8d0d88b630 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 4 Apr 2019 20:10:47 +1300 Subject: [PATCH 045/223] Add callback parameter to weakref, at the moment this callback isn't yet implement though. --- tests/snippets/test_abc.py | 5 +++++ vm/src/obj/objweakref.rs | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py index 6f0243fcd..391932c36 100644 --- a/tests/snippets/test_abc.py +++ b/tests/snippets/test_abc.py @@ -13,6 +13,7 @@ class CustomInterface(abc.ABC): return NotImplemented +# TODO raise an error if there are in any abstract methods not fulfilled # with assertRaises(TypeError): # CustomInterface() @@ -31,4 +32,8 @@ class SubConcrete(Concrete): assert issubclass(Concrete, CustomInterface) assert issubclass(SubConcrete, CustomInterface) +assert not issubclass(tuple, CustomInterface) +assert isinstance(Concrete(), CustomInterface) +assert isinstance(SubConcrete(), CustomInterface) +assert not isinstance((), CustomInterface) diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 685e32b70..05d6147ce 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -3,6 +3,7 @@ use crate::pyobject::PyValue; use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; +use crate::function::OptionalArg; use std::rc::{Rc, Weak}; #[derive(Debug)] @@ -32,7 +33,12 @@ pub type PyWeakRef = PyRef; impl PyWeakRef { // TODO callbacks - fn create(cls: PyClassRef, referent: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { PyWeak::downgrade(&referent).into_ref_with_type(vm, cls) } From aba059b81ae9d579134cbd521e624a44e3af3b93 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 02:52:51 -0600 Subject: [PATCH 046/223] impl PySliceableSequenceMut & objseqence::del_item used to impl list.__delitem__ --- vm/src/obj/objlist.rs | 39 +++++++---- vm/src/obj/objsequence.rs | 139 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 15 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 1558db6aa..97a798f35 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,8 +1,11 @@ use std::cell::{Cell, RefCell}; use std::fmt; +use std::ops::DerefMut; + use num_traits::ToPrimitive; + use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -12,7 +15,7 @@ use super::objint; use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, + PySliceableSequence, del_item, }; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -309,21 +312,27 @@ impl PyListRef { fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.borrow_mut(); - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(_) = elements.get_pos(idx) { - elements.remove(idx as usize); - Ok(vm.get_none()) - } else { - Err(vm.new_index_error("list index out of range".to_string())) - } + // if objtype::isinstance(&key, &vm.ctx.int_type()) { + // let idx = objint::get_value(&key).to_i32().unwrap(); + // if let Some(_) = elements.get_pos(idx) { + // elements.remove(idx as usize); + // Ok(vm.get_none()) + // } else { + // Err(vm.new_index_error("list index out of range".to_string())) + // } - } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) - } + // } else { + // panic!( + // "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + // elements, key + // ) + // } + del_item( + vm, + self.as_object(), + elements.deref_mut(), + key, + ) } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a07b2454d..1d700e312 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -137,6 +137,108 @@ impl PySliceableSequence for Vec { } } +pub trait PySliceableSequenceMut : PySliceableSequence { + //fn set_slice(&self, range: Range, seq: T) -> Self; + //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; + //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; + //fn set_stepped_slice_reverse(&self, range: Range, step: usize, seq: T) -> Self; + + fn del_slice(&mut self, range: Range); + fn del_stepped_slice(&mut self, range: Range, step: usize); + fn del_index(&mut self, index: usize); + + fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PyObjectRef) -> PyResult { + // TODO: we could potentially avoid this copy and use slice + match slice.payload() { + Some(PySlice { start, stop, step }) => { + let step = step.clone().unwrap_or_else(BigInt::one); + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(start, stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + }, + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + }, + None => { + self.del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + }, + } + } else { + // TODO what to delete here? + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + }, + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + }, + None => { + self.del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + }, + } + } else { + // TODO what to del here? + Ok(vm.get_none()) + } + } + } + payload => panic!("del_slice_items called with non-slice: {:?}", payload), + } + } +} + +impl PySliceableSequenceMut for Vec { + fn del_slice(&mut self, range: Range) { + self.drain(range); + } + + fn del_stepped_slice(&mut self, range: Range, step: usize) { + // no easy way to delete steped indexes so heres what we'll do + let len = self.len(); + let mut deleted = 0; + let mut indexes = range.step_by(step).peekable(); + for i in 0..len { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + self.swap(i - deleted, i); + } + } + // then truncate the vec (the values to delete should now be the last elements and thus choped off) + self.truncate(len - deleted); + } + + fn del_index(&mut self, index: usize) { + self.remove(index); + } +} + + pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, @@ -183,6 +285,43 @@ pub fn get_item( } } +pub fn del_item ( + vm: &VirtualMachine, + sequence: &PyObjectRef, + elements: &mut T, + subscript: PyObjectRef, +) -> PyResult { + if let Some(i) = subscript.payload::() { + return match i.as_bigint().to_i32() { + Some(value) => { + if let Some(pos_index) = elements.get_pos(value) { + elements.del_index(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + None => { + Err(vm.new_index_error("cannot fit 'int' into an index-sized integer".to_string())) + } + }; + } + + if subscript.payload::().is_some() { + + if sequence.payload::().is_some() { + elements.del_slice_items(vm, &subscript) + } else { + panic!("sequence del_item called for non-mutable-sequence") + } + } else { + Err(vm.new_type_error(format!( + "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", + sequence, subscript + ))) + } +} + pub fn seq_equal( vm: &VirtualMachine, zelf: &[PyObjectRef], From c97f342ef772f998cabed97baf6ebc8d916d46fc Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 03:18:58 -0600 Subject: [PATCH 047/223] cleanup and rustfmt --- vm/src/obj/objlist.rs | 27 +++------------------------ vm/src/obj/objsequence.rs | 18 ++++++++---------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 97a798f35..6d7dc770c 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -5,7 +5,6 @@ use std::ops::DerefMut; use num_traits::ToPrimitive; - use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -14,8 +13,8 @@ use super::objbool; use super::objint; use super::objiter; use super::objsequence::{ - get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, del_item, + del_item, get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, + seq_mul, PySliceableSequence, }; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -312,27 +311,7 @@ impl PyListRef { fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.borrow_mut(); - // if objtype::isinstance(&key, &vm.ctx.int_type()) { - // let idx = objint::get_value(&key).to_i32().unwrap(); - // if let Some(_) = elements.get_pos(idx) { - // elements.remove(idx as usize); - // Ok(vm.get_none()) - // } else { - // Err(vm.new_index_error("list index out of range".to_string())) - // } - - // } else { - // panic!( - // "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - // elements, key - // ) - // } - del_item( - vm, - self.as_object(), - elements.deref_mut(), - key, - ) + del_item(vm, self.as_object(), elements.deref_mut(), key) } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 1d700e312..2168166e8 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -137,7 +137,7 @@ impl PySliceableSequence for Vec { } } -pub trait PySliceableSequenceMut : PySliceableSequence { +pub trait PySliceableSequenceMut: PySliceableSequence { //fn set_slice(&self, range: Range, seq: T) -> Self; //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; @@ -162,15 +162,15 @@ pub trait PySliceableSequenceMut : PySliceableSequence { Some(1) => { self.del_slice(range); Ok(vm.get_none()) - }, + } Some(num) => { self.del_stepped_slice(range, num as usize); Ok(vm.get_none()) - }, + } None => { self.del_slice(range.start..range.start + 1); Ok(vm.get_none()) - }, + } } } else { // TODO what to delete here? @@ -187,15 +187,15 @@ pub trait PySliceableSequenceMut : PySliceableSequence { Some(1) => { self.del_slice(range); Ok(vm.get_none()) - }, + } Some(num) => { self.del_stepped_slice(range, num as usize); Ok(vm.get_none()) - }, + } None => { self.del_slice(range.end - 1..range.end); Ok(vm.get_none()) - }, + } } } else { // TODO what to del here? @@ -238,7 +238,6 @@ impl PySliceableSequenceMut for Vec { } } - pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, @@ -285,7 +284,7 @@ pub fn get_item( } } -pub fn del_item ( +pub fn del_item( vm: &VirtualMachine, sequence: &PyObjectRef, elements: &mut T, @@ -308,7 +307,6 @@ pub fn del_item ( } if subscript.payload::().is_some() { - if sequence.payload::().is_some() { elements.del_slice_items(vm, &subscript) } else { From e3994010f3595bc39508d1beb8b9bf9c25aaaacd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 04:04:26 -0600 Subject: [PATCH 048/223] add list.__delitem__ test snippits --- tests/snippets/list.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6b9839321..6efa52cc0 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -170,3 +170,30 @@ def f(x): return x assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] + +# __delitem__ +x = ['a', 'b', 'c'] +del x[0] +assert x == ['b', 'c'] + +x = ['a', 'b', 'c'] +del x[-1] +assert x == ['a', 'b'] + +x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[2:14:3] +assert x == y +assert x == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] +assert y == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] + +x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[-5:] +assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] + +def bad_del_1(): + del ['a', 'b']['a'] +assert_raises(TypeError, bad_del_1) + +def bad_del_2(): + del ['a', 'b'][2] +assert_raises(IndexError, bad_del_2) From d0e6c3bd9266ce0c70e9b8d8d4ffdbc414a8c8e8 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Thu, 4 Apr 2019 04:52:33 -0600 Subject: [PATCH 049/223] convert to PySliceRef as it is the only option --- vm/src/obj/objsequence.rs | 113 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 2168166e8..9593e57fe 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -11,7 +11,7 @@ use crate::vm::VirtualMachine; use super::objbool; use super::objint::PyInt; use super::objlist::PyList; -use super::objslice::PySlice; +use super::objslice::{PySlice, PySliceRef}; use super::objtuple::PyTuple; pub trait PySliceableSequence { @@ -147,64 +147,64 @@ pub trait PySliceableSequenceMut: PySliceableSequence { fn del_stepped_slice(&mut self, range: Range, step: usize); fn del_index(&mut self, index: usize); - fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PyObjectRef) -> PyResult { - // TODO: we could potentially avoid this copy and use slice - match slice.payload() { - Some(PySlice { start, stop, step }) => { - let step = step.clone().unwrap_or_else(BigInt::one); - if step.is_zero() { - Err(vm.new_value_error("slice step cannot be zero".to_string())) - } else if step.is_positive() { - let range = self.get_slice_range(start, stop); - if range.start < range.end { - #[allow(clippy::range_plus_one)] - match step.to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.start..range.start + 1); - Ok(vm.get_none()) - } - } - } else { - // TODO what to delete here? + fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PySliceRef) -> PyResult { + + let start = &slice.start; + let stop = &slice.stop; + + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(start, stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self.del_slice(range); Ok(vm.get_none()) } - } else { - // calculate the range for the reverse slice, first the bounds needs to be made - // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); - let range = self.get_slice_range(&stop, &start); - if range.start < range.end { - match (-step).to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.end - 1..range.end); - Ok(vm.get_none()) - } - } - } else { - // TODO what to del here? + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self.del_slice(range.start..range.start + 1); Ok(vm.get_none()) } } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self.del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self.del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self.del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) } - payload => panic!("del_slice_items called with non-slice: {:?}", payload), } + } } @@ -305,10 +305,15 @@ pub fn del_item( } }; } - + if subscript.payload::().is_some() { if sequence.payload::().is_some() { - elements.del_slice_items(vm, &subscript) + if let Ok(slice) = subscript.downcast::() { + elements.del_slice_items(vm, &slice) + } else { + panic!("PySlice is not a slice? this should not be") + } + } else { panic!("sequence del_item called for non-mutable-sequence") } From 05ede30c519f117998e9282bfed219e049cfd673 Mon Sep 17 00:00:00 2001 From: Joey Date: Thu, 4 Apr 2019 07:52:53 -0700 Subject: [PATCH 050/223] range index --- vm/src/obj/objrange.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index ae946d05e..71adb4118 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,12 +6,14 @@ use num_integer::Integer; use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{Either, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; use super::objiter; -use super::objslice::PySliceRef; +use super::objslice::{PySlice, PySliceRef}; use super::objtype::PyClassRef; #[derive(Debug, Clone)] @@ -261,16 +263,16 @@ impl PyRangeRef { } } - fn getitem(self, subscript: Either, vm: &VirtualMachine) -> PyResult { + fn getitem(self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { - Either::A(index) => { + RangeIndex::Int(index) => { if let Some(value) = self.get(index.as_bigint()) { Ok(PyInt::new(value).into_ref(vm).into_object()) } else { Err(vm.new_index_error("range object index out of range".to_string())) } } - Either::B(slice) => { + RangeIndex::Slice(slice) => { let new_start = if let Some(int) = slice.start.as_ref() { if let Some(i) = self.get(int) { PyInt::new(i).into_ref(vm) @@ -349,3 +351,21 @@ impl PyRangeIteratorRef { self } } + +pub enum RangeIndex { + Int(PyIntRef), + Slice(PySliceRef), +} + +impl TryFromObject for RangeIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(RangeIndex::Int(i)), + s @ PySlice => Ok(RangeIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } +} From d96a5b11f7574d7214509c750d94e043a79424d3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Thu, 4 Apr 2019 21:32:04 +0200 Subject: [PATCH 051/223] add resources to snippets --- .gitignore | 1 + tests/snippets/builtin_slice.py | 66 +- tests/snippets/slice_res.py | 2252 ------------------------------- tests/test_snippets.py | 112 +- 4 files changed, 104 insertions(+), 2327 deletions(-) delete mode 100644 tests/snippets/slice_res.py diff --git a/.gitignore b/.gitignore index 967a51a83..e9dd6d220 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode wasm-pack.log .idea/ +tests/snippets/resources \ No newline at end of file diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 619356211..d61698034 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -78,53 +78,35 @@ def test_all_slices(): """ test all possible slices except big number """ - MODE = None # set to "build" to rebuild slice_res.py - ll = [0, 1, 2, 3] - start = list(range(-7, 7)) - end = list(range(-7, 7)) - step = list(range(-5, 5)) - step.pop(step.index(0)) - for i in [start, end, step]: - i.append(None) + import resources + from resources.cpython_generated_slices import SLICES_RES, START, END, STEP, LL - def build(): - # loop used to build slices_res.py with cpython - with open("slice_res.py", "wt") as f: - for s in start: - for e in end: - for t in step: - f.write(str(ll[s:e:t]) + "\n") + ll = LL + start = START + end = END + step = STEP - def run(): - # test utility - from slice_res import SLICES_RES - - count = 0 - failures = [] - for s in start: - for e in end: - for t in step: - lhs = ll[s:e:t] - try: - assert lhs == SLICES_RES[count] - except AssertionError: - failures.append( - "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( - s, e, t, lhs, SLICES_RES[count] - ) + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == SLICES_RES[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, SLICES_RES[count] ) - count += 1 + ) + count += 1 - if failures: - for f in failures: - print(f) - print(len(failures), "slices failed") - - if MODE == "build": - build() - else: - run() + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") test_all_slices() diff --git a/tests/snippets/slice_res.py b/tests/snippets/slice_res.py deleted file mode 100644 index 38eb87729..000000000 --- a/tests/snippets/slice_res.py +++ /dev/null @@ -1,2252 +0,0 @@ -SLICES_RES = [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [2, 3], - [2], - [2], - [2], - [2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [3], - [3], - [3], - [3], - [3], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [0], - [0], - [0], - [0], - [0], - [], - [], - [], - [], - [], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [0], - [0], - [0], - [0], - [0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1, 0], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [1], - [1], - [1], - [1], - [1], - [], - [], - [], - [], - [], - [1, 2], - [1], - [1], - [1], - [1, 2], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [], - [], - [], - [], - [], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [1], - [1], - [1], - [1], - [1, 0], - [1, 2, 3], - [1, 3], - [1], - [1], - [1, 2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2], - [2, 1], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [2], - [2], - [2], - [2], - [2], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [], - [], - [], - [], - [], - [2, 3], - [2], - [2], - [2], - [2, 3], - [2], - [2], - [2], - [2, 0], - [2, 1, 0], - [2, 3], - [2], - [2], - [2], - [2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [3], - [3], - [3], - [3], - [3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [0], - [0], - [0], - [0], - [0], - [3], - [3], - [3], - [3], - [3], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [3], - [3], - [3], - [3, 1], - [3, 2, 1], - [], - [], - [], - [], - [], - [3], - [3], - [3], - [3], - [3, 2], - [0], - [0], - [0], - [0], - [0], - [3], - [3], - [3], - [3], - [3], - [0, 1], - [0], - [0], - [0], - [0, 1], - [], - [], - [], - [], - [], - [0, 1, 2], - [0, 2], - [0], - [0], - [0, 1, 2], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [], - [], - [], - [], - [], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], - [3], - [3], - [3, 0], - [3, 1], - [3, 2, 1, 0], - [0, 1, 2, 3], - [0, 2], - [0, 3], - [0], - [0, 1, 2, 3], -] diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7edbd7b7e..7a888f093 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -1,4 +1,3 @@ - # This is a python unittest class automatically populating with all tests # in the tests folder. @@ -11,6 +10,8 @@ import logging import subprocess import contextlib import enum +from pathlib import Path +import shutil import compile_code @@ -19,14 +20,15 @@ class _TestType(enum.Enum): functional = 1 -logger = logging.getLogger('tests') -ROOT_DIR = '..' -TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) -TEST_DIRS = { - _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), -} -CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) +logger = logging.getLogger("tests") +ROOT_DIR = ".." +TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) +TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} +CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) +RESOURCES_DIR = os.path.abspath( + os.path.join(TEST_DIRS[_TestType.functional], "resources") +) @contextlib.contextmanager @@ -38,12 +40,12 @@ def pushd(path): def perform_test(filename, method, test_type): - logger.info('Running %s via %s', filename, method) - if method == 'cpython': + logger.info("Running %s via %s", filename, method) + if method == "cpython": run_via_cpython(filename) - elif method == 'cpython_bytecode': + elif method == "cpython_bytecode": run_via_cpython_bytecode(filename, test_type) - elif method == 'rustpython': + elif method == "rustpython": run_via_rustpython(filename, test_type) else: raise NotImplementedError(method) @@ -57,27 +59,27 @@ def run_via_cpython(filename): def run_via_cpython_bytecode(filename, test_type): # Step1: Create bytecode file: - bytecode_filename = filename + '.bytecode' - with open(bytecode_filename, 'w') as f: + bytecode_filename = filename + ".bytecode" + with open(bytecode_filename, "w") as f: compile_code.compile_to_bytecode(filename, out_file=f) # Step2: run cpython bytecode: env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" with pushd(CPYTHON_RUNNER_DIR): - subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) + subprocess.check_call(["cargo", "run", bytecode_filename], env=env) def run_via_rustpython(filename, test_type): env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" - target = 'release' - if env.get('CODE_COVERAGE', 'false') == 'true': - target = 'debug' - binary = os.path.abspath(os.path.join(ROOT_DIR, 'target', target, 'rustpython')) + target = "release" + if env.get("CODE_COVERAGE", "false") == "true": + target = "debug" + binary = os.path.abspath(os.path.join(ROOT_DIR, "target", target, "rustpython")) subprocess.check_call([binary, filename], env=env) @@ -85,15 +87,15 @@ def run_via_rustpython(filename, test_type): def create_test_function(cls, filename, method, test_type): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) - test_function_name = 'test_{}_'.format(method) \ - + os.path.splitext(snippet_filename)[0] \ - .replace('.', '_').replace('-', '_') + test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ + 0 + ].replace(".", "_").replace("-", "_") def test_function(self): perform_test(filename, method, test_type) if hasattr(cls, test_function_name): - raise ValueError('Duplicate test case {}'.format(test_function_name)) + raise ValueError("Duplicate test case {}".format(test_function_name)) setattr(cls, test_function_name, test_function) @@ -103,25 +105,69 @@ def populate(method): for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls + return wrapper def get_test_files(): """ Retrieve test files """ for test_type, test_dir in TEST_DIRS.items(): - for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))): + for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] - if filename.startswith('xfail_'): + if filename.startswith("xfail_"): continue yield test_type, os.path.abspath(filepath) -@populate('cpython') +def generate_slices(path): + # loop used to build slices_res.py with cpython + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + for i in [start, end, step]: + i.append(None) + + slices_res = [] + for s in start: + for e in end: + for t in step: + slices_res.append(ll[s:e:t]) + + path.write_text( + "SLICES_RES={}\nSTART= {}\nEND= {}\nSTEP= {}\nLL={}\n".format( + slices_res, start, end, step, ll + ) + ) + + +@populate("cpython") # @populate('cpython_bytecode') -@populate('rustpython') +@populate("rustpython") class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - subprocess.check_call(['cargo', 'build']) - subprocess.check_call(['cargo', 'build', '--release']) + # setup resource dir + cls.resources = Path(RESOURCES_DIR) + if cls.resources.exists(): + shutil.rmtree( + RESOURCES_DIR, ignore_errors=True + ) # we don't care if dir doesnt exist + cls.resources.mkdir() + (cls.resources / "__init__.py").touch() + + # Here add resource files + cls.slices_resource_path = Path(RESOURCES_DIR) / "cpython_generated_slices.py" + generate_slices(cls.slices_resource_path) + + # cargo stuff + subprocess.check_call(["cargo", "build"]) + subprocess.check_call(["cargo", "build", "--release"]) + + @classmethod + def tearDownClass(cls): + shutil.rmtree( + RESOURCES_DIR, ignore_errors=True + ) # we don't care if dir doesnt exist From 79caaa5a0b025ab18462667f1b6bf3fe14a9ebe9 Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 5 Apr 2019 19:19:55 +1300 Subject: [PATCH 052/223] Move PSF-LICENSE and add Lib README.md --- PSF-LICENSE => Lib/PSF-LICENSE | 0 Lib/README.md | 10 ++++++++++ Lib/re.py | 11 ----------- 3 files changed, 10 insertions(+), 11 deletions(-) rename PSF-LICENSE => Lib/PSF-LICENSE (100%) create mode 100644 Lib/README.md delete mode 100644 Lib/re.py diff --git a/PSF-LICENSE b/Lib/PSF-LICENSE similarity index 100% rename from PSF-LICENSE rename to Lib/PSF-LICENSE diff --git a/Lib/README.md b/Lib/README.md new file mode 100644 index 000000000..b3aade7bf --- /dev/null +++ b/Lib/README.md @@ -0,0 +1,10 @@ +Standard Library for RustPython +=============================== + +This directory contains all of the Python files that make up the standard library for RustPython. + +Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them +to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. +Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation. + +The first target is to run the ``unittest`` module, so we can leverage the CPython test suite. diff --git a/Lib/re.py b/Lib/re.py deleted file mode 100644 index 8511222be..000000000 --- a/Lib/re.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" Regular expressions """ - - -def match(pattern, string, flags=0): - return _compile(pattern, flags).match(string) - - -def _compile(pattern, flags): - p = sre_compile.compile(pattern, flags) - return p From bce4f1e48373f6d4bcdc21fce17c6002505ab6fa Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 3 Apr 2019 17:23:27 +0100 Subject: [PATCH 053/223] Simplify/shrink the dict interface. --- vm/src/frame.rs | 15 ++--- vm/src/import.rs | 6 +- vm/src/obj/objdict.rs | 120 +++++++++------------------------------- vm/src/obj/objobject.rs | 4 +- vm/src/obj/objstr.rs | 6 ++ vm/src/obj/objtype.rs | 8 +-- vm/src/pyobject.rs | 69 +++++++++-------------- vm/src/stdlib/json.rs | 11 ++-- 8 files changed, 78 insertions(+), 161 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index b3cf2149b..6a09f085a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -385,21 +385,21 @@ impl Frame { Ok(None) } bytecode::Instruction::BuildMap { size, unpack } => { - let map_obj = vm.ctx.new_dict().into_object(); + let map_obj = vm.ctx.new_dict(); for _x in 0..*size { let obj = self.pop_value(); if *unpack { // Take all key-value pairs from the dict: let dict_elements = objdict::get_key_value_pairs(&obj); for (key, value) in dict_elements.iter() { - objdict::set_item(&map_obj, vm, key, value); + map_obj.set_item(&vm.ctx, &objstr::get_value(key), value.clone()); } } else { let key = self.pop_value(); - objdict::set_item(&map_obj, vm, &key, &obj); + map_obj.set_item(&vm.ctx, &objstr::get_value(&key), obj) } } - self.push_value(map_obj); + self.push_value(map_obj.into_object()); Ok(None) } bytecode::Instruction::BuildSlice { size } => { @@ -613,10 +613,10 @@ impl Frame { bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { let kw_dict = self.pop_value(); - let dict_elements = objdict::get_elements(&kw_dict).clone(); + let dict_elements = objdict::get_key_value_pairs(&kw_dict).clone(); dict_elements .into_iter() - .map(|elem| (elem.0, (elem.1).1)) + .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { vec![] @@ -1210,7 +1210,8 @@ impl fmt::Debug for Frame { .map(|elem| format!("\n > {:?}", elem)) .collect::(); let dict = self.scope.get_locals(); - let local_str = objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) + let local_str = dict + .get_key_value_pairs() .iter() .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) .collect::(); diff --git a/vm/src/import.rs b/vm/src/import.rs index 936b89680..179dd4321 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use crate::compile; use crate::frame::Scope; use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, PyResult}; +use crate::pyobject::{DictProtocol, ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; @@ -47,11 +47,11 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { // First, see if we already loaded the module: let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; - if let Some(module) = sys_modules.get_item(module_name) { + if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { return Ok(module); } let module = import_uncached_module(vm, current_path, module_name)?; - sys_modules.set_item(&vm.ctx, module_name, module.clone()); + sys_modules.set_item(module_name, module.clone(), vm)?; Ok(module) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 79c98c2dd..15046a405 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,7 +1,6 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::fmt; -use std::ops::{Deref, DerefMut}; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ @@ -37,94 +36,16 @@ impl PyValue for PyDict { } } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap().entries.borrow() -} - -pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut + 'a { - obj.payload::().unwrap().entries.borrow_mut() -} - -pub fn set_item( - dict: &PyObjectRef, - _vm: &VirtualMachine, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // TODO: use vm to call eventual __hash__ and __eq__methods. - let mut elements = get_mut_elements(dict); - set_item_in_content(&mut elements, needle, value); -} - -pub fn set_item_in_content( - elements: &mut DictContentType, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // XXX: Currently, we only support String keys, so we have to unwrap the - // PyObject (and ensure it is a String). - - // TODO: invoke __hash__ function here! - let needle_str = objstr::get_value(needle); - elements.insert(needle_str, (needle.clone(), value.clone())); -} - pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = get_elements(dict); - get_key_value_pairs_from_content(&dict_elements) -} - -pub fn get_key_value_pairs_from_content( - dict_content: &DictContentType, -) -> Vec<(PyObjectRef, PyObjectRef)> { + let dict_elements = dict.payload::().unwrap().entries.borrow(); let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_content.iter() { + for (_str_key, pair) in dict_elements.iter() { let (key, obj) = pair; pairs.push((key.clone(), obj.clone())); } pairs } -pub fn get_item(dict: &PyObjectRef, key: &PyObjectRef) -> Option { - let needle_str = objstr::get_value(key); - get_key_str(dict, &needle_str) -} - -// Special case for the case when requesting a str key from a dict: -pub fn get_key_str(dict: &PyObjectRef, key: &str) -> Option { - let elements = get_elements(dict); - content_get_key_str(&elements, key) -} - -/// Retrieve a key from dict contents: -pub fn content_get_key_str(elements: &DictContentType, key: &str) -> Option { - // TODO: let hash: usize = key; - match elements.get(key) { - Some(v) => Some(v.1.clone()), - None => None, - } -} - -pub fn contains_key_str(dict: &PyObjectRef, key: &str) -> bool { - let elements = get_elements(dict); - content_contains_key_str(&elements, key) -} - -pub fn content_contains_key_str(elements: &DictContentType, key: &str) -> bool { - // TODO: let hash: usize = key; - elements.get(key).is_some() -} - -/// Take a python dictionary and convert it to attributes. -pub fn py_dict_to_attributes(dict: &PyObjectRef) -> PyAttributes { - let mut attrs = PyAttributes::new(); - for (key, value) in get_key_value_pairs(dict) { - let key = objstr::get_value(&key); - attrs.insert(key, value); - } - attrs -} - // Python dict methods: impl PyDictRef { fn new( @@ -137,7 +58,7 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { for (needle, value) in get_key_value_pairs(&dict_obj) { - set_item(dict.as_object(), vm, &needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -156,13 +77,13 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - set_item(dict.as_object(), vm, &needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); } } } for (needle, value) in kwargs.into_iter() { let py_needle = vm.new_str(needle); - set_item(&dict.as_object(), vm, &py_needle, &value); + dict.set_item(&vm.ctx, &objstr::get_value(&py_needle), value); } Ok(dict) } @@ -259,9 +180,8 @@ impl PyDictRef { } } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, _vm: &VirtualMachine) { - let mut elements = self.entries.borrow_mut(); - set_item_in_content(&mut elements, &needle, &value) + fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { + self.set_item(&vm.ctx, &objstr::get_value(&needle), value) } fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { @@ -295,15 +215,28 @@ impl PyDictRef { } } } + + /// Take a python dictionary and convert it to attributes. + pub fn to_attributes(self) -> PyAttributes { + let mut attrs = PyAttributes::new(); + for (key, value) in self.get_key_value_pairs() { + let key = objstr::get_value(&key); + attrs.insert(key, value); + } + attrs + } } impl DictProtocol for PyDictRef { fn contains_key(&self, k: &str) -> bool { - content_contains_key_str(&self.entries.borrow(), k) + self.entries.borrow().get(k).is_some() } fn get_item(&self, k: &str) -> Option { - content_get_key_str(&self.entries.borrow(), k) + match self.entries.borrow().get(k) { + Some(v) => Some(v.1.clone()), + None => None, + } } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -311,13 +244,14 @@ impl DictProtocol for PyDictRef { } // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - let key = ctx.new_str(key.to_string()); - set_item_in_content(&mut self.entries.borrow_mut(), &key, &v); + fn set_item(&self, ctx: &PyContext, key_str: &str, v: PyObjectRef) { + let key = ctx.new_str(key_str.to_string()); + let elements = &mut self.entries.borrow_mut(); + elements.insert(key_str.to_string(), (key.clone(), v.clone())); } fn del_item(&self, key: &str) { - let mut elements = get_mut_elements(self.as_object()); + let elements = &mut self.entries.borrow_mut(); elements.remove(key).unwrap(); } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c43e31a96..11a8309ab 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,4 +1,4 @@ -use super::objdict::{self, PyDictRef}; +use super::objdict::PyDictRef; use super::objlist::PyList; use super::objstr::PyStringRef; use super::objtype; @@ -233,7 +233,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in objdict::get_key_value_pairs(dict.as_object()) { + for (key, value) in dict.get_key_value_pairs() { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a8264b2de..592a4950f 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -756,6 +756,12 @@ impl IntoPyObject for String { } } +impl IntoPyObject for &str { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.to_string())) + } +} + pub fn init(ctx: &PyContext) { PyStringRef::extend_class(ctx, &ctx.str_type); } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 14958300b..16cfbaa5e 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -9,7 +9,6 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objdict; use super::objdict::PyDictRef; use super::objlist::PyList; use super::objproperty::PropertyBuilder; @@ -244,12 +243,7 @@ pub fn type_new_class( ) -> PyResult { let mut bases: Vec = bases.iter(vm)?.collect::, _>>()?; bases.push(vm.ctx.object()); - new( - typ.clone(), - &name.value, - bases, - objdict::py_dict_to_attributes(dict.as_object()), - ) + new(typ.clone(), &name.value, bases, dict.to_attributes()) } pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d6982aff6..12d6fa1ca 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -684,16 +684,6 @@ impl PyContext { .into_ref() } - // Item set/get: - pub fn set_item(&self, obj: &PyObjectRef, key: &str, v: PyObjectRef) { - if let Some(dict) = obj.payload::() { - let key = self.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - unimplemented!() - }; - } - pub fn set_attr<'a, T: Into<&'a PyObjectRef>, V: Into>( &'a self, obj: T, @@ -948,44 +938,35 @@ pub trait DictProtocol { fn del_item(&self, key: &str); } -impl DictProtocol for PyObjectRef { - fn contains_key(&self, k: &str) -> bool { - if let Some(dict) = self.payload::() { - objdict::content_contains_key_str(&dict.entries.borrow(), k) - } else { - unimplemented!() - } +pub trait ItemProtocol { + // // Move: doesn't really belong in this protocol. + // fn get_key_value_pairs(&self, vm: &VirtualMachine) -> PyResult; + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; +} + +impl ItemProtocol for PyObjectRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } - fn get_item(&self, k: &str) -> Option { - if let Some(dict) = self.payload::() { - objdict::content_get_key_str(&dict.entries.borrow(), k) - } else { - panic!("TODO {:?}", k) - } + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - if self.payload_is::() { - objdict::get_key_value_pairs(self) - } else { - panic!("TODO") - } - } - - // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - if let Some(dict) = self.payload::() { - let key = ctx.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - panic!("TODO {:?}", self); - } - } - - fn del_item(&self, key: &str) { - let mut elements = objdict::get_mut_elements(self); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index fbe7ea6b1..80f0f45b6 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -12,7 +12,8 @@ use crate::obj::{ objtype, }; use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol, + create_type, DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, + TypeProtocol, }; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -63,10 +64,10 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { let elements = objsequence::get_elements(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_elements(self.pyobject); + let pairs = objdict::get_key_value_pairs(self.pyobject); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { - map.serialize_entry(&key, &self.clone_with_object(&e.1))?; + map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; } map.end() } else if self.pyobject.is(&self.vm.get_none()) { @@ -201,9 +202,9 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { let json_decode_error = vm .get_attribute(vm.sys_module.clone(), "modules") .unwrap() - .get_item("json") + .get_item("json", vm) .unwrap() - .get_item("JSONDecodeError") + .get_item("JSONDecodeError", vm) .unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); From f840bdcd7fa8b47434e0470f617d8b488c9dccd3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:47:44 +0100 Subject: [PATCH 054/223] Generalise set_attr. --- vm/src/vm.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 2fad29124..1649d4efc 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -568,13 +568,16 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr( - &self, - obj: &PyObjectRef, - attr_name: PyObjectRef, - attr_value: PyObjectRef, - ) -> PyResult { - self.call_method(&obj, "__setattr__", vec![attr_name, attr_value]) + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: PyObjectRef) -> PyResult + where + K: TryIntoRef, + { + let attr_name = attr_name.try_into_ref(self)?; + self.call_method( + obj, + "__setattr__", + vec![attr_name.into_object(), attr_value], + ) } pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult<()> { From 25c17b482985f3641134f7a06b109845ae19ba49 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:48:23 +0100 Subject: [PATCH 055/223] Change all the ctx.set_attr in ast. --- vm/src/stdlib/ast.rs | 158 +++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 75 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 65fcdf660..f6aa93aad 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -50,7 +50,7 @@ fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { // let ast_node = ctx.new_instance(this.Module); let ast_node = create_node(vm, "program"); let py_body = vm.ctx.new_list(body); - vm.ctx.set_attr(&ast_node, "body", py_body); + vm.set_attr(&ast_node, "body", py_body).unwrap(); ast_node } @@ -80,15 +80,16 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "ClassDef"); // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); + vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) + .unwrap(); // Set body: let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); + vm.set_attr(&node, "decorator_list", py_decorator_list) + .unwrap(); node } ast::Statement::FunctionDef { @@ -101,24 +102,26 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "FunctionDef"); // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); + vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) + .unwrap(); - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); + vm.set_attr(&node, "args", parameters_to_ast(vm, args)) + .unwrap(); // Set body: let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); + vm.set_attr(&node, "decorator_list", py_decorator_list) + .unwrap(); let py_returns = if let Some(hint) = returns { expression_to_ast(vm, hint) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "returns", py_returns); + vm.set_attr(&node, "returns", py_returns).unwrap(); node } ast::Statement::Continue => create_node(vm, "Continue"), @@ -127,13 +130,14 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P ast::Statement::Assert { test, msg } => { let node = create_node(vm, "Pass"); - vm.ctx.set_attr(&node, "test", expression_to_ast(vm, test)); + vm.set_attr(&node, "test", expression_to_ast(vm, test)) + .unwrap(); let py_msg = match msg { Some(msg) => expression_to_ast(vm, msg), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "msg", py_msg); + vm.set_attr(&node, "msg", py_msg).unwrap(); node } @@ -143,7 +147,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let py_targets = vm .ctx .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); - vm.ctx.set_attr(&node, "targets", py_targets); + vm.set_attr(&node, "targets", py_targets).unwrap(); node } @@ -155,7 +159,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -163,17 +167,17 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "If"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -186,20 +190,20 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "For"); let py_target = expression_to_ast(vm, target); - vm.ctx.set_attr(&node, "target", py_target); + vm.set_attr(&node, "target", py_target).unwrap(); let py_iter = expression_to_ast(vm, iter); - vm.ctx.set_attr(&node, "iter", py_iter); + vm.set_attr(&node, "iter", py_iter).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -207,17 +211,17 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "While"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = if let Some(orelse) = orelse { statements_to_ast(vm, orelse) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -225,7 +229,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P let node = create_node(vm, "Expr"); let value = expression_to_ast(vm, expression); - vm.ctx.set_attr(&node, "value", value); + vm.set_attr(&node, "value", value).unwrap(); node } @@ -236,7 +240,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); - vm.ctx.set_attr(&node, "lineno", lineno); + vm.set_attr(&node, "lineno", lineno).unwrap(); node } @@ -255,10 +259,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Call"); let py_func_ast = expression_to_ast(vm, function); - vm.ctx.set_attr(&node, "func", py_func_ast); + vm.set_attr(&node, "func", py_func_ast).unwrap(); let py_args = expressions_to_ast(vm, args); - vm.ctx.set_attr(&node, "args", py_args); + vm.set_attr(&node, "args", py_args).unwrap(); node } @@ -266,7 +270,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "BinOp"); let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "left", py_a); + vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let str_op = match op { @@ -285,10 +289,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Operator::FloorDiv => "FloorDiv", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); let py_b = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "right", py_b); + vm.set_attr(&node, "right", py_b).unwrap(); node } ast::Expression::Unop { op, a } => { @@ -301,10 +305,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::UnaryOperator::Pos => "UAdd", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "operand", py_a); + vm.set_attr(&node, "operand", py_a).unwrap(); node } @@ -315,14 +319,14 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let py_a = expression_to_ast(vm, a); let py_b = expression_to_ast(vm, b); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); + vm.set_attr(&node, "op", py_op).unwrap(); node } @@ -330,7 +334,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Compare"); let py_a = expression_to_ast(vm, &vals[0]); - vm.ctx.set_attr(&node, "left", py_a); + vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -351,7 +355,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .collect(), ); - vm.ctx.set_attr(&node, "ops", py_ops); + vm.set_attr(&node, "ops", py_ops).unwrap(); let py_b = vm.ctx.new_list( vals.iter() @@ -359,7 +363,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .map(|x| expression_to_ast(vm, x)) .collect(), ); - vm.ctx.set_attr(&node, "comparators", py_b); + vm.set_attr(&node, "comparators", py_b).unwrap(); node } ast::Expression::Identifier { name } => { @@ -367,16 +371,17 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj // Id: let py_name = vm.ctx.new_str(name.clone()); - vm.ctx.set_attr(&node, "id", py_name); + vm.set_attr(&node, "id", py_name).unwrap(); node } ast::Expression::Lambda { args, body } => { let node = create_node(vm, "Lambda"); - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); + vm.set_attr(&node, "args", parameters_to_ast(vm, args)) + .unwrap(); let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); node } @@ -384,13 +389,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "IfExp"); let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); + vm.set_attr(&node, "test", py_test).unwrap(); let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); + vm.set_attr(&node, "body", py_body).unwrap(); let py_orelse = expression_to_ast(vm, orelse); - vm.ctx.set_attr(&node, "orelse", py_orelse); + vm.set_attr(&node, "orelse", py_orelse).unwrap(); node } @@ -404,28 +409,28 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - vm.ctx.set_attr(&node, "n", py_n); + vm.set_attr(&node, "n", py_n).unwrap(); node } ast::Expression::True => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(true)); + vm.set_attr(&node, "value", vm.ctx.new_bool(true)).unwrap(); node } ast::Expression::False => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(false)); + vm.set_attr(&node, "value", vm.ctx.new_bool(false)).unwrap(); node } ast::Expression::None => { let node = create_node(vm, "NameConstant"); - vm.ctx.set_attr(&node, "value", vm.ctx.none()); + vm.set_attr(&node, "value", vm.ctx.none()).unwrap(); node } @@ -435,7 +440,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -444,7 +449,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -453,7 +458,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); + vm.set_attr(&node, "elts", py_elts).unwrap(); node } @@ -468,10 +473,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } let py_keys = vm.ctx.new_list(keys); - vm.ctx.set_attr(&node, "keys", py_keys); + vm.set_attr(&node, "keys", py_keys).unwrap(); let py_values = vm.ctx.new_list(values); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); node } @@ -490,7 +495,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .map(|g| comprehension_to_ast(vm, g)) .collect(); let py_generators = vm.ctx.new_list(g); - vm.ctx.set_attr(&node, "generators", py_generators); + vm.set_attr(&node, "generators", py_generators).unwrap(); node } @@ -501,7 +506,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj Some(value) => expression_to_ast(vm, value), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -509,7 +514,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "YieldFrom"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -517,10 +522,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Subscript"); let py_value = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); let py_slice = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "slice", py_slice); + vm.set_attr(&node, "slice", py_slice).unwrap(); node } @@ -528,10 +533,10 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Attribute"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); let py_attr = vm.ctx.new_str(name.to_string()); - vm.ctx.set_attr(&node, "attr", py_attr); + vm.set_attr(&node, "attr", py_attr).unwrap(); node } @@ -539,7 +544,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Starred"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } @@ -547,21 +552,22 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let node = create_node(vm, "Slice"); let py_value = expressions_to_ast(vm, elements); - vm.ctx.set_attr(&node, "bounds", py_value); + vm.set_attr(&node, "bounds", py_value).unwrap(); node } ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { let node = create_node(vm, "Bytes"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())); + vm.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())) + .unwrap(); node } }; // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); - vm.ctx.set_attr(&node, "lineno", lineno); + vm.set_attr(&node, "lineno", lineno).unwrap(); node } @@ -569,12 +575,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { let node = create_node(vm, "arguments"); - vm.ctx.set_attr( + vm.set_attr( &node, "args", vm.ctx .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), - ); + ) + .unwrap(); node } @@ -583,14 +590,14 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObject let node = create_node(vm, "arg"); let py_arg = vm.ctx.new_str(parameter.arg.to_string()); - vm.ctx.set_attr(&node, "arg", py_arg); + vm.set_attr(&node, "arg", py_arg).unwrap(); let py_annotation = if let Some(annotation) = ¶meter.annotation { expression_to_ast(vm, annotation) } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "annotation", py_annotation); + vm.set_attr(&node, "annotation", py_annotation).unwrap(); node } @@ -599,13 +606,13 @@ fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) let node = create_node(vm, "comprehension"); let py_target = expression_to_ast(vm, &comprehension.target); - vm.ctx.set_attr(&node, "target", py_target); + vm.set_attr(&node, "target", py_target).unwrap(); let py_iter = expression_to_ast(vm, &comprehension.iter); - vm.ctx.set_attr(&node, "iter", py_iter); + vm.set_attr(&node, "iter", py_iter).unwrap(); let py_ifs = expressions_to_ast(vm, &comprehension.ifs); - vm.ctx.set_attr(&node, "ifs", py_ifs); + vm.set_attr(&node, "ifs", py_ifs).unwrap(); node } @@ -614,13 +621,14 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef match string { ast::StringGroup::Constant { value } => { let node = create_node(vm, "Str"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_str(value.clone())); + vm.set_attr(&node, "s", vm.ctx.new_str(value.clone())) + .unwrap(); node } ast::StringGroup::FormattedValue { value, .. } => { let node = create_node(vm, "FormattedValue"); let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); + vm.set_attr(&node, "value", py_value).unwrap(); node } ast::StringGroup::Joined { values } => { @@ -631,7 +639,7 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef .map(|value| string_to_ast(vm, value)) .collect(), ); - vm.ctx.set_attr(&node, "values", py_values); + vm.set_attr(&node, "values", py_values).unwrap(); node } } From c8eda3733d211a2b77b5a40f70bc1ce8fbc61821 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 14:49:10 +0100 Subject: [PATCH 056/223] Eliminate ctx.set_attr. --- derive/src/pyclass.rs | 2 +- vm/src/builtins.rs | 17 ++++++++------- vm/src/exceptions.rs | 4 ++-- vm/src/frame.rs | 16 +++++++------- vm/src/import.rs | 2 +- vm/src/macros.rs | 22 +++++++++---------- vm/src/obj/objbytes.rs | 4 +--- vm/src/obj/objcode.rs | 3 +-- vm/src/obj/objdict.rs | 45 ++++++++++++++++++++++++++++----------- vm/src/obj/objellipsis.rs | 3 +-- vm/src/obj/objmemory.rs | 2 +- vm/src/obj/objobject.rs | 8 +++---- vm/src/obj/objstr.rs | 6 ++++++ vm/src/obj/objsuper.rs | 2 +- vm/src/obj/objtype.rs | 7 ++++++ vm/src/pyobject.rs | 28 ++++-------------------- vm/src/stdlib/io.rs | 16 +++++++------- vm/src/stdlib/json.rs | 8 ++++--- vm/src/stdlib/os.rs | 20 ++++++++--------- vm/src/sysmodule.rs | 11 +++++----- vm/src/vm.rs | 24 +++++++++------------ wasm/lib/src/convert.rs | 2 +- wasm/lib/src/vm_class.rs | 3 +-- 23 files changed, 129 insertions(+), 126 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index ad4ebf66e..e533e8009 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -118,7 +118,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { }| { let constructor_fn = kind.to_ctx_constructor_fn(); quote! { - ctx.set_attr(class, #py_name, ctx.#constructor_fn(Self::#fn_name)); + class.set_str_attr(#py_name, ctx.#constructor_fn(Self::#fn_name)); } }, ); diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 205d0c66e..7264c9c6e 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -669,7 +669,12 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_vars pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "__builtins__", { + #[cfg(target_arch = "wasm32")] + let open = ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = ctx.new_rustfunc(io_open); + + py_module!(ctx, "__builtins__", { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -715,6 +720,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "min" => ctx.new_rustfunc(builtin_min), "object" => ctx.object(), "oct" => ctx.new_rustfunc(builtin_oct), + "open" => open, "ord" => ctx.new_rustfunc(builtin_ord), "next" => ctx.new_rustfunc(builtin_next), "pow" => ctx.new_rustfunc(builtin_pow), @@ -759,12 +765,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), - }); - - #[cfg(not(target_arch = "wasm32"))] - ctx.set_attr(&py_mod, "open", ctx.new_rustfunc(io_open)); - - py_mod + }) } pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { @@ -801,6 +802,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu "__call__", vec![name_arg, bases, namespace.into_object()], )?; - cells.set_item(&vm.ctx, "__class__", class.clone()); + cells.set_item("__class__", class.clone(), vm); Ok(class) } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index fb757cd0f..cec4b3125 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -13,8 +13,8 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { vm.new_str("No msg".to_string()) }; let traceback = vm.ctx.new_list(Vec::new()); - vm.ctx.set_attr(&zelf, "msg", msg); - vm.ctx.set_attr(&zelf, "__traceback__", traceback); + vm.set_attr(&zelf, "msg", msg)?; + vm.set_attr(&zelf, "__traceback__", traceback)?; Ok(vm.get_none()) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 6a09f085a..1a0cc4de7 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -132,21 +132,21 @@ pub trait NameProtocol { impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item(name, vm) { return Some(value); } } - if let Some(value) = self.globals.get_item(name) { + if let Some(value) = self.globals.get_item(name, vm) { return Some(value); } vm.get_attribute(vm.builtins.clone(), name).ok() } - fn load_cell(&self, _vm: &VirtualMachine, name: &str) -> Option { + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item(name, vm) { return Some(value); } } @@ -154,7 +154,7 @@ impl NameProtocol for Scope { } fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(&vm.ctx, key, value) + self.get_locals().set_item(key, value, vm) } fn delete_name(&self, _vm: &VirtualMachine, key: &str) { @@ -392,11 +392,11 @@ impl Frame { // Take all key-value pairs from the dict: let dict_elements = objdict::get_key_value_pairs(&obj); for (key, value) in dict_elements.iter() { - map_obj.set_item(&vm.ctx, &objstr::get_value(key), value.clone()); + map_obj.set_item(key.clone(), value.clone(), vm); } } else { let key = self.pop_value(); - map_obj.set_item(&vm.ctx, &objstr::get_value(&key), obj) + map_obj.set_item(key, obj, vm) } } self.push_value(map_obj.into_object()); @@ -585,7 +585,7 @@ impl Frame { let scope = self.scope.clone(); let obj = vm.ctx.new_function(code_obj, scope, defaults); - vm.ctx.set_attr(&obj, "__annotations__", annotations); + vm.set_attr(&obj, "__annotations__", annotations)?; self.push_value(obj); Ok(None) diff --git a/vm/src/import.rs b/vm/src/import.rs index 179dd4321..28b3d02ef 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -39,7 +39,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); + attrs.set_item("__name__", vm.new_str(module.to_string()), vm); vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 47d3c531a..0c37e0dfc 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -114,15 +114,15 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => { - { - let py_mod = $ctx.new_module($module_name, $ctx.new_dict()); - $( - $ctx.set_attr(&py_mod, $name, $value); - )* - py_mod - } - } + ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let mut attributes = $crate::pyobject::PyAttributes::new(); + $( + let value: PyObjectRef = $value.into(); + attributes.insert($name.to_string(), value); + )* + let module_dict = $crate::obj::objdict::PyDictRef::from_attributes($ctx, attributes); + $ctx.new_module($module_name, module_dict) + }}; } #[macro_export] @@ -131,7 +131,7 @@ macro_rules! py_class { { let py_class = $ctx.new_class($class_name, $class_base); $( - $ctx.set_attr(&py_class, $name, $value); + py_class.set_str_attr($name, $value); )* py_class } @@ -143,7 +143,7 @@ macro_rules! extend_class { ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { let class = $class; $( - $ctx.set_attr(class, $name, $value); + class.set_str_attr($name, $value); )* } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 4119d688f..f24c9273c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -43,8 +43,6 @@ impl PyValue for PyBytes { // Fill bytes class methods: pub fn init(context: &PyContext) { - let bytes_type = context.bytes_type.as_object(); - let bytes_doc = "bytes(iterable_of_ints) -> bytes\n\ bytes(string, encoding[, errors]) -> bytes\n\ @@ -56,7 +54,7 @@ pub fn init(context: &PyContext) { - any object implementing the buffer API.\n \ - an integer"; - extend_class!(context, bytes_type, { + extend_class!(context, &context.bytes_type, { "__new__" => context.new_rustfunc(bytes_new), "__eq__" => context.new_rustfunc(PyBytesRef::eq), "__lt__" => context.new_rustfunc(PyBytesRef::lt), diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index a26645b91..60549cd3a 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -80,8 +80,7 @@ impl PyCodeRef { } pub fn init(context: &PyContext) { - let code_type = context.code_type.as_object(); - extend_class!(context, code_type, { + extend_class!(context, &context.code_type, { "__new__" => context.new_rustfunc(PyCodeRef::new), "__repr__" => context.new_rustfunc(PyCodeRef::repr), diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 15046a405..e110e71f8 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -4,7 +4,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, IntoPyObject, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -58,7 +58,7 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { for (needle, value) in get_key_value_pairs(&dict_obj) { - dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); + dict.set_item(needle, value, vm); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -77,13 +77,12 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(&vm.ctx, &objstr::get_value(&needle), value); + dict.set_item(needle, value, vm); } } } for (needle, value) in kwargs.into_iter() { - let py_needle = vm.new_str(needle); - dict.set_item(&vm.ctx, &objstr::get_value(&py_needle), value); + dict.set_item(vm.new_str(needle), value, vm); } Ok(dict) } @@ -181,7 +180,7 @@ impl PyDictRef { } fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(&vm.ctx, &objstr::get_value(&needle), value) + self.set_item(needle, value, vm) } fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { @@ -216,6 +215,23 @@ impl PyDictRef { } } + // Used during module initialisation when vm isn't available. + pub fn from_attributes(ctx: &PyContext, attributes: PyAttributes) -> PyDictRef { + let dict = ctx.new_dict(); + for (key_str, value) in attributes.into_iter() { + dict.unsafe_str_insert(&key_str, value, ctx); + } + dict + } + + // Pub needed for some nasty edge cases. + // It will be unsafe if there are entries in the dictionary that compare equal. + pub fn unsafe_str_insert(&self, key: &str, value: PyObjectRef, ctx: &PyContext) { + self.entries + .borrow_mut() + .insert(key.to_string(), (ctx.new_str(key.to_string()), value)); + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -228,12 +244,14 @@ impl PyDictRef { } impl DictProtocol for PyDictRef { - fn contains_key(&self, k: &str) -> bool { - self.entries.borrow().get(k).is_some() + fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); + self.entries.borrow().get(key_str).is_some() } - fn get_item(&self, k: &str) -> Option { - match self.entries.borrow().get(k) { + fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { + let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); + match self.entries.borrow().get(key_str) { Some(v) => Some(v.1.clone()), None => None, } @@ -244,10 +262,11 @@ impl DictProtocol for PyDictRef { } // Item set/get: - fn set_item(&self, ctx: &PyContext, key_str: &str, v: PyObjectRef) { - let key = ctx.new_str(key_str.to_string()); + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { + let key = key.into_pyobject(vm).unwrap(); + let key_str = &objstr::get_value(&key); let elements = &mut self.entries.borrow_mut(); - elements.insert(key_str.to_string(), (key.clone(), v.clone())); + elements.insert(key_str.to_string(), (key.clone(), value)); } fn del_item(&self, key: &str) { diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 8ce843352..6591d8173 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -3,8 +3,7 @@ use crate::pyobject::{PyContext, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let ellipsis_type = context.ellipsis_type.as_object(); - extend_class!(context, ellipsis_type, { + extend_class!(context, &context.ellipsis_type, { "__new__" => context.new_rustfunc(ellipsis_new), "__repr__" => context.new_rustfunc(ellipsis_repr) }); diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 7f53fb5ae..cfe44dea3 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -20,7 +20,7 @@ pub fn new_memory_view( bytes_object: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - vm.ctx.set_attr(&cls, "obj", bytes_object.clone()); + vm.set_attr(cls.as_object(), "obj", bytes_object.clone())?; PyMemoryView { obj: bytes_object }.into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 11a8309ab..86a56de5d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -77,7 +77,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.set_item(&vm.ctx, &attr_name.value, value); + dict.set_item(attr_name, value, vm); Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -208,7 +208,7 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } - if let Some(obj_attr) = object_getattr(&obj, &name) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm) { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { vm.call_get_descriptor(attr, obj) @@ -219,9 +219,9 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } -fn object_getattr(obj: &PyObjectRef, attr_name: &str) -> Option { +fn object_getattr(obj: &PyObjectRef, attr_name: &str, vm: &VirtualMachine) -> Option { if let Some(ref dict) = obj.dict { - dict.get_item(attr_name) + dict.get_item(attr_name, vm) } else { None } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 592a4950f..62cc923b1 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -762,6 +762,12 @@ impl IntoPyObject for &str { } } +impl IntoPyObject for &String { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.clone())) + } +} + pub fn init(ctx: &PyContext) { PyStringRef::extend_class(ctx, &ctx.str_type); } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 24424d265..19c7c2ea8 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -124,7 +124,7 @@ fn super_new( } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg) { + match vm.get_locals().get_item(first_arg, vm) { Some(obj) => obj.clone(), _ => { return Err(vm diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 16cfbaa5e..b1b50dfbd 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -162,6 +162,13 @@ impl PyClassRef { Ok(()) } + // This is used for class initialisation where the vm is not yet available. + pub fn set_str_attr>(&self, attr_name: &str, value: V) { + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value.into()); + } + fn subclasses(self, _vm: &VirtualMachine) -> PyList { let mut subclasses = self.subclasses.borrow_mut(); subclasses.retain(|x| x.upgrade().is_some()); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 12d6fa1ca..0c7d4955b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -684,24 +684,6 @@ impl PyContext { .into_ref() } - pub fn set_attr<'a, T: Into<&'a PyObjectRef>, V: Into>( - &'a self, - obj: T, - attr_name: &str, - value: V, - ) { - let obj = obj.into(); - if let Some(PyClass { ref attributes, .. }) = obj.payload::() { - attributes - .borrow_mut() - .insert(attr_name.to_string(), value.into()); - } else if let Some(ref dict) = obj.dict { - dict.set_item(self, attr_name, value.into()); - } else { - unimplemented!("set_attr unimplemented for: {:?}", obj); - }; - } - pub fn unwrap_constant(&self, value: &bytecode::Constant) -> PyObjectRef { match *value { bytecode::Constant::Integer { ref value } => self.new_int(value.clone()), @@ -931,16 +913,14 @@ impl TypeProtocol for PyRef { } pub trait DictProtocol { - fn contains_key(&self, k: &str) -> bool; - fn get_item(&self, k: &str) -> Option; + fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; + fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef); + fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: &str); } pub trait ItemProtocol { - // // Move: doesn't really belong in this protocol. - // fn get_key_value_pairs(&self, vm: &VirtualMachine) -> PyResult; fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( &self, @@ -1280,7 +1260,7 @@ pub trait PyClassImpl: PyClassDef { fn extend_class(ctx: &PyContext, class: &PyClassRef) { Self::impl_extend_class(ctx, class); if let Some(doc) = Self::DOC { - ctx.set_attr(class, "__doc__", ctx.new_str(doc.into())); + class.set_str_attr("__doc__", ctx.new_str(doc.into())); } } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 9daaf95d2..c6dbbc161 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -99,7 +99,7 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); - vm.ctx.set_attr(buffered, "raw", raw.clone()); + vm.set_attr(buffered, "raw", raw.clone())?; Ok(vm.get_none()) } @@ -148,10 +148,10 @@ fn file_io_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let args = vec![name.clone(), vm.ctx.new_int(os_mode)]; let file_no = os::os_open(vm, PyFuncArgs::new(args, vec![]))?; - vm.ctx.set_attr(file_io, "name", name.clone()); - vm.ctx.set_attr(file_io, "fileno", file_no); - vm.ctx.set_attr(file_io, "closefd", vm.new_bool(false)); - vm.ctx.set_attr(file_io, "closed", vm.new_bool(false)); + vm.set_attr(file_io, "name", name.clone())?; + vm.set_attr(file_io, "fileno", file_no)?; + vm.set_attr(file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(file_io, "closed", vm.new_bool(false))?; Ok(vm.get_none()) } @@ -215,7 +215,7 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }; let updated = os::raw_file_number(f.into_inner()); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; Ok(vm.get_none()) } @@ -241,7 +241,7 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(len) => { //reset raw fd on the FileIO object let updated = os::raw_file_number(handle); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; //return number of bytes written Ok(vm.ctx.new_int(len)) @@ -273,7 +273,7 @@ fn text_io_wrapper_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(text_io_wrapper, None), (buffer, None)] ); - vm.ctx.set_attr(text_io_wrapper, "buffer", buffer.clone()); + vm.set_attr(text_io_wrapper, "buffer", buffer.clone())?; Ok(vm.get_none()) } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 80f0f45b6..4695b7acb 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -176,7 +176,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { Some(PyString { ref value }) => value.clone(), _ => unimplemented!("map keys must be strings"), }; - dict.set_item(&self.vm.ctx, &key, value); + dict.set_item(&key, value, self.vm); } Ok(dict.into_object()) } @@ -208,8 +208,10 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { .unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); - vm.ctx.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())); - vm.ctx.set_attr(&exc, "colno", vm.ctx.new_int(err.column())); + vm.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())) + .unwrap(); + vm.set_attr(&exc, "colno", vm.ctx.new_int(err.column())) + .unwrap(); exc }) } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 42b4160ca..31b3ee644 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -114,24 +114,22 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "os", { + let os_name = if cfg!(windows) { + "nt".to_string() + } else { + "posix".to_string() + }; + + py_module!(ctx, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), "O_NONBLOCK" => ctx.new_int(4), "O_APPEND" => ctx.new_int(8), "O_CREAT" => ctx.new_int(512) - }); - - if cfg!(windows) { - ctx.set_attr(&py_mod, "name", ctx.new_str("nt".to_string())); - } else { - // Assume we're on a POSIX system - ctx.set_attr(&py_mod, "name", ctx.new_str("posix".to_string())); - } - - py_mod + }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a4a37b350..c803daaad 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -124,8 +124,7 @@ setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; let modules = ctx.new_dict(); - let sys_name = "sys"; - let sys_mod = py_module!(ctx, sys_name, { + let sys_mod = py_module!(ctx, "sys", { "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), @@ -135,11 +134,11 @@ settrace() -- set the global debug tracing function "ps2" => ctx.new_str("..... ".to_string()), "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), + "modules" => modules.clone(), }); - modules.set_item(&ctx, sys_name, sys_mod.clone()); - modules.set_item(&ctx, "builtins", builtins); - ctx.set_attr(&sys_mod, "modules", modules); + modules.unsafe_str_insert("sys", sys_mod.clone(), &ctx); + modules.unsafe_str_insert("builtins", builtins, &ctx); sys_mod } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 1649d4efc..6c35983e7 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -419,7 +419,7 @@ impl VirtualMachine { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &args.args[i]; - locals.set_item(&self.ctx, arg_name, arg.clone()); + locals.set_item(arg_name, arg.clone(), self); } // Pack other positional arguments in to *args: @@ -432,7 +432,7 @@ impl VirtualMachine { } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(&self.ctx, vararg_name, vararg_value); + locals.set_item(vararg_name, vararg_value, self); } bytecode::Varargs::Unnamed => { // just ignore the rest of the args @@ -452,7 +452,7 @@ impl VirtualMachine { let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { let d = self.ctx.new_dict(); - locals.set_item(&self.ctx, kwargs_name, d.as_object().clone()); + locals.set_item(kwargs_name, d.as_object().clone(), self); Some(d) } bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), @@ -464,15 +464,15 @@ impl VirtualMachine { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { - if locals.contains_key(&name) { + if locals.contains_key(&name, self) { return Err( self.new_type_error(format!("Got multiple values for argument '{}'", name)) ); } - locals.set_item(&self.ctx, &name, value); + locals.set_item(&name, value, self); } else if let Some(d) = &kwargs { - d.set_item(&self.ctx, &name, value); + d.set_item(&name, value, self); } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -499,7 +499,7 @@ impl VirtualMachine { let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; - if !locals.contains_key(variable_name) { + if !locals.contains_key(variable_name, self) { missing.push(variable_name) } } @@ -515,12 +515,8 @@ impl VirtualMachine { // the default if we don't already have a value for (default_index, i) in (required_args..nexpected_args).enumerate() { let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name) { - locals.set_item( - &self.ctx, - arg_name, - available_defaults[default_index].clone(), - ); + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, available_defaults[default_index].clone(), self); } } }; @@ -528,7 +524,7 @@ impl VirtualMachine { // Check if kw only arguments are all present: let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { - if !locals.contains_key(arg_name) { + if !locals.contains_key(arg_name, self) { if kwdefs.contains_key(arg_name) { // If not yet specified, take the default value unimplemented!(); diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index af6ad029e..717c9a973 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -192,7 +192,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&vm.ctx, &String::from(js_sys::JsString::from(key)), py_val); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm); } dict.into_object() } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c7e637a9f..6e4ceaee2 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -243,8 +243,7 @@ impl WASMVirtualMachine { } else { return Err(error()); }; - let rustfunc = vm.ctx.new_rustfunc(print_fn); - vm.ctx.set_attr(&vm.builtins, "print", rustfunc); + vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn))?; Ok(()) })? } From 7438b0685c6d923921f0a615021898cae6693d62 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 15:03:56 +0100 Subject: [PATCH 057/223] Changes suggested by code review. --- vm/src/obj/objdict.rs | 5 +---- vm/src/pyobject.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e110e71f8..6062e07e6 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -251,10 +251,7 @@ impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - match self.entries.borrow().get(key_str) { - Some(v) => Some(v.1.clone()), - None => None, - } + self.entries.borrow().get(key_str).map(|v| v.1.clone()) } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 0c7d4955b..139b2a1e1 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -921,18 +921,18 @@ pub trait DictProtocol { } pub trait ItemProtocol { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( &self, key: T, value: PyObjectRef, vm: &VirtualMachine, - ) -> PyResult; - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; } impl ItemProtocol for PyObjectRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } @@ -941,11 +941,11 @@ impl ItemProtocol for PyObjectRef { key: T, value: PyObjectRef, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } From 9ed051e3b7d7cbe7140d628f06cae4c80c90dafc Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 17:56:39 +0100 Subject: [PATCH 058/223] Module Initialisation takes VirtualMachine rather than PyContext. --- vm/src/builtins.rs | 18 ++++++++++-------- vm/src/dictdatatype.rs | 15 +++++++++------ vm/src/import.rs | 2 +- vm/src/macros.rs | 19 +++++++++++++------ vm/src/obj/objdict.rs | 17 ----------------- vm/src/stdlib/ast.rs | 8 +++++--- vm/src/stdlib/dis.rs | 8 +++++--- vm/src/stdlib/io.rs | 10 +++++----- vm/src/stdlib/json.rs | 9 +++++---- vm/src/stdlib/keyword.rs | 8 +++++--- vm/src/stdlib/math.rs | 8 +++++--- vm/src/stdlib/mod.rs | 6 ++++-- vm/src/stdlib/os.rs | 8 +++++--- vm/src/stdlib/platform.rs | 7 ++++--- vm/src/stdlib/pystruct.rs | 8 +++++--- vm/src/stdlib/random.rs | 8 +++++--- vm/src/stdlib/re.rs | 8 +++++--- vm/src/stdlib/socket.rs | 10 +++++----- vm/src/stdlib/string.rs | 9 ++++++--- vm/src/stdlib/time_module.rs | 8 +++++--- vm/src/stdlib/tokenize.rs | 8 +++++--- vm/src/stdlib/types.rs | 8 +++++--- vm/src/stdlib/weakref.rs | 9 ++++++--- vm/src/sysmodule.rs | 14 +++++++------- vm/src/vm.rs | 21 +++++++++++++-------- wasm/lib/src/browser_module.rs | 10 +++++----- wasm/lib/src/vm_class.rs | 10 +++++----- 27 files changed, 153 insertions(+), 121 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 7264c9c6e..5aa5ee226 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -20,7 +20,7 @@ use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + DictProtocol, IdProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -668,13 +668,15 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_vars -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - #[cfg(target_arch = "wasm32")] - let open = ctx.none(); - #[cfg(not(target_arch = "wasm32"))] - let open = ctx.new_rustfunc(io_open); +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { + let ctx = &vm.ctx; - py_module!(ctx, "__builtins__", { + #[cfg(target_arch = "wasm32")] + let open = vm.ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = vm.ctx.new_rustfunc(io_open); + + extend_module!(vm, module, { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -765,7 +767,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), - }) + }); } pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 436923ea9..92a93a4e9 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,6 +9,7 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; +#[derive(Default)] pub struct Dict { size: usize, indices: HashMap, @@ -88,6 +89,12 @@ impl Dict { } } + pub fn clear(&mut self) { + self.entries.clear(); + self.indices.clear(); + self.size = 0 + } + /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { @@ -175,12 +182,8 @@ fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { } /// Invoke __eq__ on two keys -fn do_eq( - vm: &VirtualMachine, - key1: &PyObjectRef, - key2: &PyObjectRef, -) -> Result { - let result = vm._eq(key1, key2.clone())?; +fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { + let result = vm._eq(key1.clone(), key2.clone())?; Ok(objbool::get_value(&result)) } diff --git a/vm/src/import.rs b/vm/src/import.rs index 28b3d02ef..713393deb 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -15,7 +15,7 @@ use crate::vm::VirtualMachine; fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &str) -> PyResult { // Check for Rust-native modules if let Some(module) = vm.stdlib_inits.borrow().get(module) { - return Ok(module(&vm.ctx).clone()); + return Ok(module(vm).clone()); } let notfound_error = vm.context().exceptions.module_not_found_error.clone(); diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 0c37e0dfc..27d164718 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -114,17 +114,24 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ - let mut attributes = $crate::pyobject::PyAttributes::new(); + ( $vm:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let module = $vm.ctx.new_module($module_name, $vm.ctx.new_dict()); $( - let value: PyObjectRef = $value.into(); - attributes.insert($name.to_string(), value); + $vm.set_attr(&module, $name, $value).unwrap(); )* - let module_dict = $crate::obj::objdict::PyDictRef::from_attributes($ctx, attributes); - $ctx.new_module($module_name, module_dict) + module }}; } +#[macro_export] +macro_rules! extend_module { + ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)* }) => { + $( + $vm.set_attr(&$module, $name, $value).unwrap(); + )* + } +} + #[macro_export] macro_rules! py_class { ( $ctx:expr, $class_name:expr, $class_base:expr, { $($name:expr => $value:expr),* $(,)* }) => { diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6062e07e6..6ebe5aa2e 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -215,23 +215,6 @@ impl PyDictRef { } } - // Used during module initialisation when vm isn't available. - pub fn from_attributes(ctx: &PyContext, attributes: PyAttributes) -> PyDictRef { - let dict = ctx.new_dict(); - for (key_str, value) in attributes.into_iter() { - dict.unsafe_str_insert(&key_str, value, ctx); - } - dict - } - - // Pub needed for some nasty edge cases. - // It will be unsafe if there are entries in the dictionary that compare equal. - pub fn unsafe_str_insert(&self, key: &str, value: PyObjectRef, ctx: &PyContext) { - self.entries - .borrow_mut() - .insert(key.to_string(), (ctx.new_str(key.to_string()), value)); - } - /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index f6aa93aad..e7ac5308c 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -12,7 +12,7 @@ use rustpython_parser::{ast, parser}; use crate::function::PyFuncArgs; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -656,8 +656,10 @@ fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(ast_node) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "ast", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index 222f23383..4c35f45e9 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -1,5 +1,5 @@ use crate::obj::objcode::PyCodeRef; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TryFromObject}; +use crate::pyobject::{PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -17,8 +17,10 @@ fn dis_disassemble(co: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "dis", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "dis", { "dis" => ctx.new_rustfunc(dis_dis), "disassemble" => ctx.new_rustfunc(dis_disassemble) }) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index c6dbbc161..bd33eac75 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,9 +20,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{ - BufferProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; fn compute_c_flag(mode: &str) -> u16 { @@ -365,7 +363,9 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), @@ -421,7 +421,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "getvalue" => ctx.new_rustfunc(bytes_io_getvalue) }); - py_module!(ctx, "io", { + py_module!(vm, "io", { "open" => ctx.new_rustfunc(io_open), "IOBase" => io_base, "RawIOBase" => raw_io_base, diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 4695b7acb..631f8257f 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -12,8 +12,7 @@ use crate::obj::{ objtype, }; use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, - TypeProtocol, + create_type, DictProtocol, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol, }; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -232,7 +231,9 @@ fn json_loads(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { de_pyobject(vm, &objstr::get_value(&string)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + // TODO: Make this a proper type with a constructor let json_decode_error = create_type( "JSONDecodeError", @@ -240,7 +241,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { &ctx.exceptions.exception_type, ); - py_module!(ctx, "json", { + py_module!(vm, "json", { "dumps" => ctx.new_rustfunc(json_dumps), "loads" => ctx.new_rustfunc(json_loads), "JSONDecodeError" => json_decode_error diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index fe27399c5..3143af21a 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,7 +18,9 @@ fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let keyword_kwlist = ctx.new_list( lexer::get_keywords() .keys() @@ -26,7 +28,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { .collect(), ); - py_module!(ctx, "keyword", { + py_module!(vm, "keyword", { "iskeyword" => ctx.new_rustfunc(keyword_iskeyword), "kwlist" => keyword_kwlist }) diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 5af599cb7..dc7981ab6 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; // Helper macro: @@ -172,8 +172,10 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "math", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "math", { // Number theory functions: "fabs" => ctx.new_rustfunc(math_fabs), "isfinite" => ctx.new_rustfunc(math_isfinite), diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index b0b61119c..a69a80d11 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -15,14 +15,16 @@ mod types; mod weakref; use std::collections::HashMap; +use crate::vm::VirtualMachine; + #[cfg(not(target_arch = "wasm32"))] pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; -pub type StdlibInitFunc = Box PyObjectRef>; +pub type StdlibInitFunc = Box PyObjectRef>; pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 31b3ee644..3f0f557f1 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,7 +7,7 @@ use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::objint; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -113,14 +113,16 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let os_name = if cfg!(windows) { "nt".to_string() } else { "posix".to_string() }; - py_module!(ctx, "os", { + py_module!(vm, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), diff --git a/vm/src/stdlib/platform.rs b/vm/src/stdlib/platform.rs index a87383a72..6d4751a35 100644 --- a/vm/src/stdlib/platform.rs +++ b/vm/src/stdlib/platform.rs @@ -1,9 +1,10 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "platform", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "platform", { "python_compiler" => ctx.new_rustfunc(platform_python_compiler), "python_implementation" => ctx.new_rustfunc(platform_python_implementation), "python_version" => ctx.new_rustfunc(platform_python_version), diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 9dadc520a..2ff61d630 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -15,7 +15,7 @@ use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::VirtualMachine; #[derive(Debug)] @@ -297,8 +297,10 @@ fn struct_unpack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_tuple(items)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "struct", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "struct", { "pack" => ctx.new_rustfunc(struct_pack), "unpack" => ctx.new_rustfunc(struct_unpack) }) diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index 82c1f410c..c6cc44a36 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,11 +4,13 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "random", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "random", { "guass" => ctx.new_rustfunc(random_gauss), "normalvariate" => ctx.new_rustfunc(random_normalvariate), "random" => ctx.new_rustfunc(random_random), diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index fee100bfe..b9f921502 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -8,7 +8,7 @@ use regex::{Match, Regex}; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; impl PyValue for Regex { @@ -98,7 +98,9 @@ impl PyMatchRef { } /// Create the python `re` module with all its members. -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let match_type = py_class!(ctx, "Match", ctx.object(), { "start" => ctx.new_rustfunc(PyMatchRef::start), "end" => ctx.new_rustfunc(PyMatchRef::end) @@ -109,7 +111,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "search" => ctx.new_rustfunc(PyRegexRef::search) }); - py_module!(ctx, "re", { + py_module!(vm, "re", { "compile" => ctx.new_rustfunc(re_compile), "Match" => match_type, "match" => ctx.new_rustfunc(re_match), diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index fe0ce29a0..ab5072f7c 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,9 +10,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objsequence::get_elements; use crate::obj::objstr; -use crate::pyobject::{ - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use crate::obj::objtype::PyClassRef; @@ -411,7 +409,9 @@ fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { Ok(vm.ctx.new_tuple(vec![ip, port])) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let socket = py_class!(ctx, "socket", ctx.object(), { "__new__" => ctx.new_rustfunc(socket_new), "connect" => ctx.new_rustfunc(socket_connect), @@ -426,7 +426,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "recvfrom" => ctx.new_rustfunc(socket_recvfrom), }); - py_module!(ctx, "socket", { + py_module!(vm, "socket", { "AF_INET" => ctx.new_int(AddressFamily::Inet as i32), "SOCK_STREAM" => ctx.new_int(SocketKind::Stream as i32), "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 157dae035..c255ffc93 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -3,9 +3,12 @@ * */ -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { let ascii_lowercase = "abcdefghijklmnopqrstuvwxyz".to_string(); let ascii_uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_string(); let ascii_letters = format!("{}{}", ascii_lowercase, ascii_uppercase); @@ -19,7 +22,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { */ // Constants: - py_module!(ctx, "string", { + py_module!(vm, "string", { "ascii_letters" => ctx.new_str(ascii_letters), "ascii_lowercase" => ctx.new_str(ascii_lowercase), "ascii_uppercase" => ctx.new_str(ascii_uppercase), diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index dfe3ab8e3..93beabf8a 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -32,8 +32,10 @@ fn time_time(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "time", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "time", { "sleep" => ctx.new_rustfunc(time_sleep), "time" => ctx.new_rustfunc(time_time) }) diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 0aa5df3d2..88a26881c 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -25,8 +25,10 @@ fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: create main function when called with -m -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "tokenize", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "tokenize", { "tokenize" => ctx.new_rustfunc(tokenize_tokenize) }) } diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs index 8a5ab2939..ad76a80a4 100644 --- a/vm/src/stdlib/types.rs +++ b/vm/src/stdlib/types.rs @@ -7,7 +7,7 @@ use crate::obj::objdict::PyDict; use crate::obj::objstr::PyStringRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; +use crate::pyobject::{PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; use crate::VirtualMachine; fn types_new_class( @@ -25,8 +25,10 @@ fn types_new_class( objtype::type_new_class(vm, vm.ctx.type_type(), name, bases, dict) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "types", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "types", { "new_class" => ctx.new_rustfunc(types_new_class), "FunctionType" => ctx.function_type(), "LambdaType" => ctx.function_type(), diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 5616c3ddf..b5f7b756a 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -5,10 +5,13 @@ //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -use super::super::pyobject::{PyContext, PyObjectRef}; +use super::super::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "_weakref", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "_weakref", { "ref" => ctx.weakref_type() }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index c803daaad..772dfe198 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -39,7 +39,9 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } -pub fn make_module(ctx: &PyContext, builtins: PyObjectRef) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { + let ctx = &vm.ctx; + let path_list = match env::var_os("PYTHONPATH") { Some(paths) => env::split_paths(&paths) .map(|path| { @@ -124,7 +126,7 @@ setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; let modules = ctx.new_dict(); - let sys_mod = py_module!(ctx, "sys", { + extend_module!(vm, module, { "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), @@ -137,8 +139,6 @@ settrace() -- set the global debug tracing function "modules" => modules.clone(), }); - modules.unsafe_str_insert("sys", sys_mod.clone(), &ctx); - modules.unsafe_str_insert("builtins", builtins, &ctx); - - sys_mod + modules.set_item("sys", module.clone(), vm); + modules.set_item("builtins", builtins.clone(), vm); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 6c35983e7..0bd255574 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -58,18 +58,22 @@ impl VirtualMachine { let ctx = PyContext::new(); // Hard-core modules: - let builtins = builtins::make_module(&ctx); - let sysmod = sysmodule::make_module(&ctx, builtins.clone()); + let builtins = ctx.new_module("builtins", ctx.new_dict()); + let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); - VirtualMachine { - builtins, - sys_module: sysmod, + let vm = VirtualMachine { + builtins: builtins.clone(), + sys_module: sysmod.clone(), stdlib_inits, ctx, frames: RefCell::new(vec![]), wasm_id: None, - } + }; + + builtins::make_module(&vm, builtins.clone()); + sysmodule::make_module(&vm, sysmod, builtins); + vm } pub fn run_code_obj(&self, code: PyCodeRef, scope: Scope) -> PyResult { @@ -564,15 +568,16 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: PyObjectRef) -> PyResult + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: V) -> PyResult where K: TryIntoRef, + V: Into, { let attr_name = attr_name.try_into_ref(self)?; self.call_method( obj, "__setattr__", - vec![attr_name.into_object(), attr_value], + vec![attr_name.into_object(), attr_value.into()], ) } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 0e13aeb3a..cb00d175c 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -12,9 +12,7 @@ use rustpython_vm::obj::{ objstr::{self, PyStringRef}, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{ - PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use rustpython_vm::VirtualMachine; use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; @@ -348,7 +346,9 @@ fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(result) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let promise = py_class!(ctx, "Promise", ctx.object(), { "then" => ctx.new_rustfunc(PyPromise::then), "catch" => ctx.new_rustfunc(PyPromise::catch), @@ -371,7 +371,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "set_attr" => ctx.new_rustfunc(Element::set_attr), }); - py_module!(ctx, "browser", { + py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), "request_animation_frame" => ctx.new_rustfunc(browser_request_animation_frame), "cancel_animation_frame" => ctx.new_rustfunc(browser_cancel_animation_frame), diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 6e4ceaee2..dbf148f2e 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -8,7 +8,7 @@ use wasm_bindgen::prelude::*; use rustpython_vm::compile; use rustpython_vm::frame::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyResult}; +use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult}; use rustpython_vm::VirtualMachine; use crate::browser_module::setup_browser_module; @@ -260,12 +260,12 @@ impl WASMVirtualMachine { let mod_name = name.clone(); - let stdlib_init_fn = move |ctx: &PyContext| { - let py_mod = ctx.new_module(&name, ctx.new_dict()); + let stdlib_init_fn = move |vm: &VirtualMachine| { + let module = vm.ctx.new_module(&name, vm.ctx.new_dict()); for (key, value) in module_items.clone() { - ctx.set_attr(&py_mod, &key, value); + vm.set_attr(&module, key, value).unwrap(); } - py_mod + module }; vm.stdlib_inits From b943f4a4bb465d88af212461dfc7406259247b83 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 17:58:14 +0100 Subject: [PATCH 059/223] Historic real proper dictionary support. --- tests/snippets/dict.py | 4 ++ vm/src/frame.rs | 4 +- vm/src/lib.rs | 1 + vm/src/obj/objdict.rs | 90 ++++++++++++++++------------------------ vm/src/obj/objobject.rs | 2 +- vm/src/pyobject.rs | 2 +- wasm/lib/src/vm_class.rs | 3 +- 7 files changed, 47 insertions(+), 59 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 170a6492d..a80b5b9e2 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -34,3 +34,7 @@ res = set() for key in a.keys(): res.add(key) assert res == set(['a','b']) + +x = {} +x[1] = 1 +assert x[1] == 1 diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 1a0cc4de7..0b735df87 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -157,8 +157,8 @@ impl NameProtocol for Scope { self.get_locals().set_item(key, value, vm) } - fn delete_name(&self, _vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key) + fn delete_name(&self, vm: &VirtualMachine, key: &str) { + self.get_locals().del_item(key, vm) } } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 4a300dcef..89b4dd872 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -44,6 +44,7 @@ pub mod macros; mod builtins; pub mod bytecode; pub mod compile; +mod dictdatatype; pub mod error; pub mod eval; mod exceptions; diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6ebe5aa2e..22e4fb518 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,20 +1,22 @@ use std::cell::{Cell, RefCell}; -use std::collections::HashMap; use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, IntoPyObject, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + DictProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, + PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; +use crate::dictdatatype; + use super::objiter; use super::objlist::PyListIterator; -use super::objstr::{self, PyStringRef}; +use super::objstr; use super::objtype; use crate::obj::objtype::PyClassRef; -pub type DictContentType = HashMap; +pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { @@ -37,13 +39,11 @@ impl PyValue for PyDict { } pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = dict.payload::().unwrap().entries.borrow(); - let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_elements.iter() { - let (key, obj) = pair; - pairs.push((key.clone(), obj.clone())); - } - pairs + dict.payload::() + .unwrap() + .entries + .borrow() + .get_items() } // Python dict methods: @@ -112,18 +112,12 @@ impl PyDictRef { Ok(vm.new_str(s)) } - fn contains(self, key: PyStringRef, _vm: &VirtualMachine) -> bool { - self.entries.borrow().contains_key(&key.value) + fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let key = &key.value; - // Delete the item: - let mut elements = self.entries.borrow_mut(); - match elements.remove(key) { - Some(_) => Ok(()), - None => Err(vm.new_value_error(format!("Key not found: {}", key))), - } + fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().delete(vm, &key) } fn clear(self, _vm: &VirtualMachine) { @@ -136,7 +130,8 @@ impl PyDictRef { let keys = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(k, _v)| k.clone()) .collect(); let key_list = vm.ctx.new_list(keys); @@ -152,7 +147,8 @@ impl PyDictRef { let values = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(_k, v)| v.clone()) .collect(); let values_list = vm.ctx.new_list(values); @@ -168,7 +164,8 @@ impl PyDictRef { let items = self .entries .borrow() - .values() + .get_items() + .iter() .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) .collect(); let items_list = vm.ctx.new_list(items); @@ -183,35 +180,22 @@ impl PyDictRef { self.set_item(needle, value, vm) } - fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { - let key = &key.value; - - // What we are looking for: - let elements = self.entries.borrow(); - if elements.contains_key(key) { - Ok(elements[key].1.clone()) - } else { - Err(vm.new_value_error(format!("Key not found: {}", key))) - } + fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().get(vm, &key) } fn get( self, - key: PyStringRef, + key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - // What we are looking for: - let key = &key.value; - - let elements = self.entries.borrow(); - if elements.contains_key(key) { - elements[key].1.clone() - } else { - match default { + match self.into_object().get_item(key, vm) { + Ok(value) => value, + Err(_) => match default { OptionalArg::Present(value) => value, OptionalArg::Missing => vm.ctx.none(), - } + }, } } @@ -228,13 +212,13 @@ impl PyDictRef { impl DictProtocol for PyDictRef { fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { - let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - self.entries.borrow().get(key_str).is_some() + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().contains(vm, &key).unwrap() } fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { - let key_str = &objstr::get_value(&key.into_pyobject(vm).unwrap()); - self.entries.borrow().get(key_str).map(|v| v.1.clone()) + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().get(vm, &key).ok() } fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -244,14 +228,12 @@ impl DictProtocol for PyDictRef { // Item set/get: fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { let key = key.into_pyobject(vm).unwrap(); - let key_str = &objstr::get_value(&key); - let elements = &mut self.entries.borrow_mut(); - elements.insert(key_str.to_string(), (key.clone(), value)); + self.entries.borrow_mut().insert(vm, &key, value).unwrap() } - fn del_item(&self, key: &str) { - let elements = &mut self.entries.borrow_mut(); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) { + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow_mut().delete(vm, &key).unwrap(); } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 86a56de5d..1336e5246 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -98,7 +98,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.del_item(&attr_name.value); + dict.del_item(attr_name, vm); Ok(()) } else { Err(vm.new_attribute_error(format!( diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 139b2a1e1..de4d29628 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -917,7 +917,7 @@ pub trait DictProtocol { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); - fn del_item(&self, key: &str); + fn del_item(&self, key: T, vm: &VirtualMachine); } pub trait ItemProtocol { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index dbf148f2e..d6f9ede0e 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -243,7 +243,8 @@ impl WASMVirtualMachine { } else { return Err(error()); }; - vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn))?; + vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn)) + .unwrap(); Ok(()) })? } From 43347e2d8bb67395aacfde800dd68ddc9a6d2f7a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 4 Apr 2019 18:00:14 +0100 Subject: [PATCH 060/223] Dictionary: KeyError -> ValueError. --- vm/src/dictdatatype.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 92a93a4e9..ac1df994a 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -85,7 +85,7 @@ impl Dict { } } else { let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } @@ -103,7 +103,7 @@ impl Dict { Ok(()) } else { let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } From 77d5f57df1dd1f63e5393d279c711d9eac8605ce Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 14:36:19 +0100 Subject: [PATCH 061/223] Add some additional dictionary tests. --- tests/snippets/dict.py | 50 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index a80b5b9e2..5c5ef62b1 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + def dict_eq(d1, d2): return (all(k in d2 and d1[k] == d2[k] for k in d1) and all(k in d1 and d1[k] == d2[k] for k in d2)) @@ -21,20 +23,60 @@ assert len(a) == 0 a = {'a': 5, 'b': 6} res = set() for value in a.values(): - res.add(value) + res.add(value) assert res == set([5,6]) count = 0 for (key, value) in a.items(): - assert a[key] == value - count += 1 + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) + res.add(key) assert res == set(['a','b']) x = {} x[1] = 1 assert x[1] == 1 + + +x[7] = 7 +x[2] = 2 +x[(5, 6)] = 5 + +with assertRaises(KeyError): + x["not here"] + +with assertRaises(TypeError): + x[[]] # Unhashable type. + +x["here"] = "here" +assert x.get("not here", "default") == "default" +assert x.get("here", "default") == "here" +assert x.get("not here") == None + +# An object that hashes to the same value always, and compares equal if any its values match. +class Hashable(object): + def __init__(self, *args): + self.values = args + def __hash__(self): + return 1 + def __eq__(self, other): + for x in self.values: + for y in other.values: + if x == y: + return True + return False + +x = {} +x[Hashable(1,2)] = 8 + +assert x[Hashable(1,2)] == 8 +assert x[Hashable(3,1)] == 8 + +x[Hashable(8)] = 19 +x[Hashable(19,8)] = 1 +assert x[Hashable(8)] == 1 +assert len(x) == 2 From ec94168a15dedc80297659934623a1730a99ce6a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:00:10 +0100 Subject: [PATCH 062/223] Remove objdict::get_key_value_pairs. --- vm/src/frame.rs | 11 +++++++---- vm/src/obj/objdict.rs | 25 ++++++++----------------- vm/src/obj/objmodule.rs | 2 +- vm/src/pyobject.rs | 1 - vm/src/stdlib/json.rs | 7 +++++-- wasm/lib/src/browser_module.rs | 4 +++- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 0b735df87..f6fc8e96e 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,7 +12,7 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict::{self, PyDictRef}; +use crate::obj::objdict::PyDictRef; use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; @@ -390,7 +390,9 @@ impl Frame { let obj = self.pop_value(); if *unpack { // Take all key-value pairs from the dict: - let dict_elements = objdict::get_key_value_pairs(&obj); + let dict: PyDictRef = + obj.downcast().expect("Need a dictionary to build a map."); + let dict_elements = dict.get_key_value_pairs(); for (key, value) in dict_elements.iter() { map_obj.set_item(key.clone(), value.clone(), vm); } @@ -612,8 +614,9 @@ impl Frame { } bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { - let kw_dict = self.pop_value(); - let dict_elements = objdict::get_key_value_pairs(&kw_dict).clone(); + let kw_dict: PyDictRef = + self.pop_value().downcast().expect("Kwargs must be a dict."); + let dict_elements = kw_dict.get_key_value_pairs(); dict_elements .into_iter() .map(|elem| (objstr::get_value(&elem.0), elem.1)) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4ac1431b3..b2bf45304 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -13,7 +13,6 @@ use crate::dictdatatype; use super::objiter; use super::objlist::PyListIterator; use super::objstr; -use super::objtype; use crate::obj::objtype::PyClassRef; pub type DictContentType = dictdatatype::Dict; @@ -38,14 +37,6 @@ impl PyValue for PyDict { } } -pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - dict.payload::() - .unwrap() - .entries - .borrow() - .get_items() -} - // Python dict methods: impl PyDictRef { fn new( @@ -56,8 +47,9 @@ impl PyDictRef { ) -> PyResult { let dict = vm.ctx.new_dict(); if let OptionalArg::Present(dict_obj) = dict_obj { - if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { - for (needle, value) in get_key_value_pairs(&dict_obj) { + let dicted: PyResult = dict_obj.clone().downcast(); + if let Ok(dict_obj) = dicted { + for (needle, value) in dict_obj.get_key_value_pairs() { dict.set_item(needle, value, vm); } } else { @@ -97,9 +89,8 @@ impl PyDictRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { - let elements = get_key_value_pairs(self.as_object()); let mut str_parts = vec![]; - for (key, value) in elements { + for (key, value) in self.get_key_value_pairs() { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -176,6 +167,10 @@ impl PyDictRef { } } + pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + self.entries.borrow().get_items() + } + fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { self.set_item(needle, value, vm) } @@ -225,10 +220,6 @@ impl DictProtocol for PyDictRef { self.entries.borrow().get(vm, &key).ok() } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - get_key_value_pairs(self.as_object()) - } - // Item set/get: fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { let key = key.into_pyobject(vm).unwrap(); diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index b641f0426..75749d557 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -1,5 +1,5 @@ use crate::obj::objtype::PyClassRef; -use crate::pyobject::{DictProtocol, PyContext, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index de4d29628..4df7e57fe 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -915,7 +915,6 @@ impl TypeProtocol for PyRef { pub trait DictProtocol { fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: T, vm: &VirtualMachine); } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 631f8257f..b7cb704b1 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -7,7 +7,9 @@ use serde_json; use crate::function::PyFuncArgs; use crate::obj::{ - objbool, objdict, objfloat, objint, objsequence, + objbool, + objdict::PyDictRef, + objfloat, objint, objsequence, objstr::{self, PyString}, objtype, }; @@ -63,7 +65,8 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { let elements = objsequence::get_elements(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_key_value_pairs(self.pyobject); + let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); + let pairs = dict.get_key_value_pairs(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index cb00d175c..6f7c31964 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,6 +7,7 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ + objdict::PyDictRef, objfunction::PyFunction, objint, objstr::{self, PyStringRef}, @@ -72,7 +73,8 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(headers) = headers { let h = request.headers(); - for (key, value) in rustpython_vm::obj::objdict::get_key_value_pairs(&headers) { + let headers: PyDictRef = headers.downcast().unwrap(); + for (key, value) in headers.get_key_value_pairs() { let key = &vm.to_str(&key)?.value; let value = &vm.to_str(&value)?.value; h.set(key, value) From 4212594c5b108afcd076fa56930ec861c559bf8a Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:01:44 +0100 Subject: [PATCH 063/223] Prefer key to needle. --- vm/src/obj/objdict.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index b2bf45304..449c60726 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -49,8 +49,8 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - for (needle, value) in dict_obj.get_key_value_pairs() { - dict.set_item(needle, value, vm); + for (key, value) in dict_obj.get_key_value_pairs() { + dict.set_item(key, value, vm); } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -63,18 +63,17 @@ impl PyDictRef { None => break, }; let elem_iter = objiter::get_iter(vm, &element)?; - let needle = - objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + let key = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(needle, value, vm); + dict.set_item(key, value, vm); } } } - for (needle, value) in kwargs.into_iter() { - dict.set_item(vm.new_str(needle), value, vm); + for (key, value) in kwargs.into_iter() { + dict.set_item(vm.new_str(key), value, vm); } Ok(dict) } @@ -171,8 +170,8 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(needle, value, vm) + fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { + self.set_item(key, value, vm) } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { From 0dce9bb96aac9e446f0b87111e8b27c0135f2489 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 15:10:52 +0100 Subject: [PATCH 064/223] Use item protocol when executing instructions. --- vm/src/frame.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index f6fc8e96e..858b33a08 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -21,8 +21,8 @@ use crate::obj::objstr; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, + DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -990,22 +990,18 @@ impl Frame { } } - fn subscript(&self, vm: &VirtualMachine, a: PyObjectRef, b: PyObjectRef) -> PyResult { - vm.call_method(&a, "__getitem__", vec![b]) - } - fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); let value = self.pop_value(); - vm.call_method(&obj, "__setitem__", vec![idx, value])?; + obj.set_item(idx, value, vm)?; Ok(None) } fn execute_delete_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); - vm.call_method(&obj, "__delitem__", vec![idx])?; + obj.del_item(idx, vm)?; Ok(None) } @@ -1040,7 +1036,7 @@ impl Frame { bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), // TODO: Subscript should probably have its own op bytecode::BinaryOperator::Subscript if inplace => unreachable!(), - bytecode::BinaryOperator::Subscript => self.subscript(vm, a_ref, b_ref), + bytecode::BinaryOperator::Subscript => a_ref.get_item(b_ref, vm), bytecode::BinaryOperator::Modulo if inplace => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::Modulo => vm._mod(a_ref, b_ref), bytecode::BinaryOperator::Lshift if inplace => vm._ilshift(a_ref, b_ref), From f354f4ce028cd86b63f33641dc0aa4c53d253274 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 5 Apr 2019 16:14:02 +0100 Subject: [PATCH 065/223] Move contains_key to PyDictRef. --- vm/src/obj/objdict.rs | 6 +++--- vm/src/pyobject.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 449c60726..541645d5f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -206,14 +206,14 @@ impl PyDictRef { fn hash(self, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } -} -impl DictProtocol for PyDictRef { - fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + pub fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().contains(vm, &key).unwrap() } +} +impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().get(vm, &key).ok() diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4df7e57fe..bf5cf3c86 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -913,7 +913,6 @@ impl TypeProtocol for PyRef { } pub trait DictProtocol { - fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool; fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); fn del_item(&self, key: T, vm: &VirtualMachine); From 37e7972dcd438dfd0e0305236e4a0bbbc66a99eb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 17:51:03 +0300 Subject: [PATCH 066/223] Add os.read --- tests/snippets/stdlib_os.py | 9 ++++++++- vm/src/stdlib/os.rs | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ed9e91c70..d70050495 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -2,8 +2,15 @@ import os from testutils import assert_raises -assert os.open('README.md', 0) > 0 +fd = os.open('README.md', 0) +assert fd > 0 +assert len(os.read(fd, 10)) == 10 +assert len(os.read(fd, 5)) == 5 + +assert_raises(OSError, lambda: os.read(fd + 1, 10)) +os.close(fd) +assert_raises(OSError, lambda: os.read(fd, 10)) assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 3f0f557f1..0f5762014 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,6 +1,6 @@ use std::fs::File; use std::fs::OpenOptions; -use std::io::ErrorKind; +use std::io::{ErrorKind, Read}; use num_traits::cast::ToPrimitive; @@ -113,6 +113,25 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } +fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(fd, Some(vm.ctx.int_type())), (n, Some(vm.ctx.int_type()))] + ); + + let mut buffer = vec![0u8; objint::get_value(n).to_usize().unwrap()]; + let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); + match file.read_exact(&mut buffer) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_bytes(buffer)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -126,6 +145,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "read" => ctx.new_rustfunc(os_read), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 28c3ef1ae3c8a867c52f4306bd81e7587b484bff Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:17:36 +0300 Subject: [PATCH 067/223] Add os.write --- vm/src/stdlib/os.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0f5762014..b7e4280e5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,10 +1,11 @@ use std::fs::File; use std::fs::OpenOptions; -use std::io::{ErrorKind, Read}; +use std::io::{ErrorKind, Read, Write}; use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; +use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; @@ -132,6 +133,27 @@ fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(buffer)) } +fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [ + (fd, Some(vm.ctx.int_type())), + (data, Some(vm.ctx.bytes_type())) + ] + ); + + let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); + let written = match file.write(&objbytes::get_value(&data)) { + Ok(written) => written, + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_int(written)) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -146,6 +168,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), "read" => ctx.new_rustfunc(os_read), + "write" => ctx.new_rustfunc(os_write), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 1f19951ece36d1e229f0054abacdcfa4a2cf0907 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:43:46 +0300 Subject: [PATCH 068/223] Add os.{remove,unlink} --- vm/src/stdlib/os.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b7e4280e5..cd4c0709b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,3 +1,4 @@ +use std::fs; use std::fs::File; use std::fs::OpenOptions; use std::io::{ErrorKind, Read, Write}; @@ -154,6 +155,18 @@ fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(written)) } +fn os_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(path, Some(vm.ctx.str_type()))]); + let fname = objstr::get_value(&path); + + match fs::remove_file(fname) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -169,6 +182,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "error" => ctx.new_rustfunc(os_error), "read" => ctx.new_rustfunc(os_read), "write" => ctx.new_rustfunc(os_write), + "remove" => ctx.new_rustfunc(os_remove), + "unlink" => ctx.new_rustfunc(os_remove), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From cdd39bc50e7054b6ffbb9e12245b2e5140e59d83 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 5 Apr 2019 19:44:09 +0300 Subject: [PATCH 069/223] Add tests for os.{read, write} --- tests/snippets/stdlib_os.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index d70050495..29c8005dc 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -5,13 +5,36 @@ from testutils import assert_raises fd = os.open('README.md', 0) assert fd > 0 -assert len(os.read(fd, 10)) == 10 -assert len(os.read(fd, 5)) == 5 - assert_raises(OSError, lambda: os.read(fd + 1, 10)) os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) +FNAME = "test_file_that_no_one_will_have_on_disk" +CONTENT = b"testing" +CONTENT2 = b"rustpython" +CONTENT3 = b"BOYA" + +class TestWithFile(): + def __enter__(self): + open(FNAME, "wb") + return FNAME + + def __exit__(self, exc_type, exc_val, exc_tb): + os.remove(FNAME) + + +with TestWithFile() as fname: + fd = os.open(fname, 1) + assert os.write(fd, CONTENT2) == len(CONTENT2) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) + + fd = os.open(fname, 0) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) + + assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) From ed8c1e1a955725093c1ab04ea6b4e2c9c52e94ac Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 29 Mar 2019 16:39:52 +0300 Subject: [PATCH 070/223] Implement set and frozenset with PySetInner --- tests/snippets/set.py | 7 + vm/src/obj/objset.rs | 1110 +++++++++++++++++++++++------------------ 2 files changed, 622 insertions(+), 495 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 9e49dff89..011716335 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -205,6 +205,13 @@ assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) assert_raises(TypeError, lambda: frozenset([[]])) +a = frozenset([1,2,3]) +b = set() +for e in a: + assert e == 1 or e == 2 or e == 3 + b.add(e) +assert a == b + # set and frozen set assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 34d487fbb..eaa0cd09c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -7,7 +7,7 @@ use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; use std::hash::{Hash, Hasher}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::OptionalArg; use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::{ReprGuard, VirtualMachine}; @@ -20,13 +20,13 @@ use super::objtype::PyClassRef; #[derive(Default)] pub struct PySet { - elements: RefCell>, + inner: RefCell, } pub type PySetRef = PyRef; #[derive(Default)] pub struct PyFrozenSet { - elements: HashMap, + inner: PySetInner, } pub type PyFrozenSetRef = PyRef; @@ -56,13 +56,564 @@ impl PyValue for PyFrozenSet { } } -pub fn get_elements(obj: &PyObjectRef) -> HashMap { +#[derive(Default, Clone)] +struct PySetInner { + elements: HashMap, +} + +impl PySetInner { + fn len(&self) -> usize { + self.elements.len() + } + fn copy(&self) -> PySetInner { + PySetInner { + elements: self.elements.clone(), + } + } + + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + for element in self.elements.iter() { + let value = vm._eq(needle.clone(), element.1.clone())?; + if objbool::get_value(&value) { + return Ok(vm.new_bool(true)); + } + } + Ok(vm.new_bool(false)) + } + + fn _compare_inner( + &self, + other: &PySetInner, + size_func: &Fn(usize, usize) -> bool, + swap: bool, + vm: &VirtualMachine, + ) -> PyResult { + let get_zelf = |swap: bool| -> &PySetInner { + if swap { + other + } else { + self + } + }; + let get_other = |swap: bool| -> &PySetInner { + if swap { + self + } else { + other + } + }; + + if size_func(get_zelf(swap).len(), get_other(swap).len()) { + return Ok(vm.new_bool(false)); + } + for element in get_other(swap).elements.iter() { + let value = get_zelf(swap).contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + return Ok(vm.new_bool(false)); + } + } + Ok(vm.new_bool(true)) + } + + fn eq(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf != other }, + false, + vm, + ) + } + + fn ge(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + false, + vm, + ) + } + + fn gt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + false, + vm, + ) + } + + fn le(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + true, + vm, + ) + } + + fn lt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + true, + vm, + ) + } + + fn union(&self, other: &PySetInner, _vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + elements.extend(other.elements.clone()); + + Ok(PySetInner { elements }) + } + + fn intersection(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._combine_inner(other, vm, SetCombineOperation::Intersection) + } + + fn difference(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._combine_inner(other, vm, SetCombineOperation::Difference) + } + + fn _combine_inner( + &self, + other: &PySetInner, + vm: &VirtualMachine, + op: SetCombineOperation, + ) -> PyResult { + let mut elements = HashMap::new(); + + for element in self.elements.iter() { + let value = other.contains(element.1.clone(), vm)?; + let should_add = match op { + SetCombineOperation::Intersection => objbool::get_value(&value), + SetCombineOperation::Difference => !objbool::get_value(&value), + }; + if should_add { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + Ok(PySetInner { elements }) + } + + fn symmetric_difference( + &self, + other: &PySetInner, + vm: &VirtualMachine, + ) -> PyResult { + let mut elements = HashMap::new(); + + for element in self.elements.iter() { + let value = other.contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + for element in other.elements.iter() { + let value = self.contains(element.1.clone(), vm)?; + if !objbool::get_value(&value) { + elements.insert(element.0.clone(), element.1.clone()); + } + } + + Ok(PySetInner { elements }) + } + + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + let items = self.elements.values().cloned().collect(); + let set_list = vm.ctx.new_list(items); + PyListIterator { + position: Cell::new(0), + list: set_list.downcast().unwrap(), + } + } + + fn repr(&self, vm: &VirtualMachine) -> PyResult { + let mut str_parts = vec![]; + for elem in self.elements.values() { + let part = vm.to_repr(elem)?; + str_parts.push(part.value.clone()); + } + + Ok(format!("{{{}}}", str_parts.join(", "))) + } + + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + insert_into_set(vm, &mut self.elements, &item) + } + + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn remove( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + value: &PyObjectRef, + ) -> PyResult { + match elements.remove(&key) { + None => { + let item_str = format!("{:?}", value); + Err(vm.new_key_error(item_str)) + } + Some(_) => Ok(vm.get_none()), + } + } + perform_action_with_hash(vm, &mut self.elements, &item, &remove) + } + + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn discard( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + _value: &PyObjectRef, + ) -> PyResult { + elements.remove(&key); + Ok(vm.get_none()) + } + perform_action_with_hash(vm, &mut self.elements, &item, &discard) + } + + fn clear(&mut self, vm: &VirtualMachine) -> PyResult { + self.elements.clear(); + Ok(vm.get_none()) + } + + fn pop(&mut self, vm: &VirtualMachine) -> PyResult { + let elements = &mut self.elements; + match elements.clone().keys().next() { + Some(key) => Ok(elements.remove(key).unwrap()), + None => Err(vm.new_key_error("pop from an empty set".to_string())), + } + } + + fn update(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut self.elements, &v)?; + } + Ok(vm.get_none()) + } + + fn combine_update_inner( + &mut self, + iterable: &PyObjectRef, + vm: &VirtualMachine, + op: SetCombineOperation, + ) -> PyResult { + let elements = &mut self.elements; + for element in elements.clone().iter() { + let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; + let should_remove = match op { + SetCombineOperation::Intersection => !objbool::get_value(&value), + SetCombineOperation::Difference => objbool::get_value(&value), + }; + if should_remove { + elements.remove(&element.0.clone()); + } + } + Ok(vm.get_none()) + } + + fn ixor(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let elements_original = self.elements.clone(); + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut self.elements, &v)?; + } + for element in elements_original.iter() { + let value = vm.call_method(&iterable, "__contains__", vec![element.1.clone()])?; + if objbool::get_value(&value) { + self.elements.remove(&element.0.clone()); + } + } + + Ok(vm.get_none()) + } +} + +impl PySetRef { + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.borrow().len() + } + fn copy(self, _vm: &VirtualMachine) -> PySet { + PySet { + inner: RefCell::new(self.inner.borrow().copy()), + } + } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().eq(&get_inner(vm, &other)?, vm) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().ge(&get_inner(vm, &other)?, vm) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().gt(&get_inner(vm, &other)?, vm) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().le(&get_inner(vm, &other)?, vm) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().lt(&get_inner(vm, &other)?, vm) + } + + fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new(self.inner.borrow().union(&get_inner(vm, &other)?, vm)?), + }, + PySet::class(vm), + None, + )) + } + + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .intersection(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .difference(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new( + self.inner + .borrow() + .symmetric_difference(&get_inner(vm, &other)?, vm)?, + ), + }, + PySet::class(vm), + None, + )) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.borrow().iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = self.inner.borrow(); + let s = if inner.len() == 0 { + format!("set()") + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + inner.repr(vm)? + } else { + format!("set(...)") + }; + Ok(vm.new_str(s)) + } + + fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().add(&item, vm) + } + + fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().remove(&item, vm) + } + + fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().discard(&item, vm) + } + + fn clear(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().clear(vm) + } + + fn pop(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().pop(vm) + } + + fn ior(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(self.as_object().clone()) + } + + fn update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn intersection_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Intersection, + )?; + Ok(vm.get_none()) + } + + fn iand(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Intersection, + )?; + Ok(self.as_object().clone()) + } + + fn difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Difference, + )?; + Ok(vm.get_none()) + } + + fn isub(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().combine_update_inner( + &iterable, + vm, + SetCombineOperation::Difference, + )?; + Ok(self.as_object().clone()) + } + + fn symmetric_difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().ixor(iterable, vm)?; + Ok(vm.get_none()) + } + + fn ixor(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().ixor(iterable, vm)?; + Ok(self.as_object().clone()) + } +} + +impl PyFrozenSetRef { + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } + fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { + PyFrozenSet { + inner: self.inner.copy(), + } + } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.eq(&get_inner(vm, &other)?, vm) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.ge(&get_inner(vm, &other)?, vm) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.gt(&get_inner(vm, &other)?, vm) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.le(&get_inner(vm, &other)?, vm) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.lt(&get_inner(vm, &other)?, vm) + } + + fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.union(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.intersection(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.difference(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self + .inner + .symmetric_difference(&get_inner(vm, &other)?, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = &self.inner; + let s = if inner.len() == 0 { + format!("frozenset()") + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + format!("frozenset({})", inner.repr(vm)?) + } else { + format!("frozenset(...)") + }; + Ok(vm.new_str(s)) + } +} + +fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { if let Some(set) = obj.payload::() { - return set.elements.borrow().clone(); + Ok(set.inner.borrow().clone()) } else if let Some(frozenset) = obj.payload::() { - return frozenset.elements.clone(); + Ok(frozenset.inner.clone()) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) } - panic!("Not frozenset or set"); } fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { @@ -82,14 +633,16 @@ fn create_set( if objtype::issubclass(&cls, &vm.ctx.set_type()) { Ok(PyObject::new( PySet { - elements: RefCell::new(elements), + inner: RefCell::new(PySetInner { elements: elements }), }, PySet::class(vm), None, )) } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { Ok(PyObject::new( - PyFrozenSet { elements: elements }, + PyFrozenSet { + inner: PySetInner { elements: elements }, + }, PyFrozenSet::class(vm), None, )) @@ -130,84 +683,6 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } -fn set_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.add called with: {:?}", args); - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (item, None)] - ); - match zelf.payload::() { - Some(set) => insert_into_set(vm, &mut set.elements.borrow_mut(), item), - _ => Err(vm.new_type_error("set.add is called with no item".to_string())), - } -} - -fn set_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.remove called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &remove) - } - _ => Err(vm.new_type_error("set.remove is called with no item".to_string())), - } -} - -fn set_discard(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.discard called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &discard) - } - None => Err(vm.new_type_error("set.discard is called with no item".to_string())), - } -} - -fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.clear called"); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - match s.payload::() { - Some(set) => { - set.elements.borrow_mut().clear(); - Ok(vm.get_none()) - } - None => Err(vm.new_type_error("".to_string())), - } -} - /* Create a new object of sub-type of set */ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { validate_set_or_frozenset(vm, cls.clone())?; @@ -227,367 +702,11 @@ fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMach create_set(vm, elements, cls.clone()) } -fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.len called with: {:?}", args); - arg_check!(vm, args, required = [(s, None)]); - validate_set_or_frozenset(vm, s.class())?; - let elements = get_elements(s); - Ok(vm.context().new_int(elements.len())) -} - -fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - trace!("set.copy called with: {:?}", obj); - validate_set_or_frozenset(vm, obj.class())?; - let elements = get_elements(&obj).clone(); - create_set(vm, elements, obj.class()) -} - -fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.set_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(o) { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("{{{}}}", str_parts.join(", ")) - } else { - "set(...)".to_string() - }; - Ok(vm.new_str(s)) -} - -pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(set, None), (needle, None)]); - validate_set_or_frozenset(vm, set.class())?; - for element in get_elements(set).iter() { - match vm._eq(needle.clone(), element.1.clone()) { - Ok(value) => { - if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), - } - } - - Ok(vm.new_bool(false)) -} - -fn set_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf != other }, - false, - ) -} - -fn set_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - false, - ) -} - -fn set_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - false, - ) -} - -fn set_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - true, - ) -} - -fn set_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - true, - ) -} - -fn set_compare_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - size_func: &Fn(usize, usize) -> bool, - swap: bool, -) -> PyResult { - arg_check!(vm, args, required = [(zelf, None), (other, None)]); - - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - - let get_zelf = |swap: bool| -> &PyObjectRef { - if swap { - other - } else { - zelf - } - }; - let get_other = |swap: bool| -> &PyObjectRef { - if swap { - zelf - } else { - other - } - }; - - let zelf_elements = get_elements(get_zelf(swap)); - let other_elements = get_elements(get_other(swap)); - if size_func(zelf_elements.len(), other_elements.len()) { - return Ok(vm.new_bool(false)); - } - for element in other_elements.iter() { - match vm.call_method(get_zelf(swap), "__contains__", vec![element.1.clone()]) { - Ok(value) => { - if !objbool::get_value(&value) { - return Ok(vm.new_bool(false)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), - } - } - Ok(vm.new_bool(true)) -} - -fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - - let mut elements = get_elements(&zelf).clone(); - elements.extend(get_elements(&other).clone()); - - create_set(vm, elements, zelf.class()) -} - -fn set_intersection(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Intersection) -} - -fn set_difference(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Difference) -} - -fn set_symmetric_difference( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - for element in get_elements(&other).iter() { - let value = vm.call_method(&zelf, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - enum SetCombineOperation { Intersection, Difference, } -fn set_combine_inner( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, - op: SetCombineOperation, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), - }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - -fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - - match s.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), - } - } - _ => Err(vm.new_type_error("".to_string())), - } -} - -fn set_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ior(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ior(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - } - _ => return Err(vm.new_type_error("set.update is called with no other".to_string())), - } - Ok(zelf.clone()) -} - -fn set_intersection_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection)?; - Ok(vm.get_none()) -} - -fn set_iand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection) -} - -fn set_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference)?; - Ok(vm.get_none()) -} - -fn set_isub(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference) -} - -fn set_combine_update_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - op: SetCombineOperation, -) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - Ok(zelf.clone()) -} - -fn set_symmetric_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ixor(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let elements_original = set.elements.borrow().clone(); - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - set.elements.borrow_mut().remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - - Ok(zelf.clone()) -} - -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type - let items = zelf.elements.borrow().values().cloned().collect(); - let set_list = vm.ctx.new_list(items); - PyListIterator { - position: Cell::new(0), - list: set_list.downcast().unwrap(), - } -} - -fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.frozenset_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "frozenset()".to_string() - } else { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("frozenset({{{}}})", str_parts.join(", ")) - }; - Ok(vm.new_str(s)) -} - fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } @@ -600,42 +719,42 @@ pub fn init(context: &PyContext) { Build an unordered collection of unique elements."; extend_class!(context, set_type, { - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), + "__contains__" => context.new_rustfunc(PySetRef::contains), + "__len__" => context.new_rustfunc(PySetRef::len), "__new__" => context.new_rustfunc(set_new), - "__repr__" => context.new_rustfunc(set_repr), "__hash__" => context.new_rustfunc(set_hash), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), + "__repr__" => context.new_rustfunc(PySetRef::repr), + "__eq__" => context.new_rustfunc(PySetRef::eq), + "__ge__" => context.new_rustfunc(PySetRef::ge), + "__gt__" => context.new_rustfunc(PySetRef::gt), + "__le__" => context.new_rustfunc(PySetRef::le), + "__lt__" => context.new_rustfunc(PySetRef::lt), + "issubset" => context.new_rustfunc(PySetRef::le), + "issuperset" => context.new_rustfunc(PySetRef::ge), + "union" => context.new_rustfunc(PySetRef::union), + "__or__" => context.new_rustfunc(PySetRef::union), + "intersection" => context.new_rustfunc(PySetRef::intersection), + "__and__" => context.new_rustfunc(PySetRef::intersection), + "difference" => context.new_rustfunc(PySetRef::difference), + "__sub__" => context.new_rustfunc(PySetRef::difference), + "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), "__doc__" => context.new_str(set_doc.to_string()), - "add" => context.new_rustfunc(set_add), - "remove" => context.new_rustfunc(set_remove), - "discard" => context.new_rustfunc(set_discard), - "clear" => context.new_rustfunc(set_clear), - "copy" => context.new_rustfunc(set_copy), - "pop" => context.new_rustfunc(set_pop), - "update" => context.new_rustfunc(set_update), - "__ior__" => context.new_rustfunc(set_ior), - "intersection_update" => context.new_rustfunc(set_intersection_update), - "__iand__" => context.new_rustfunc(set_iand), - "difference_update" => context.new_rustfunc(set_difference_update), - "__isub__" => context.new_rustfunc(set_isub), - "symmetric_difference_update" => context.new_rustfunc(set_symmetric_difference_update), - "__ixor__" => context.new_rustfunc(set_ixor), - "__iter__" => context.new_rustfunc(set_iter) + "add" => context.new_rustfunc(PySetRef::add), + "remove" => context.new_rustfunc(PySetRef::remove), + "discard" => context.new_rustfunc(PySetRef::discard), + "clear" => context.new_rustfunc(PySetRef::clear), + "copy" => context.new_rustfunc(PySetRef::copy), + "pop" => context.new_rustfunc(PySetRef::pop), + "update" => context.new_rustfunc(PySetRef::update), + "__ior__" => context.new_rustfunc(PySetRef::ior), + "intersection_update" => context.new_rustfunc(PySetRef::intersection_update), + "__iand__" => context.new_rustfunc(PySetRef::iand), + "difference_update" => context.new_rustfunc(PySetRef::difference_update), + "__isub__" => context.new_rustfunc(PySetRef::isub), + "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), + "__ixor__" => context.new_rustfunc(PySetRef::ixor), + "__iter__" => context.new_rustfunc(PySetRef::iter) }); let frozenset_type = &context.frozenset_type; @@ -646,25 +765,26 @@ pub fn init(context: &PyContext) { extend_class!(context, frozenset_type, { "__new__" => context.new_rustfunc(set_new), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), + "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), + "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), + "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), + "__le__" => context.new_rustfunc(PyFrozenSetRef::le), + "__lt__" => context.new_rustfunc(PyFrozenSetRef::lt), + "issubset" => context.new_rustfunc(PyFrozenSetRef::le), + "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), + "union" => context.new_rustfunc(PyFrozenSetRef::union), + "__or__" => context.new_rustfunc(PyFrozenSetRef::union), + "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), + "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), + "difference" => context.new_rustfunc(PyFrozenSetRef::difference), + "__sub__" => context.new_rustfunc(PyFrozenSetRef::difference), + "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), + "__len__" => context.new_rustfunc(PyFrozenSetRef::len), "__doc__" => context.new_str(frozenset_doc.to_string()), - "__repr__" => context.new_rustfunc(frozenset_repr), - "copy" => context.new_rustfunc(set_copy) + "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), + "copy" => context.new_rustfunc(PyFrozenSetRef::copy), + "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter) }); } From a685216f29ecc75e2bc3f9637a42e5fe878c5b37 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 4 Apr 2019 20:47:29 +0300 Subject: [PATCH 071/223] Use new arg style for set and frozenset new --- vm/src/obj/objset.rs | 103 +++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 58 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index eaa0cd09c..fd31044aa 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -15,7 +15,6 @@ use super::objbool; use super::objint; use super::objiter; use super::objlist::PyListIterator; -use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] @@ -62,6 +61,22 @@ struct PySetInner { } impl PySetInner { + fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + let elements: HashMap = match iterable { + OptionalArg::Missing => HashMap::new(), + OptionalArg::Present(iterable) => { + let mut elements = HashMap::new(); + let iterator = objiter::get_iter(vm, &iterable)?; + while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { + insert_into_set(vm, &mut elements, &v)?; + } + elements + } + }; + + Ok(PySetInner { elements }) + } + fn len(&self) -> usize { self.elements.len() } @@ -334,14 +349,27 @@ impl PySetInner { } impl PySetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PySet { + inner: RefCell::new(PySetInner::new(iterable, vm)?), + } + .into_ref_with_type(vm, cls) + } + fn len(self, _vm: &VirtualMachine) -> usize { self.inner.borrow().len() } + fn copy(self, _vm: &VirtualMachine) -> PySet { PySet { inner: RefCell::new(self.inner.borrow().copy()), } } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow().contains(needle, vm) } @@ -512,14 +540,27 @@ impl PySetRef { } impl PyFrozenSetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyFrozenSet { + inner: PySetInner::new(iterable, vm)?, + } + .into_ref_with_type(vm, cls) + } + fn len(self, _vm: &VirtualMachine) -> usize { self.inner.len() } + fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { PyFrozenSet { inner: self.inner.copy(), } } + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.contains(needle, vm) } @@ -616,41 +657,6 @@ fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { } } -fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { - if !(objtype::issubclass(&cls, &vm.ctx.set_type()) - || objtype::issubclass(&cls, &vm.ctx.frozenset_type())) - { - return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))); - } - Ok(()) -} - -fn create_set( - vm: &VirtualMachine, - elements: HashMap, - cls: PyClassRef, -) -> PyResult { - if objtype::issubclass(&cls, &vm.ctx.set_type()) { - Ok(PyObject::new( - PySet { - inner: RefCell::new(PySetInner { elements: elements }), - }, - PySet::class(vm), - None, - )) - } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { - Ok(PyObject::new( - PyFrozenSet { - inner: PySetInner { elements: elements }, - }, - PyFrozenSet::class(vm), - None, - )) - } else { - Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) - } -} - fn perform_action_with_hash( vm: &VirtualMachine, elements: &mut HashMap, @@ -683,25 +689,6 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } -/* Create a new object of sub-type of set */ -fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, cls.clone())?; - - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; - } - elements - } - }; - - create_set(vm, elements, cls.clone()) -} - enum SetCombineOperation { Intersection, Difference, @@ -719,10 +706,10 @@ pub fn init(context: &PyContext) { Build an unordered collection of unique elements."; extend_class!(context, set_type, { + "__hash__" => context.new_rustfunc(set_hash), "__contains__" => context.new_rustfunc(PySetRef::contains), "__len__" => context.new_rustfunc(PySetRef::len), - "__new__" => context.new_rustfunc(set_new), - "__hash__" => context.new_rustfunc(set_hash), + "__new__" => context.new_rustfunc(PySetRef::new), "__repr__" => context.new_rustfunc(PySetRef::repr), "__eq__" => context.new_rustfunc(PySetRef::eq), "__ge__" => context.new_rustfunc(PySetRef::ge), @@ -764,7 +751,7 @@ pub fn init(context: &PyContext) { Build an immutable unordered collection of unique elements."; extend_class!(context, frozenset_type, { - "__new__" => context.new_rustfunc(set_new), + "__new__" => context.new_rustfunc(PyFrozenSetRef::new), "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), From 2a3a88c2f2cf0f13f0ed89843f19680f43b8d403 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 4 Apr 2019 21:23:00 +0300 Subject: [PATCH 072/223] Use match_class macro to avoid unwanted clones --- vm/src/obj/objset.rs | 135 ++++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 45 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index fd31044aa..6c67312fb 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -375,29 +375,53 @@ impl PySetRef { } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().eq(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().ge(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().gt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().le(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().lt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(self.inner.borrow().union(&get_inner(vm, &other)?, vm)?), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().union(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().union(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -407,11 +431,11 @@ impl PySetRef { fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .intersection(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().intersection(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().intersection(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -421,11 +445,11 @@ impl PySetRef { fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .difference(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -435,11 +459,11 @@ impl PySetRef { fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new( - self.inner - .borrow() - .symmetric_difference(&get_inner(vm, &other)?, vm)?, - ), + inner: RefCell::new(match_class!(other, + set @ PySet => self.inner.borrow().symmetric_difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.borrow().symmetric_difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + )), }, PySet::class(vm), None, @@ -566,29 +590,53 @@ impl PyFrozenSetRef { } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.eq(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.ge(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.gt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.le(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.lt(&get_inner(vm, &other)?, vm) + match_class!(other, + set @ PySet => self.inner.lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) } fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.union(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.union(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.union(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -598,7 +646,11 @@ impl PyFrozenSetRef { fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.intersection(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.intersection(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.intersection(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -608,7 +660,11 @@ impl PyFrozenSetRef { fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self.inner.difference(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -618,9 +674,11 @@ impl PyFrozenSetRef { fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: self - .inner - .symmetric_difference(&get_inner(vm, &other)?, vm)?, + inner: match_class!(other, + set @ PySet => self.inner.symmetric_difference(&set.inner.borrow(), vm)?, + frozen @ PyFrozenSet => self.inner.symmetric_difference(&frozen.inner, vm)?, + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ), }, PyFrozenSet::class(vm), None, @@ -644,19 +702,6 @@ impl PyFrozenSetRef { } } -fn get_inner(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - if let Some(set) = obj.payload::() { - Ok(set.inner.borrow().clone()) - } else if let Some(frozenset) = obj.payload::() { - Ok(frozenset.inner.clone()) - } else { - Err(vm.new_type_error(format!( - "{} is not a subtype of set or frozenset", - obj.class() - ))) - } -} - fn perform_action_with_hash( vm: &VirtualMachine, elements: &mut HashMap, From ab3d004c4227d79ada563c72a65cb2ac569fd47c Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Fri, 5 Apr 2019 21:59:20 +0200 Subject: [PATCH 073/223] Fix syntax for float literals, statements seperated by semicolons and starargs after keyword arguments. --- parser/src/ast.rs | 2 +- parser/src/lexer.rs | 22 ++++++++++-------- parser/src/parser.rs | 18 +++++++-------- parser/src/python.lalrpop | 38 ++++++++++++++++++++------------ tests/snippets/comprehensions.py | 5 +++++ tests/snippets/floats.py | 4 ++++ tests/snippets/function_args.py | 9 ++++++++ tests/snippets/statements.py | 9 ++++++++ vm/src/compile.rs | 16 ++++++++------ 9 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 tests/snippets/statements.py diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 21c8dd736..9fbe8153e 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -16,7 +16,7 @@ pub struct Node { #[derive(Debug, PartialEq)] pub enum Top { Program(Program), - Statement(LocatedStatement), + Statement(Vec), Expression(Expression), } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index edf21c973..ebdb9555d 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1044,16 +1044,20 @@ where return Some(Ok((tok_start, Tok::Comma, tok_end))); } '.' => { - let tok_start = self.get_pos(); - self.next_char(); - if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { - self.next_char(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + if let Some('0'..='9') = self.chr1 { + return Some(self.lex_number()); } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Dot, tok_end))); + let tok_start = self.get_pos(); + self.next_char(); + if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { + self.next_char(); + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Dot, tok_end))); + } } } '\n' => { diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 4c37f2842..dd58bd11d 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result { do_lalr_parsing!(source, Program, StartProgram) } -pub fn parse_statement(source: &str) -> Result { +pub fn parse_statement(source: &str) -> Result, ParseError> { do_lalr_parsing!(source, Statement, StartStatement) } @@ -180,7 +180,7 @@ mod tests { let parse_ast = parse_statement(&source).unwrap(); assert_eq!( parse_ast, - ast::LocatedStatement { + vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::If { test: ast::Expression::Number { @@ -229,7 +229,7 @@ mod tests { } },]), } - } + }] ); } @@ -239,7 +239,7 @@ mod tests { let parse_ast = parse_statement(&source); assert_eq!( parse_ast, - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Expression { expression: ast::Expression::Lambda { @@ -271,7 +271,7 @@ mod tests { }) } } - }) + }]) ) } @@ -281,7 +281,7 @@ mod tests { assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Assign { targets: vec![ast::Expression::Tuple { @@ -309,7 +309,7 @@ mod tests { ] } } - }) + }]) ) } @@ -320,7 +320,7 @@ mod tests { ); assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::ClassDef { name: String::from("Foo"), @@ -393,7 +393,7 @@ mod tests { ], decorator_list: vec![], } - }) + }]) ) } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 305f97c10..a9663943f 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -24,29 +24,32 @@ pub Top: ast::Top = { Program: ast::Program = { => ast::Program { - statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) + statements: Vec::from_iter(lines.into_iter().flatten()) }, }; // A file line either has a declaration, or an empty newline: -FileLine: Option = { - => Some(s), - "\n" => None, +FileLine: Vec = { + Statement, + "\n" => vec![], }; Suite: Vec = { - => vec![s], - "\n" indent dedent => s, -}; - -Statement: ast::LocatedStatement = { SimpleStatement, - CompoundStatement, + "\n" indent dedent => s.into_iter().flatten().collect(), }; -SimpleStatement: ast::LocatedStatement = { - "\n" => s, - ";" => s, +Statement: Vec = { + SimpleStatement, + => vec![s], +}; + +SimpleStatement: Vec = { + ";"? "\n" => { + let mut statements = vec![s1]; + statements.extend(s2.into_iter().map(|e| e.1)); + statements + } }; SmallStatement: ast::LocatedStatement = { @@ -899,7 +902,14 @@ ArgumentList: (Vec, Vec) = { keywords.push(ast::Keyword { name: n, value: value }); }, None => { - if keywords.len() > 0 { + // Allow starred args after keyword arguments. + let is_starred = if let ast::Expression::Starred { .. } = &value { + true + } else { + false + }; + + if keywords.len() > 0 && !is_starred { panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); diff --git a/tests/snippets/comprehensions.py b/tests/snippets/comprehensions.py index 14c545de3..132f3cdcb 100644 --- a/tests/snippets/comprehensions.py +++ b/tests/snippets/comprehensions.py @@ -22,3 +22,8 @@ assert u['1'] == -1 y = [a+2 for a in x if a % 2] print(y) assert y == [3, 5] + +z = [(9,), (10,)] +w = [x for x, in z] +assert w == [9, 10] + diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 6675615c5..08e7d4b2b 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -106,3 +106,7 @@ assert_raises(OverflowError, float('inf').as_integer_ratio) assert_raises(OverflowError, float('-inf').as_integer_ratio) assert_raises(ValueError, float('nan').as_integer_ratio) +# Test special case for lexer, float starts with a dot: +a = .5 +assert a == 0.5 + diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 745648f9d..884cb47df 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -42,3 +42,12 @@ va2(5, *x[1:]) x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) + +# star arg after keyword args: +def fubar(x, y, obj=None): + assert x == 4 + assert y == 5 + assert obj == 6 + +rest = [4, 5] +fubar(obj=6, *rest) diff --git a/tests/snippets/statements.py b/tests/snippets/statements.py new file mode 100644 index 000000000..1921363f5 --- /dev/null +++ b/tests/snippets/statements.py @@ -0,0 +1,9 @@ + +# Test several statement types + +# small ones, seperated by ';': + +if True: + 5;4; + b=4 + b; diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 88818f52f..5802d9e30 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -129,15 +129,17 @@ impl Compiler { // Compile statement in eval mode: fn compile_statement_eval( &mut self, - statement: &ast::LocatedStatement, + statements: &[ast::LocatedStatement], ) -> Result<(), CompileError> { - if let ast::Statement::Expression { ref expression } = statement.node { - self.compile_expression(expression)?; - self.emit(Instruction::ReturnValue); - Ok(()) - } else { - Err(CompileError::ExpectExpr) + for statement in statements { + if let ast::Statement::Expression { ref expression } = statement.node { + self.compile_expression(expression)?; + } else { + return Err(CompileError::ExpectExpr); + } } + self.emit(Instruction::ReturnValue); + Ok(()) } fn compile_statements( From dda0d561ac8f23da8f16951bc33891630e0be955 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 10:43:39 +1300 Subject: [PATCH 074/223] Fix order of dict literal --- Cargo.lock | 1 + tests/snippets/dict.py | 6 ++++++ vm/Cargo.toml | 1 + vm/src/frame.rs | 14 ++++++++------ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36e313e26..864e90ce5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,6 +809,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 5c5ef62b1..2d369465c 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -80,3 +80,9 @@ x[Hashable(8)] = 19 x[Hashable(19,8)] = 1 assert x[Hashable(8)] == 1 assert len(x) == 2 + +assert list({'a': 2, 'b': 10}) == ['a', 'b'] +x = {} +x['a'] = 2 +x['b'] = 10 +assert list(x) == ['a', 'b'] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e1b1cc9ff..ef1a9cbc3 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -26,3 +26,4 @@ caseless = "0.2.1" unicode-segmentation = "1.2.1" lazy_static = "^1.0.1" lexical = "2.0.0" +itertools = "^0.8.0" diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 858b33a08..2f525afdd 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -25,6 +25,7 @@ use crate::pyobject::{ TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use itertools::Itertools; /* * So a scope is a linked list of scopes. @@ -386,9 +387,8 @@ impl Frame { } bytecode::Instruction::BuildMap { size, unpack } => { let map_obj = vm.ctx.new_dict(); - for _x in 0..*size { - let obj = self.pop_value(); - if *unpack { + if *unpack { + for obj in self.pop_multiple(*size) { // Take all key-value pairs from the dict: let dict: PyDictRef = obj.downcast().expect("Need a dictionary to build a map."); @@ -396,11 +396,13 @@ impl Frame { for (key, value) in dict_elements.iter() { map_obj.set_item(key.clone(), value.clone(), vm); } - } else { - let key = self.pop_value(); - map_obj.set_item(key, obj, vm) + } + } else { + for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { + map_obj.set_item(key, value, vm) } } + self.push_value(map_obj.into_object()); Ok(None) } From a7d5b0e9ac1ecd8153d5ceeaa796340755ccb692 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 6 Apr 2019 01:44:00 +0200 Subject: [PATCH 075/223] start --- vm/src/obj/mod.rs | 1 + vm/src/obj/objbyteinner.rs | 9 +++ vm/src/obj/objbytes.rs | 151 +++++++++++++++++++++---------------- 3 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 vm/src/obj/objbyteinner.rs diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index 0419d827f..c824e726d 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -3,6 +3,7 @@ pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; +pub mod objbyteinner; pub mod objbytes; pub mod objclassmethod; pub mod objcode; diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs new file mode 100644 index 000000000..0607621f1 --- /dev/null +++ b/vm/src/obj/objbyteinner.rs @@ -0,0 +1,9 @@ +#[derive(Debug, Default, Clone)] +pub struct PyByteInner { + pub elements: Vec, +} +impl PyByteInner { + pub fn new(data: Vec) -> Self { + PyByteInner { elements: data } + } +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f24c9273c..8f43cd05f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,22 +6,37 @@ use std::ops::Deref; use num_traits::ToPrimitive; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; use crate::vm::VirtualMachine; +use super::objbyteinner::PyByteInner; use super::objint; use super::objiter; use super::objtype::PyClassRef; +/// "bytes(iterable_of_ints) -> bytes\n\ +/// bytes(string, encoding[, errors]) -> bytes\n\ +/// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ +/// bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ +/// bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytes", __inside_vm)] #[derive(Debug)] pub struct PyBytes { - value: Vec, + inner: PyByteInner, } type PyBytesRef = PyRef; impl PyBytes { - pub fn new(data: Vec) -> Self { - PyBytes { value: data } + pub fn new(elements: Vec) -> Self { + PyBytes { + inner: PyByteInner { elements }, + } } } @@ -29,7 +44,7 @@ impl Deref for PyBytes { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.value + &self.inner.elements } } @@ -42,62 +57,74 @@ impl PyValue for PyBytes { // Binary data support // Fill bytes class methods: -pub fn init(context: &PyContext) { - let bytes_doc = - "bytes(iterable_of_ints) -> bytes\n\ - bytes(string, encoding[, errors]) -> bytes\n\ - bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ - bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ - bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - any object implementing the buffer API.\n \ - - an integer"; - extend_class!(context, &context.bytes_type, { - "__new__" => context.new_rustfunc(bytes_new), - "__eq__" => context.new_rustfunc(PyBytesRef::eq), - "__lt__" => context.new_rustfunc(PyBytesRef::lt), - "__le__" => context.new_rustfunc(PyBytesRef::le), - "__gt__" => context.new_rustfunc(PyBytesRef::gt), - "__ge__" => context.new_rustfunc(PyBytesRef::ge), - "__hash__" => context.new_rustfunc(PyBytesRef::hash), - "__repr__" => context.new_rustfunc(PyBytesRef::repr), - "__len__" => context.new_rustfunc(PyBytesRef::len), - "__iter__" => context.new_rustfunc(PyBytesRef::iter), - "__doc__" => context.new_str(bytes_doc.to_string()) - }); - - let bytesiterator_type = &context.bytesiterator_type; - extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), - }); +pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { + &obj.payload::().unwrap().inner.elements } -fn bytes_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); - } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; - - PyBytes::new(value).into_ref_with_type(vm, cls) +// pub fn init(context: &PyContext) { +// let bytes_doc = +pub fn init(ctx: &PyContext) { + PyBytesRef::extend_class(ctx, &ctx.bytes_type); } +//extend_class!(context, &context.bytes_type, { +//"__new__" => context.new_rustfunc(bytes_new), +/* "__eq__" => context.new_rustfunc(PyBytesRef::eq), +"__lt__" => context.new_rustfunc(PyBytesRef::lt), +"__le__" => context.new_rustfunc(PyBytesRef::le), +"__gt__" => context.new_rustfunc(PyBytesRef::gt), +"__ge__" => context.new_rustfunc(PyBytesRef::ge), +"__hash__" => context.new_rustfunc(PyBytesRef::hash), +"__repr__" => context.new_rustfunc(PyBytesRef::repr), +"__len__" => context.new_rustfunc(PyBytesRef::len), +"__iter__" => context.new_rustfunc(PyBytesRef::iter), +"__doc__" => context.new_str(bytes_doc.to_string())*/ +// }); +/* let bytesiterator_type = &context.bytesiterator_type; +extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), +});*/ +//} +#[pyimpl(__inside_vm)] impl PyBytesRef { + #[pymethod(name = "__new__")] + fn bytes_new( + cls: PyClassRef, + val_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // Create bytes data: + let value = if let OptionalArg::Present(ival) = val_option { + let elements = vm.extract_elements(&ival)?; + let mut data_bytes = vec![]; + for elem in elements.iter() { + let v = objint::to_int(vm, elem, 10)?; + data_bytes.push(v.to_u8().unwrap()); + } + data_bytes + // return Err(vm.new_type_error("Cannot construct bytes".to_string())); + } else { + vec![] + }; + + PyBytes::new(value).into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__repr__")] + fn repr(self, _vm: &VirtualMachine) -> String { + // TODO: don't just unwrap + let data = self.inner.elements.clone(); + format!("b'{:?}'", data) + } + + #[pymethod(name = "__len__")] + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.elements.len() + } +} +/* fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if let Ok(other) = other.downcast::() { vm.ctx.new_bool(self.value == other.value) @@ -138,9 +165,6 @@ impl PyBytesRef { } } - fn len(self, _vm: &VirtualMachine) -> usize { - self.value.len() - } fn hash(self, _vm: &VirtualMachine) -> u64 { let mut hasher = DefaultHasher::new(); @@ -148,11 +172,6 @@ impl PyBytesRef { hasher.finish() } - fn repr(self, _vm: &VirtualMachine) -> String { - // TODO: don't just unwrap - let data = String::from_utf8(self.value.clone()).unwrap(); - format!("b'{}'", data) - } fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { PyBytesIterator { @@ -162,9 +181,7 @@ impl PyBytesRef { } } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - &obj.payload::().unwrap().value -} + #[derive(Debug)] pub struct PyBytesIterator { @@ -194,4 +211,4 @@ impl PyBytesIteratorRef { fn iter(self, _vm: &VirtualMachine) -> Self { self } -} +}*/ From 7fffc572559d5d2b889af747f083f5dad16645f9 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 12:46:48 +1300 Subject: [PATCH 076/223] Fix decorator on functions with defaults --- tests/snippets/decorators.py | 10 ++++++++++ vm/src/compile.rs | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/snippets/decorators.py b/tests/snippets/decorators.py index fcc196f93..2633308f1 100644 --- a/tests/snippets/decorators.py +++ b/tests/snippets/decorators.py @@ -15,6 +15,16 @@ c = add(10, 3) assert c == 14 +@logged +def add3(a, b, c=2): + return a + b + c + + +d = add3(12, 5) + +assert d == 20 + + def f(func): return lambda: 42 class A: pass a = A() diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 88818f52f..d7f774b83 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -585,6 +585,9 @@ impl Compiler { let was_in_function_def = self.in_function_def; self.in_loop = false; self.in_function_def = true; + + self.prepare_decorators(decorator_list)?; + let mut flags = self.enter_function(name, args)?; let (new_body, doc_str) = get_doc(body); @@ -598,8 +601,6 @@ impl Compiler { self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - self.prepare_decorators(decorator_list)?; - // Prepare type annotations: let mut num_annotations = 0; From 909d94ec233749d9bfc42fdab830e08eb3cfa4fd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Fri, 5 Apr 2019 20:25:53 -0600 Subject: [PATCH 077/223] impl and use SequenceIndex for list.__delitem__ - removal of del_item - all del function impl on PyListRef - stepped deletion now loops only over the range insted of the whole list --- tests/snippets/list.py | 4 + vm/src/obj/objlist.rs | 176 ++++++++++++++++++++++++++++++++++++-- vm/src/obj/objsequence.rs | 152 +++----------------------------- 3 files changed, 188 insertions(+), 144 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6efa52cc0..3d9743c0b 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -190,6 +190,10 @@ x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] del x[-5:] assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] +x = list(range(12)) +del x[10:2:-2] +assert x == [0,1,2,3,5,7,9,11] + def bad_del_1(): del ['a', 'b']['a'] assert_raises(TypeError, bad_del_1) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index f3e812615..60c14b107 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,9 +1,10 @@ use std::cell::{Cell, RefCell}; use std::fmt; -use std::ops::DerefMut; +use std::ops::Range; -use num_traits::ToPrimitive; +use num_bigint::BigInt; +use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; @@ -13,9 +14,10 @@ use super::objbool; use super::objint; use super::objiter; use super::objsequence::{ - del_item, get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, - seq_mul, PySliceableSequence, + get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, + PySliceableSequence, SequenceIndex, }; +use super::objslice::PySliceRef; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -46,6 +48,54 @@ impl PyValue for PyList { } } +impl PyList { + pub fn get_len(&self) -> usize { + self.elements.borrow().len() + } + + pub fn get_pos(&self, p: i32) -> Option { + // convert a (potentially negative) positon into a real index + if p < 0 { + if -p as usize > self.get_len() { + None + } else { + Some(self.get_len() - ((-p) as usize)) + } + } else if p as usize >= self.get_len() { + None + } else { + Some(p as usize) + } + } + + pub fn get_slice_pos(&self, slice_pos: &BigInt) -> usize { + if let Some(pos) = slice_pos.to_i32() { + if let Some(index) = self.get_pos(pos) { + // within bounds + return index; + } + } + + if slice_pos.is_negative() { + // slice past start bound, round to start + 0 + } else { + // slice past end bound, round to end + self.get_len() + } + } + + pub fn get_slice_range(&self, start: &Option, stop: &Option) -> Range { + let start = start.as_ref().map(|x| self.get_slice_pos(x)).unwrap_or(0); + let stop = stop + .as_ref() + .map(|x| self.get_slice_pos(x)) + .unwrap_or_else(|| self.get_len()); + + start..stop + } +} + pub type PyListRef = PyRef; impl PyListRef { @@ -313,9 +363,123 @@ impl PyListRef { } } - fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn delitem(self, subscript: SequenceIndex, vm: &VirtualMachine) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.delindex(index, vm), + SequenceIndex::Slice(slice) => self.delslice(slice, vm), + } + } + + fn delindex(self, index: i32, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut().remove(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + + fn delslice(self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { + let start = &slice.start; + let stop = &slice.stop; + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&start, &stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| x + 1); + let stop = stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice_reverse(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } + } + + fn _del_slice(self, range: Range) { + self.elements.borrow_mut().drain(range); + } + + fn _del_stepped_slice(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; let mut elements = self.elements.borrow_mut(); - del_item(vm, self.as_object(), elements.deref_mut(), key) + let mut indexes = range.clone().step_by(step).peekable(); + + for i in range.clone() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + elements.swap(i - deleted, i); + } + } + // then drain (the values to delete should now be contiguous at the end of the range) + elements.drain((range.end - deleted)..range.end); + } + + fn _del_stepped_slice_reverse(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().rev().step_by(step).peekable(); + + for i in range.clone().rev() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards back + elements.swap(i + deleted, i); + } + } + // then drain (the values to delete should now be contiguous at teh start of the range) + elements.drain(range.start..(range.start + deleted)); } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 9593e57fe..14f9c5c93 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut, Range}; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; -use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; use super::objbool; @@ -137,104 +137,21 @@ impl PySliceableSequence for Vec { } } -pub trait PySliceableSequenceMut: PySliceableSequence { - //fn set_slice(&self, range: Range, seq: T) -> Self; - //fn set_slice_reverse(&self, range: Range, seq: T) -> Self; - //fn set_stepped_slice(&self, range: Range, step: usize, seq: T) -> Self; - //fn set_stepped_slice_reverse(&self, range: Range, step: usize, seq: T) -> Self; - - fn del_slice(&mut self, range: Range); - fn del_stepped_slice(&mut self, range: Range, step: usize); - fn del_index(&mut self, index: usize); - - fn del_slice_items(&mut self, vm: &VirtualMachine, slice: &PySliceRef) -> PyResult { - - let start = &slice.start; - let stop = &slice.stop; - - let step = slice.step.clone().unwrap_or_else(BigInt::one); - - if step.is_zero() { - Err(vm.new_value_error("slice step cannot be zero".to_string())) - } else if step.is_positive() { - let range = self.get_slice_range(start, stop); - if range.start < range.end { - #[allow(clippy::range_plus_one)] - match step.to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.start..range.start + 1); - Ok(vm.get_none()) - } - } - } else { - // no del to do - Ok(vm.get_none()) - } - } else { - // calculate the range for the reverse slice, first the bounds needs to be made - // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); - let range = self.get_slice_range(&stop, &start); - if range.start < range.end { - match (-step).to_i32() { - Some(1) => { - self.del_slice(range); - Ok(vm.get_none()) - } - Some(num) => { - self.del_stepped_slice(range, num as usize); - Ok(vm.get_none()) - } - None => { - self.del_slice(range.end - 1..range.end); - Ok(vm.get_none()) - } - } - } else { - // no del to do - Ok(vm.get_none()) - } - } - - } +pub enum SequenceIndex { + Int(i32), + Slice(PySliceRef), } -impl PySliceableSequenceMut for Vec { - fn del_slice(&mut self, range: Range) { - self.drain(range); - } - - fn del_stepped_slice(&mut self, range: Range, step: usize) { - // no easy way to delete steped indexes so heres what we'll do - let len = self.len(); - let mut deleted = 0; - let mut indexes = range.step_by(step).peekable(); - for i in 0..len { - // is this an index to delete? - if indexes.peek() == Some(&i) { - // record and move on - indexes.next(); - deleted += 1; - } else { - // swap towards front - self.swap(i - deleted, i); - } - } - // then truncate the vec (the values to delete should now be the last elements and thus choped off) - self.truncate(len - deleted); - } - - fn del_index(&mut self, index: usize) { - self.remove(index); +impl TryFromObject for SequenceIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(SequenceIndex::Int(i32::try_from_object(vm, i.into_object())?)), + s @ PySlice => Ok(SequenceIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) } } @@ -284,47 +201,6 @@ pub fn get_item( } } -pub fn del_item( - vm: &VirtualMachine, - sequence: &PyObjectRef, - elements: &mut T, - subscript: PyObjectRef, -) -> PyResult { - if let Some(i) = subscript.payload::() { - return match i.as_bigint().to_i32() { - Some(value) => { - if let Some(pos_index) = elements.get_pos(value) { - elements.del_index(pos_index); - Ok(vm.get_none()) - } else { - Err(vm.new_index_error("Index out of bounds!".to_string())) - } - } - None => { - Err(vm.new_index_error("cannot fit 'int' into an index-sized integer".to_string())) - } - }; - } - - if subscript.payload::().is_some() { - if sequence.payload::().is_some() { - if let Ok(slice) = subscript.downcast::() { - elements.del_slice_items(vm, &slice) - } else { - panic!("PySlice is not a slice? this should not be") - } - - } else { - panic!("sequence del_item called for non-mutable-sequence") - } - } else { - Err(vm.new_type_error(format!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - sequence, subscript - ))) - } -} - pub fn seq_equal( vm: &VirtualMachine, zelf: &[PyObjectRef], From 157204c61ad6f80503c413245019e1a4ffd88054 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 5 Apr 2019 23:24:26 -0500 Subject: [PATCH 078/223] Clean up the wasm crate --- derive/src/from_args.rs | 23 +++--- wasm/lib/src/browser_module.rs | 123 +++++++++++++++------------------ wasm/lib/src/convert.rs | 8 +-- wasm/lib/src/vm_class.rs | 58 +++++----------- 4 files changed, 89 insertions(+), 123 deletions(-) diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs index 7845b2880..040ebf36c 100644 --- a/derive/src/from_args.rs +++ b/derive/src/from_args.rs @@ -109,7 +109,7 @@ impl ArgAttribute { } } -fn generate_field(field: &Field) -> TokenStream2 { +fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { let mut pyarg_attrs = field .attrs .iter() @@ -132,7 +132,7 @@ fn generate_field(field: &Field) -> TokenStream2 { let name = &field.ident; let middle = quote! { - .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + .map(|x| #rp_path::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? }; let ending = if let Some(default) = attr.default { quote! { @@ -140,16 +140,16 @@ fn generate_field(field: &Field) -> TokenStream2 { } } else if attr.optional { quote! { - .map(crate::function::OptionalArg::Present) - .unwrap_or(crate::function::OptionalArg::Missing) + .map(#rp_path::function::OptionalArg::Present) + .unwrap_or(#rp_path::function::OptionalArg::Missing) } } else { let err = match attr.kind { ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - crate::function::ArgumentError::TooFewArgs + #rp_path::function::ArgumentError::TooFewArgs }, ParameterKind::KeywordOnly => quote! { - crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + #rp_path::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) }, }; quote! { @@ -181,7 +181,10 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { let fields = match input.data { Data::Struct(ref data) => { match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(generate_field), + Fields::Named(ref fields) => fields + .named + .iter() + .map(|field| generate_field(field, &rp_path)), Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message } } @@ -192,9 +195,9 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { quote! { impl #rp_path::function::FromArgs for #name { fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { + vm: &#rp_path::VirtualMachine, + args: &mut #rp_path::function::PyFuncArgs + ) -> Result { Ok(#name { #(#fields)* }) } } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 6f7c31964..607cfc886 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,16 +7,13 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ - objdict::PyDictRef, - objfunction::PyFunction, - objint, - objstr::{self, PyStringRef}, + objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; -use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; +use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; enum FetchResponseFormat { Json, @@ -42,25 +39,38 @@ impl FetchResponseFormat { } } -fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); +#[derive(FromArgs)] +struct FetchArgs { + #[pyarg(keyword_only, default = "None")] + response_format: Option, + #[pyarg(keyword_only, default = "None")] + method: Option, + #[pyarg(keyword_only, default = "None")] + headers: Option, + #[pyarg(keyword_only, default = "None")] + body: Option, + #[pyarg(keyword_only, default = "None")] + content_type: Option, +} - let response_format = - args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; - let method = args.get_optional_kwarg_with_type("method", vm.ctx.str_type(), vm)?; - let headers = args.get_optional_kwarg_with_type("headers", vm.ctx.dict_type(), vm)?; - let body = args.get_optional_kwarg("body"); - let content_type = args.get_optional_kwarg_with_type("content_type", vm.ctx.str_type(), vm)?; +fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyResult { + let FetchArgs { + response_format, + method, + headers, + body, + content_type, + } = args; let response_format = match response_format { - Some(s) => FetchResponseFormat::from_str(vm, &objstr::get_value(&s))?, + Some(s) => FetchResponseFormat::from_str(vm, s.as_str())?, None => FetchResponseFormat::Text, }; let mut opts = web_sys::RequestInit::new(); match method { - Some(s) => opts.method(&objstr::get_value(&s)), + Some(s) => opts.method(s.as_str()), None => opts.method("GET"), }; @@ -68,16 +78,15 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { opts.body(Some(&convert::py_to_js(vm, body))); } - let request = web_sys::Request::new_with_str_and_init(&objstr::get_value(url), &opts) + let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; if let Some(headers) = headers { let h = request.headers(); - let headers: PyDictRef = headers.downcast().unwrap(); for (key, value) in headers.get_key_value_pairs() { - let key = &vm.to_str(&key)?.value; - let value = &vm.to_str(&value)?.value; - h.set(key, value) + let key = vm.to_str(&key)?; + let value = vm.to_str(&value)?; + h.set(key.as_str(), value.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } } @@ -85,7 +94,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(content_type) = content_type { request .headers() - .set("Content-Type", &objstr::get_value(&content_type)) + .set("Content-Type", content_type.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } @@ -104,9 +113,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(func, Some(vm.ctx.function_type()))]); - +fn browser_request_animation_frame(func: PyFunctionRef, vm: &VirtualMachine) -> PyResult { use std::{cell::RefCell, rc::Rc}; // this basic setup for request_animation_frame taken from: @@ -115,18 +122,16 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR let f = Rc::new(RefCell::new(None)); let g = f.clone(); - let func = func.clone(); - - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); *g.borrow_mut() = Some(Closure::wrap(Box::new(move |time: f64| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid from inside of request_animation_frame"); let vm = &stored_vm.vm; let func = func.clone(); let args = vec![vm.ctx.new_float(time)]; - let _ = vm.invoke(func, args); + let _ = vm.invoke(func.into_object(), args); let closure = f.borrow_mut().take(); drop(closure); @@ -141,10 +146,8 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR Ok(vm.ctx.new_int(id)) } -fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(id, Some(vm.ctx.int_type()))]); - - let id = objint::get_value(id).to_i32().ok_or_else(|| { +fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult { + let id = id.as_bigint().to_i32().ok_or_else(|| { vm.new_exception( vm.ctx.exceptions.value_error.clone(), "Integer too large to convert to i32 for animationFrame id".into(), @@ -186,14 +189,14 @@ impl PyPromise { fn then( zelf: PyPromiseRef, - on_fulfill: PyRef, - on_reject: OptionalArg>, + on_fulfill: PyFunctionRef, + on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { - let stored_vm = &acc_vm + let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -217,16 +220,12 @@ impl PyPromise { PyPromise::from_future(ret_future).into_ref(vm) } - fn catch( - zelf: PyPromiseRef, - on_reject: PyRef, - vm: &VirtualMachine, - ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + fn catch(zelf: PyPromiseRef, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + let weak_vm = weak_vm(vm); let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { res.or_else(|err| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -303,41 +302,31 @@ impl Element { } } -fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_alert(message: PyStringRef, vm: &VirtualMachine) -> PyResult { window() - .alert_with_message(&objstr::get_value(message)) + .alert_with_message(message.as_str()) .expect("alert() not to fail"); Ok(vm.get_none()) } -fn browser_confirm(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_confirm(message: PyStringRef, vm: &VirtualMachine) -> PyResult { let result = window() - .confirm_with_message(&objstr::get_value(message)) + .confirm_with_message(message.as_str()) .expect("confirm() not to fail"); Ok(vm.new_bool(result)) } -fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(message, Some(vm.ctx.str_type()))], - optional = [(default, Some(vm.ctx.str_type()))] - ); - - let result = if let Some(default) = default { - window().prompt_with_message_and_default( - &objstr::get_value(message), - &objstr::get_value(default), - ) +fn browser_prompt( + message: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let result = if let OptionalArg::Present(default) = default { + window().prompt_with_message_and_default(message.as_str(), default.as_str()) } else { - window().prompt_with_message(&objstr::get_value(message)) + window().prompt_with_message(message.as_str()) }; let result = match result.expect("prompt() not to fail") { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 717c9a973..6fb527d76 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -8,7 +8,7 @@ use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; -use crate::vm_class::{AccessibleVM, WASMVirtualMachine}; +use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { macro_rules! map_exceptions { @@ -81,11 +81,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { return Err(err); } }; - let acc_vm = AccessibleVM::from(wasm_vm.clone()); - let stored_vm = acc_vm - .upgrade() - .expect("acc. VM to be invalid when WASM vm is valid"); - let vm = &stored_vm.vm; + let vm = &stored_vm_from_wasm(&wasm_vm).vm; let mut py_func_args = PyFuncArgs::default(); if let Some(ref args) = args { for arg in args.values() { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index d6f9ede0e..3f7bdef8a 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -43,8 +43,24 @@ impl StoredVirtualMachine { // probably gets compiled down to a normal-ish static varible, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { - static STORED_VMS: RefCell>> = - RefCell::default(); + static STORED_VMS: RefCell>> = RefCell::default(); +} + +pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { + STORED_VMS.with(|cell| { + cell.borrow() + .get(&wasm_vm.id) + .expect("VirtualMachine is not valid") + .clone() + }) +} +pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak { + let id = vm + .wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set"); + STORED_VMS + .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } #[wasm_bindgen(js_name = vmStore)] @@ -104,44 +120,6 @@ impl VMStore { } } -#[derive(Clone)] -pub(crate) struct AccessibleVM { - weak: Weak, - id: String, -} - -impl AccessibleVM { - pub fn from_id(id: String) -> AccessibleVM { - let weak = STORED_VMS - .with(|cell| Rc::downgrade(cell.borrow().get(&id).expect("WASM VM to be valid"))); - AccessibleVM { weak, id } - } - - pub fn upgrade(&self) -> Option> { - self.weak.upgrade() - } -} - -impl From for AccessibleVM { - fn from(vm: WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id) - } -} -impl From<&WASMVirtualMachine> for AccessibleVM { - fn from(vm: &WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id.clone()) - } -} -impl From<&VirtualMachine> for AccessibleVM { - fn from(vm: &VirtualMachine) -> AccessibleVM { - AccessibleVM::from_id( - vm.wasm_id - .clone() - .expect("VM passed to from::() to have wasm_id be Some()"), - ) - } -} - #[wasm_bindgen(js_name = VirtualMachine)] #[derive(Clone)] pub struct WASMVirtualMachine { From 3e42edd2613791cd91cdc262d7573640bac33970 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 08:51:44 +0100 Subject: [PATCH 079/223] dict.__new__ - support for dict subtypes. --- tests/snippets/dict.py | 7 +++++++ vm/src/obj/objdict.rs | 16 ++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 2d369465c..d2aa9ed5b 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -57,6 +57,13 @@ assert x.get("not here", "default") == "default" assert x.get("here", "default") == "here" assert x.get("not here") == None +class LengthDict(dict): + pass + +x = LengthDict() +assert type(x) == LengthDict + + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 541645d5f..6388aa9ab 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -40,17 +40,18 @@ impl PyValue for PyDict { // Python dict methods: impl PyDictRef { fn new( - _class: PyClassRef, // TODO Support subclasses of int. + class: PyClassRef, dict_obj: OptionalArg, kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult { - let dict = vm.ctx.new_dict(); + let mut dict = DictContentType::default(); + if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { for (key, value) in dict_obj.get_key_value_pairs() { - dict.set_item(key, value, vm); + dict.insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -68,14 +69,17 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.set_item(key, value, vm); + dict.insert(vm, &key, value)?; } } } for (key, value) in kwargs.into_iter() { - dict.set_item(vm.new_str(key), value, vm); + dict.insert(vm, &vm.new_str(key), value)?; } - Ok(dict) + PyDict { + entries: RefCell::new(dict), + } + .into_ref_with_type(vm, class) } fn bool(self, _vm: &VirtualMachine) -> bool { From ee9066a713e0b81802e718b79a485eeee1bf06b3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:02:49 +0100 Subject: [PATCH 080/223] dict.get shouldn't call into __getitem__ --- tests/snippets/dict.py | 6 ++++-- vm/src/dictdatatype.rs | 7 +++---- vm/src/obj/objdict.rs | 22 ++++++++++++++-------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index d2aa9ed5b..e28fd9da4 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -58,11 +58,13 @@ assert x.get("here", "default") == "here" assert x.get("not here") == None class LengthDict(dict): - pass + def __getitem__(self, k): + return len(k) x = LengthDict() assert type(x) == LengthDict - +assert x['word'] == 4 +assert x.get('word') is None # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index ac1df994a..65ad55110 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -76,16 +76,15 @@ impl Dict { } /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.entries[index] { - Ok(entry.value.clone()) + Ok(Some(entry.value.clone())) } else { panic!("Lookup returned invalid index into entries!"); } } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + Ok(None) } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 6388aa9ab..0e1a2aedd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -179,7 +179,13 @@ impl PyDictRef { } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.entries.borrow().get(vm, &key) + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => { + let key_repr = vm.to_pystr(&key)?; + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + } + } } fn get( @@ -187,12 +193,12 @@ impl PyDictRef { key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine, - ) -> PyObjectRef { - match self.into_object().get_item(key, vm) { - Ok(value) => value, - Err(_) => match default { - OptionalArg::Present(value) => value, - OptionalArg::Missing => vm.ctx.none(), + ) -> PyResult { + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => match default { + OptionalArg::Present(value) => Ok(value), + OptionalArg::Missing => Ok(vm.ctx.none()), }, } } @@ -220,7 +226,7 @@ impl PyDictRef { impl DictProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow().get(vm, &key).ok() + self.entries.borrow().get(vm, &key).unwrap() } // Item set/get: From 7b2d92f495e037fd85c19398ee2ba37b42a79cae Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:41:46 +0100 Subject: [PATCH 081/223] Delete DictProtocol. impl ItemProtocol for PyDictRef. --- tests/snippets/dict.py | 2 ++ vm/src/builtins.rs | 4 ++-- vm/src/frame.rs | 18 +++++++++--------- vm/src/import.rs | 4 ++-- vm/src/obj/objdict.rs | 34 +++++++++++++++++++--------------- vm/src/obj/objobject.rs | 18 +++++++++++------- vm/src/obj/objsuper.rs | 4 ++-- vm/src/pyobject.rs | 22 ++++++++++++++++------ vm/src/stdlib/json.rs | 6 ++---- vm/src/sysmodule.rs | 6 +++--- vm/src/vm.rs | 14 +++++++------- wasm/lib/src/convert.rs | 5 +++-- 12 files changed, 78 insertions(+), 59 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index e28fd9da4..c04165f16 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -66,6 +66,8 @@ assert type(x) == LengthDict assert x['word'] == 4 assert x.get('word') is None +assert 5 == eval("a + word", LengthDict()) + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 5aa5ee226..210d12b49 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -20,7 +20,7 @@ use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; use crate::function::{Args, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -804,6 +804,6 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu "__call__", vec![name_arg, bases, namespace.into_object()], )?; - cells.set_item("__class__", class.clone(), vm); + cells.set_item("__class__", class.clone(), vm)?; Ok(class) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2f525afdd..ca3e54d07 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -21,8 +21,8 @@ use crate::obj::objstr; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; use itertools::Itertools; @@ -133,12 +133,12 @@ pub trait NameProtocol { impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name, vm) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } - if let Some(value) = self.globals.get_item(name, vm) { + if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { return Some(value); } @@ -147,7 +147,7 @@ impl NameProtocol for Scope { fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name, vm) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } @@ -155,11 +155,11 @@ impl NameProtocol for Scope { } fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(key, value, vm) + self.get_locals().set_item(key, value, vm).unwrap(); } fn delete_name(&self, vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key, vm) + self.get_locals().del_item(key, vm).unwrap(); } } @@ -394,12 +394,12 @@ impl Frame { obj.downcast().expect("Need a dictionary to build a map."); let dict_elements = dict.get_key_value_pairs(); for (key, value) in dict_elements.iter() { - map_obj.set_item(key.clone(), value.clone(), vm); + map_obj.set_item(key.clone(), value.clone(), vm).unwrap(); } } } else { for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { - map_obj.set_item(key, value, vm) + map_obj.set_item(key, value, vm).unwrap(); } } diff --git a/vm/src/import.rs b/vm/src/import.rs index 713393deb..14c0a7566 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use crate::compile; use crate::frame::Scope; use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, ItemProtocol, PyResult}; +use crate::pyobject::{ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; @@ -39,7 +39,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item("__name__", vm.new_str(module.to_string()), vm); + attrs.set_item("__name__", vm.new_str(module.to_string()), vm)?; vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 0e1a2aedd..02f8443b5 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,8 +3,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, - PyResult, PyValue, + IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -174,8 +173,8 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) { - self.set_item(key, value, vm) + fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().insert(vm, &key, value) } fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -223,21 +222,26 @@ impl PyDictRef { } } -impl DictProtocol for PyDictRef { - fn get_item(&self, key: T, vm: &VirtualMachine) -> Option { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow().get(vm, &key).unwrap() +impl ItemProtocol for PyDictRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.as_object(), "__getitem__", key.into_pyobject(vm)?) } - // Item set/get: - fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine) { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow_mut().insert(vm, &key, value).unwrap() + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method( + self.as_object(), + "__setitem__", + vec![key.into_pyobject(vm)?, value], + ) } - fn del_item(&self, key: T, vm: &VirtualMachine) { - let key = key.into_pyobject(vm).unwrap(); - self.entries.borrow_mut().delete(vm, &key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.as_object(), "__delitem__", key.into_pyobject(vm)?) } } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c03ff31a1..a3bc41103 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -6,7 +6,7 @@ use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, + IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -77,7 +77,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.set_item(attr_name, value, vm); + dict.set_item(attr_name, value, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -98,7 +98,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.del_item(attr_name, vm); + dict.del_item(attr_name, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -208,7 +208,7 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } - if let Some(obj_attr) = object_getattr(&obj, &name, &vm) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm)? { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { vm.call_get_descriptor(attr, obj) @@ -219,11 +219,15 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } -fn object_getattr(obj: &PyObjectRef, attr_name: &str, vm: &VirtualMachine) -> Option { +fn object_getattr( + obj: &PyObjectRef, + attr_name: &str, + vm: &VirtualMachine, +) -> PyResult> { if let Some(ref dict) = obj.dict { - dict.get_item(attr_name, vm) + dict.get_item_option(attr_name, vm) } else { - None + Ok(None) } } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 19c7c2ea8..94fe0fc7b 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -12,7 +12,7 @@ use crate::obj::objfunction::PyMethod; use crate::obj::objstr; use crate::obj::objtype::{PyClass, PyClassRef}; use crate::pyobject::{ - DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -124,7 +124,7 @@ fn super_new( } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg, vm) { + match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { return Err(vm diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index bf5cf3c86..ee765f987 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -912,12 +912,6 @@ impl TypeProtocol for PyRef { } } -pub trait DictProtocol { - fn get_item(&self, key: T, vm: &VirtualMachine) -> Option; - fn set_item(&self, key: T, value: PyObjectRef, vm: &VirtualMachine); - fn del_item(&self, key: T, vm: &VirtualMachine); -} - pub trait ItemProtocol { fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; fn set_item( @@ -927,6 +921,22 @@ pub trait ItemProtocol { vm: &VirtualMachine, ) -> PyResult; fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item_option( + &self, + key: T, + vm: &VirtualMachine, + ) -> PyResult> { + match self.get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) => { + if objtype::isinstance(&exc, &vm.ctx.exceptions.key_error) { + Ok(None) + } else { + Err(exc) + } + } + } + } } impl ItemProtocol for PyObjectRef { diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index b7cb704b1..f4061bc2a 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -13,9 +13,7 @@ use crate::obj::{ objstr::{self, PyString}, objtype, }; -use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol, -}; +use crate::pyobject::{create_type, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -178,7 +176,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { Some(PyString { ref value }) => value.clone(), _ => unimplemented!("map keys must be strings"), }; - dict.set_item(&key, value, self.vm); + dict.set_item(&key, value, self.vm).unwrap(); } Ok(dict.into_object()) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 772dfe198..2f8690448 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; /* @@ -139,6 +139,6 @@ settrace() -- set the global debug tracing function "modules" => modules.clone(), }); - modules.set_item("sys", module.clone(), vm); - modules.set_item("builtins", builtins.clone(), vm); + modules.set_item("sys", module.clone(), vm).unwrap(); + modules.set_item("builtins", builtins.clone(), vm).unwrap(); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 0bd255574..90daaa668 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -30,7 +30,7 @@ use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::stdlib; @@ -423,7 +423,7 @@ impl VirtualMachine { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &args.args[i]; - locals.set_item(arg_name, arg.clone(), self); + locals.set_item(arg_name, arg.clone(), self)?; } // Pack other positional arguments in to *args: @@ -436,7 +436,7 @@ impl VirtualMachine { } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(vararg_name, vararg_value, self); + locals.set_item(vararg_name, vararg_value, self)?; } bytecode::Varargs::Unnamed => { // just ignore the rest of the args @@ -456,7 +456,7 @@ impl VirtualMachine { let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { let d = self.ctx.new_dict(); - locals.set_item(kwargs_name, d.as_object().clone(), self); + locals.set_item(kwargs_name, d.as_object().clone(), self)?; Some(d) } bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), @@ -474,9 +474,9 @@ impl VirtualMachine { ); } - locals.set_item(&name, value, self); + locals.set_item(&name, value, self)?; } else if let Some(d) = &kwargs { - d.set_item(&name, value, self); + d.set_item(&name, value, self)?; } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -520,7 +520,7 @@ impl VirtualMachine { for (default_index, i) in (required_args..nexpected_args).enumerate() { let arg_name = &code_object.arg_names[i]; if !locals.contains_key(arg_name, self) { - locals.set_item(arg_name, available_defaults[default_index].clone(), self); + locals.set_item(arg_name, available_defaults[default_index].clone(), self)?; } } }; diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 717c9a973..25024734c 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -4,7 +4,7 @@ use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbytes, objint, objsequence, objtype}; -use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; @@ -192,7 +192,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm) + .unwrap(); } dict.into_object() } From 0fe3f7748cf5a8a01097a70ee2db9292fd033a18 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:46:17 +0100 Subject: [PATCH 082/223] Simplify PyDictRef.__repr__. --- vm/src/obj/objdict.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 02f8443b5..7cefce9af 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -89,7 +89,7 @@ impl PyDictRef { self.entries.borrow().len() } - fn repr(self, vm: &VirtualMachine) -> PyResult { + fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; for (key, value) in self.get_key_value_pairs() { @@ -102,7 +102,7 @@ impl PyDictRef { } else { "{...}".to_string() }; - Ok(vm.new_str(s)) + Ok(s) } fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { From e2a4f22be87cdd43c00eed9f134379c9a3d3f21e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:49:32 +0100 Subject: [PATCH 083/223] Avoid name similarity between ItemProtocol and inner methods. --- vm/src/obj/objdict.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7cefce9af..17910db23 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -109,7 +109,7 @@ impl PyDictRef { self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn inner_delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.entries.borrow_mut().delete(vm, &key) } @@ -173,11 +173,16 @@ impl PyDictRef { self.entries.borrow().get_items() } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn inner_setitem( + self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { self.entries.borrow_mut().insert(vm, &key, value) } - fn getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self.entries.borrow().get(vm, &key)? { Some(value) => Ok(value), None => { @@ -250,12 +255,12 @@ pub fn init(context: &PyContext) { "__bool__" => context.new_rustfunc(PyDictRef::bool), "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), - "__delitem__" => context.new_rustfunc(PyDictRef::delitem), - "__getitem__" => context.new_rustfunc(PyDictRef::getitem), + "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), - "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__setitem__" => context.new_rustfunc(PyDictRef::inner_setitem), "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), From b1dd5836af08bd550cf85005d0c4eaccc1e0cdcd Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 09:52:18 +0100 Subject: [PATCH 084/223] Simpler implementation of ItemProtocol for PyDictRef. --- vm/src/obj/objdict.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 17910db23..3b7a33666 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -229,7 +229,7 @@ impl PyDictRef { impl ItemProtocol for PyDictRef { fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.as_object(), "__getitem__", key.into_pyobject(vm)?) + self.as_object().get_item(key, vm) } fn set_item( @@ -238,15 +238,11 @@ impl ItemProtocol for PyDictRef { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - vm.call_method( - self.as_object(), - "__setitem__", - vec![key.into_pyobject(vm)?, value], - ) + self.as_object().set_item(key, value, vm) } fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.as_object(), "__delitem__", key.into_pyobject(vm)?) + self.as_object().del_item(key, vm) } } From 584b707356ba6aa614ab3d7d2de02feb280c7f79 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 10:04:18 +0100 Subject: [PATCH 085/223] Add dict.__missing__ support. --- tests/snippets/dict.py | 10 ++++++++++ vm/src/obj/objdict.rs | 14 ++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index c04165f16..69040d624 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -68,6 +68,16 @@ assert x.get('word') is None assert 5 == eval("a + word", LengthDict()) + +class Squares(dict): + def __missing__(self, k): + v = k * k + self[k] = v + return v + +x = Squares() +assert x[-5] == 25 + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 3b7a33666..d54f62bb7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -183,13 +183,15 @@ impl PyDictRef { } fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match self.entries.borrow().get(vm, &key)? { - Some(value) => Ok(value), - None => { - let key_repr = vm.to_pystr(&key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) - } + if let Some(value) = self.entries.borrow().get(vm, &key)? { + return Ok(value); } + + if let Ok(method) = vm.get_method(self.clone().into_object(), "__missing__") { + return vm.invoke(method, vec![key]); + } + + Err(vm.new_key_error(format!("Key not found: {}", vm.to_pystr(&key)?))) } fn get( From afb28bfccba4e4b07f5c43b6100d330b18fdc430 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sat, 6 Apr 2019 11:53:59 +0200 Subject: [PATCH 086/223] Fix moment when docstring is set. --- tests/snippets/function.py | 14 ++++++++++++++ vm/src/compile.rs | 31 +++++++++++++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index aafc9b53c..e4d0ab544 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -41,3 +41,17 @@ def f4(): pass assert f4.__doc__ == "test4" + + +def revdocstr(f): + d = f.__doc__ + d = d + 'w00t' + f.__doc__ = d + return f + +@revdocstr +def f5(): + """abc""" + +assert f5.__doc__ == 'abcw00t', f5.__doc__ + diff --git a/vm/src/compile.rs b/vm/src/compile.rs index d7f774b83..d0ea96d32 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -650,25 +650,13 @@ impl Compiler { // Turn code object into function object: self.emit(Instruction::MakeFunction { flags }); + self.store_docstring(doc_str); self.apply_decorators(decorator_list); self.emit(Instruction::StoreName { name: name.to_string(), }); - if let Some(doc_string) = doc_str { - self.emit(Instruction::LoadConst { - value: bytecode::Constant::String { - value: doc_string.to_string(), - }, - }); - self.emit(Instruction::LoadName { - name: name.to_string(), - }); - self.emit(Instruction::StoreAttr { - name: "__doc__".to_string(), - }); - } self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -761,26 +749,33 @@ impl Compiler { }); } + self.store_docstring(doc_str); self.apply_decorators(decorator_list); self.emit(Instruction::StoreName { name: name.to_string(), }); + self.in_loop = was_in_loop; + Ok(()) + } + + fn store_docstring(&mut self, doc_str: Option) { if let Some(doc_string) = doc_str { + // Duplicate top of stack (the function or class object) + self.emit(Instruction::Duplicate); + + // Doc string value: self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: doc_string.to_string(), }, }); - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + + self.emit(Instruction::Rotate { amount: 2 }); self.emit(Instruction::StoreAttr { name: "__doc__".to_string(), }); } - self.in_loop = was_in_loop; - Ok(()) } fn compile_for( From 1f0fd39722e63468884ee4b89947b97c89d40657 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 17:29:40 +0300 Subject: [PATCH 087/223] Change os new funcs to new arg style --- vm/src/obj/objbytes.rs | 2 +- vm/src/stdlib/os.rs | 38 +++++++++++--------------------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f24c9273c..4a8b1ad5e 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -17,7 +17,7 @@ use super::objtype::PyClassRef; pub struct PyBytes { value: Vec, } -type PyBytesRef = PyRef; +pub type PyBytesRef = PyRef; impl PyBytes { pub fn new(data: Vec) -> Self { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cd4c0709b..ae487598a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -6,9 +6,11 @@ use std::io::{ErrorKind, Read, Write}; use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; -use crate::obj::objbytes; +use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; +use crate::obj::objint::PyIntRef; use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -115,15 +117,9 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(fd, Some(vm.ctx.int_type())), (n, Some(vm.ctx.int_type()))] - ); - - let mut buffer = vec![0u8; objint::get_value(n).to_usize().unwrap()]; - let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); +fn os_read(fd: PyIntRef, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; n.as_bigint().to_usize().unwrap()]; + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); match file.read_exact(&mut buffer) { Ok(_) => (), Err(s) => return Err(vm.new_os_error(s.to_string())), @@ -134,18 +130,9 @@ fn os_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(buffer)) } -fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (fd, Some(vm.ctx.int_type())), - (data, Some(vm.ctx.bytes_type())) - ] - ); - - let mut file = rust_file(objint::get_value(fd).to_i64().unwrap()); - let written = match file.write(&objbytes::get_value(&data)) { +fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + let written = match file.write(&data) { Ok(written) => written, Err(s) => return Err(vm.new_os_error(s.to_string())), }; @@ -155,11 +142,8 @@ fn os_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(path, Some(vm.ctx.str_type()))]); - let fname = objstr::get_value(&path); - - match fs::remove_file(fname) { +fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::remove_file(&path.value) { Ok(_) => (), Err(s) => return Err(vm.new_os_error(s.to_string())), } From 04acfe266523139f793e31e9cf148ab9374d27fe Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 17:29:52 +0300 Subject: [PATCH 088/223] Remove failing line in windows --- tests/snippets/stdlib_os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 29c8005dc..ba6033ff8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -5,7 +5,6 @@ from testutils import assert_raises fd = os.open('README.md', 0) assert fd > 0 -assert_raises(OSError, lambda: os.read(fd + 1, 10)) os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) From e67d5f3c627f8701f07dc55e5136f6c4389d7c9e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 15:27:46 +0100 Subject: [PATCH 089/223] Parameterise dictdatatype::Dict by value type. --- vm/src/dictdatatype.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 65ad55110..6641a1884 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,20 +9,29 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; -#[derive(Default)] -pub struct Dict { +pub struct Dict { size: usize, indices: HashMap, - entries: Vec>, + entries: Vec>>, } -struct DictEntry { +impl Default for Dict { + fn default() -> Self { + Dict { + size: 0, + indices: HashMap::new(), + entries: Vec::new(), + } + } +} + +struct DictEntry { hash: usize, key: PyObjectRef, - value: PyObjectRef, + value: T, } -impl Dict { +impl Dict { pub fn new() -> Self { Dict { size: 0, @@ -32,12 +41,7 @@ impl Dict { } /// Store a key - pub fn insert( - &mut self, - vm: &VirtualMachine, - key: &PyObjectRef, - value: PyObjectRef, - ) -> PyResult<()> { + pub fn insert(&mut self, vm: &VirtualMachine, key: &PyObjectRef, value: T) -> PyResult<()> { match self.lookup(vm, key)? { LookupResult::Existing(index) => { // Update existing key @@ -76,7 +80,7 @@ impl Dict { } /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.entries[index] { Ok(Some(entry.value.clone())) @@ -114,7 +118,7 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { self.entries .iter() .filter(|e| e.is_some()) From eb9378fe900db7a859f0d71ab99bf8ce9071e932 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:20:18 +0300 Subject: [PATCH 090/223] Add os.mkdir --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index ae487598a..312b79549 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -151,6 +151,15 @@ fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::create_dir(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -168,6 +177,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "write" => ctx.new_rustfunc(os_write), "remove" => ctx.new_rustfunc(os_remove), "unlink" => ctx.new_rustfunc(os_remove), + "mkdir" => ctx.new_rustfunc(os_mkdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 2ec6afeb11a4a400584c000b6606dbd0640f09a0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:24:06 +0300 Subject: [PATCH 091/223] Add os.mkdirs --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 312b79549..fc7651027 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -160,6 +160,15 @@ fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::create_dir_all(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -178,6 +187,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "remove" => ctx.new_rustfunc(os_remove), "unlink" => ctx.new_rustfunc(os_remove), "mkdir" => ctx.new_rustfunc(os_mkdir), + "mkdirs" => ctx.new_rustfunc(os_mkdirs), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From bf6b12266a9b3996badd01f2ef36aac1b35ca4b7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 18:30:40 +0300 Subject: [PATCH 092/223] Add os.rmdir --- vm/src/stdlib/os.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index fc7651027..6ffb36ca5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -169,6 +169,15 @@ fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::remove_dir(&path.value) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + } + + Ok(vm.get_none()) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -188,6 +197,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "unlink" => ctx.new_rustfunc(os_remove), "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), + "rmdir" => ctx.new_rustfunc(os_rmdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From a21aa6eac9ca8016e13c54689c83120eb57b482c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 19:14:23 +0300 Subject: [PATCH 093/223] Add os.listdir --- vm/src/stdlib/os.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6ffb36ca5..f0f5c3dd5 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -178,6 +178,21 @@ fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } +fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => { + let res: PyResult> = iter + .map(|entry| match entry { + Ok(path) => Ok(vm.ctx.new_str(path.file_name().into_string().unwrap())), + Err(s) => Err(vm.new_os_error(s.to_string())), + }) + .collect(); + Ok(vm.ctx.new_list(res?)) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -198,6 +213,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), + "listdir" => ctx.new_rustfunc(os_listdir), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 401ca08b659d137200a80caf3bc87f08c3dc2642 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 6 Apr 2019 21:11:58 +0300 Subject: [PATCH 094/223] Use map_err --- vm/src/stdlib/os.rs | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f0f5c3dd5..25937cd5c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -142,40 +142,20 @@ fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::remove_file(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_file(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::create_dir(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::create_dir_all(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir_all(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { - match fs::remove_dir(&path.value) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - } - - Ok(vm.get_none()) +fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { From 5fd3cf2bcda2858acb5e2ca01163cf7a7a66dee1 Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 6 Apr 2019 20:29:25 +1300 Subject: [PATCH 095/223] Implemented keyword only defaults --- tests/snippets/function_args.py | 36 +++++++++++---- vm/src/bytecode.rs | 1 + vm/src/compile.rs | 28 ++++++++++-- vm/src/frame.rs | 26 +++++++++-- vm/src/obj/objfunction.rs | 13 +++++- vm/src/pyobject.rs | 7 +-- vm/src/vm.rs | 79 +++++++++++++++------------------ 7 files changed, 127 insertions(+), 63 deletions(-) diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 884cb47df..fd164d113 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -1,21 +1,28 @@ +from testutils import assertRaises + + def sum(x, y): return x+y -# def total(a, b, c, d): -# return sum(sum(a,b), sum(c,d)) -# -# assert total(1,1,1,1) == 4 -# assert total(1,2,3,4) == 10 +def total(a, b, c, d): + return sum(sum(a,b), sum(c,d)) + + +assert total(1,1,1,1) == 4 +assert total(1,2,3,4) == 10 assert sum(1, 1) == 2 assert sum(1, 3) == 4 + def sum2y(x, y): return x+y*2 + assert sum2y(1, 1) == 3 assert sum2y(1, 3) == 7 + def va(a, b=2, *c, d, **e): assert a == 1 assert b == 22 @@ -23,21 +30,34 @@ def va(a, b=2, *c, d, **e): assert d == 1337 assert e['f'] == 42 + va(1, 22, 3, 4, d=1337, f=42) + def va2(*args, **kwargs): assert args == (5, 4) assert len(kwargs) == 0 + va2(5, 4) x = (5, 4) va2(*x) va2(5, *x[1:]) -# def va3(x, *, b=2): -# pass -# va3(1, 2, 3, b=10) + +def va3(x, *, a, b=2, c=9): + return x + b + c + + +assert va3(1, a=1, b=10) == 20 + +with assertRaises(TypeError): + va3(1, 2, 3, a=1, b=10) + +with assertRaises(TypeError): + va3(1, b=10) + x = {'f': 42, 'e': 1337} y = {'d': 1337} diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 2fe485b63..b24801861 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -31,6 +31,7 @@ pub struct CodeObject { bitflags! { pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; + const HAS_KW_ONLY_DEFAULTS = 0x02; const HAS_ANNOTATIONS = 0x04; } } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 3780595c2..dd045c3bf 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -422,8 +422,8 @@ impl Compiler { name: &str, args: &ast::Parameters, ) -> Result { - let have_kwargs = !args.defaults.is_empty(); - if have_kwargs { + let have_defaults = !args.defaults.is_empty(); + if have_defaults { // Construct a tuple: let size = args.defaults.len(); for element in &args.defaults { @@ -435,6 +435,25 @@ impl Compiler { }); } + let mut num_kw_only_defaults = 0; + for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) { + if let Some(default) = default { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: kw.arg.clone(), + }, + }); + self.compile_expression(default)?; + num_kw_only_defaults += 1; + } + } + if num_kw_only_defaults > 0 { + self.emit(Instruction::BuildMap { + size: num_kw_only_defaults, + unpack: false, + }); + } + let line_number = self.get_source_line_number(); self.code_object_stack.push(CodeObject::new( args.args.iter().map(|a| a.arg.clone()).collect(), @@ -447,9 +466,12 @@ impl Compiler { )); let mut flags = bytecode::FunctionOpArg::empty(); - if have_kwargs { + if have_defaults { flags |= bytecode::FunctionOpArg::HAS_DEFAULTS; } + if num_kw_only_defaults > 0 { + flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS; + } Ok(flags) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ca3e54d07..2570c0934 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,12 +12,13 @@ use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict::PyDictRef; +use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ @@ -578,16 +579,33 @@ impl Frame { vm.ctx.new_dict().into_object() }; + let kw_only_defaults = + if flags.contains(bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS) { + Some( + self.pop_value().downcast::().expect( + "Stack value for keyword only defaults expected to be a dict", + ), + ) + } else { + None + }; + let defaults = if flags.contains(bytecode::FunctionOpArg::HAS_DEFAULTS) { - self.pop_value() + Some( + self.pop_value() + .downcast::() + .expect("Stack value for defaults expected to be a tuple"), + ) } else { - vm.get_none() + None }; // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm.ctx.new_function(code_obj, scope, defaults); + let obj = vm + .ctx + .new_function(code_obj, scope, defaults, kw_only_defaults); vm.set_attr(&obj, "__annotations__", annotations)?; diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 410794696..e8c1d7df8 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,5 +1,7 @@ use crate::frame::Scope; use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; @@ -11,15 +13,22 @@ pub struct PyFunction { // TODO: these shouldn't be public pub code: PyCodeRef, pub scope: Scope, - pub defaults: PyObjectRef, + pub defaults: Option, + pub kw_only_defaults: Option, } impl PyFunction { - pub fn new(code: PyCodeRef, scope: Scope, defaults: PyObjectRef) -> Self { + pub fn new( + code: PyCodeRef, + scope: Scope, + defaults: Option, + kw_only_defaults: Option, + ) -> Self { PyFunction { code, scope, defaults, + kw_only_defaults, } } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index ee765f987..4d2c8353b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -48,7 +48,7 @@ use crate::obj::objslice; use crate::obj::objstaticmethod; use crate::obj::objstr; use crate::obj::objsuper; -use crate::obj::objtuple::{self, PyTuple}; +use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; use crate::obj::objweakref; use crate::obj::objzip; @@ -658,10 +658,11 @@ impl PyContext { &self, code_obj: PyCodeRef, scope: Scope, - defaults: PyObjectRef, + defaults: Option, + kw_only_defaults: Option, ) -> PyObjectRef { PyObject::new( - PyFunction::new(code_obj, scope, defaults), + PyFunction::new(code_obj, scope, defaults, kw_only_defaults), self.function_type(), Some(self.new_dict()), ) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 90daaa668..d43ee7bea 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -23,10 +23,9 @@ use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGeneratorRef; use crate::obj::objiter; -use crate::obj::objlist::PyList; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; -use crate::obj::objtuple::PyTuple; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ @@ -332,9 +331,10 @@ impl VirtualMachine { ref code, ref scope, ref defaults, + ref kw_only_defaults, }) = func_ref.payload() { - return self.invoke_python_function(code, scope, defaults, args); + return self.invoke_python_function(code, scope, defaults, kw_only_defaults, args); } if let Some(PyMethod { ref function, @@ -356,11 +356,18 @@ impl VirtualMachine { &self, code: &PyCodeRef, scope: &Scope, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, args: PyFuncArgs, ) -> PyResult { let scope = scope.child_scope(&self.ctx); - self.fill_locals_from_args(&code.code, &scope.get_locals(), args, defaults)?; + self.fill_locals_from_args( + &code.code, + &scope.get_locals(), + args, + defaults, + kw_only_defaults, + )?; // Construct frame: let frame = Frame::new(code.clone(), scope).into_ref(self); @@ -379,12 +386,7 @@ impl VirtualMachine { cells: PyDictRef, locals: PyDictRef, ) -> PyResult { - if let Some(PyFunction { - code, - scope, - defaults: _, - }) = &function.payload() - { + if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope .child_scope_with_locals(cells) .child_scope_with_locals(locals); @@ -402,7 +404,8 @@ impl VirtualMachine { code_object: &bytecode::CodeObject, locals: &PyDictRef, args: PyFuncArgs, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, ) -> PyResult<()> { let nargs = args.args.len(); let nexpected_args = code_object.arg_names.len(); @@ -438,10 +441,7 @@ impl VirtualMachine { locals.set_item(vararg_name, vararg_value, self)?; } - bytecode::Varargs::Unnamed => { - // just ignore the rest of the args - } - bytecode::Varargs::None => { + bytecode::Varargs::Unnamed | bytecode::Varargs::None => { // Check the number of positional arguments if nargs > nexpected_args { return Err(self.new_type_error(format!( @@ -487,19 +487,11 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let available_defaults = if defaults.is(&self.get_none()) { - vec![] - } else if let Some(list) = defaults.payload::() { - list.elements.borrow().clone() - } else if let Some(tuple) = defaults.payload::() { - tuple.elements.borrow().clone() - } else { - panic!("function defaults not tuple or None"); - }; + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.borrow().len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception - let required_args = nexpected_args - available_defaults.len(); + let required_args = nexpected_args - num_defaults_available; let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; @@ -514,31 +506,32 @@ impl VirtualMachine { missing ))); } - - // We have sufficient defaults, so iterate over the corresponding names and use - // the default if we don't already have a value - for (default_index, i) in (required_args..nexpected_args).enumerate() { - let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name, self) { - locals.set_item(arg_name, available_defaults[default_index].clone(), self)?; + if let Some(defaults) = defaults { + let defaults = defaults.elements.borrow(); + // We have sufficient defaults, so iterate over the corresponding names and use + // the default if we don't already have a value + for (default_index, i) in (required_args..nexpected_args).enumerate() { + let arg_name = &code_object.arg_names[i]; + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, defaults[default_index].clone(), self)?; + } } } }; // Check if kw only arguments are all present: - let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { if !locals.contains_key(arg_name, self) { - if kwdefs.contains_key(arg_name) { - // If not yet specified, take the default value - unimplemented!(); - } else { - // No default value and not specified. - return Err(self.new_type_error(format!( - "Missing required kw only argument: '{}'", - arg_name - ))); + if let Some(kw_only_defaults) = kw_only_defaults { + if let Some(default) = kw_only_defaults.get_item(arg_name, self) { + locals.set_item(arg_name, default, self); + continue; + } } + + // No default value and not specified. + return Err(self + .new_type_error(format!("Missing required kw only argument: '{}'", arg_name))); } } From d5a9d96d62126f0b2080cf2ce3a33bd822b2bf15 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 07:36:07 +1200 Subject: [PATCH 096/223] Expose function.__defaults__ and function.__kwdefaults__(only getters) --- tests/snippets/function_args.py | 10 ++++++++++ vm/src/obj/objfunction.rs | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index fd164d113..b9c5a1626 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -33,6 +33,9 @@ def va(a, b=2, *c, d, **e): va(1, 22, 3, 4, d=1337, f=42) +assert va.__defaults__ == (2,) +assert va.__kwdefaults__ is None + def va2(*args, **kwargs): assert args == (5, 4) @@ -59,6 +62,13 @@ with assertRaises(TypeError): va3(1, b=10) +assert va3.__defaults__ is None +kw_defaults = va3.__kwdefaults__ +# assert va3.__kwdefaults__ == {'b': 2, 'c': 9} +assert set(kw_defaults) == {'b', 'c'} +assert kw_defaults['b'] == 2 +assert kw_defaults['c'] == 9 + x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index e8c1d7df8..01e4b586f 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -43,6 +43,14 @@ impl PyFunctionRef { fn code(self, _vm: &VirtualMachine) -> PyCodeRef { self.code.clone() } + + fn defaults(self, _vm: &VirtualMachine) -> Option { + self.defaults.clone() + } + + fn kwdefaults(self, _vm: &VirtualMachine) -> Option { + self.kw_only_defaults.clone() + } } #[derive(Debug)] @@ -68,7 +76,9 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), - "__code__" => context.new_property(PyFunctionRef::code) + "__code__" => context.new_property(PyFunctionRef::code), + "__defaults__" => context.new_property(PyFunctionRef::defaults), + "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), }); let builtin_function_or_method_type = &context.builtin_function_or_method_type; From 51732feaed7e4f7196ecaac9ab1d28b3cdc23181 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 07:43:29 +1200 Subject: [PATCH 097/223] Use get_item_option from ItemProtocol instead of get_item --- vm/src/vm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index d43ee7bea..7be05023b 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -523,8 +523,8 @@ impl VirtualMachine { for arg_name in &code_object.kwonlyarg_names { if !locals.contains_key(arg_name, self) { if let Some(kw_only_defaults) = kw_only_defaults { - if let Some(default) = kw_only_defaults.get_item(arg_name, self) { - locals.set_item(arg_name, default, self); + if let Some(default) = kw_only_defaults.get_item_option(arg_name, self)? { + locals.set_item(arg_name, default, self)?; continue; } } From 10cbf2ae740c0fe7de2151a2e3daf88576c7a394 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 01:49:16 +0200 Subject: [PATCH 098/223] bytes._new stuff --- vm/src/obj/objbytes.rs | 83 +++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8f43cd05f..26de25010 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,3 +1,9 @@ +use crate::obj::objint::PyInt; +use crate::obj::objlist; +use crate::obj::objlist::PyList; +use crate::obj::objstr::PyString; +use crate::obj::objtuple::PyTuple; +use crate::obj::objtype; use std::cell::Cell; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -7,7 +13,8 @@ use num_traits::ToPrimitive; use crate::function::OptionalArg; use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -87,29 +94,79 @@ extend_class!(context, bytesiterator_type, { "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), });*/ //} + +fn load_byte( + elements: PyResult>, + vm: &VirtualMachine, +) -> Result, PyObjectRef> { + if let Ok(value) = elements { + let mut data_bytes = vec![]; + for elem in value.iter() { + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); + } + } + Ok(data_bytes) + } else { + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } +} + #[pyimpl(__inside_vm)] impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( cls: PyClassRef, val_option: OptionalArg, + enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); + if let OptionalArg::Present(enc) = enc_option { + if let OptionalArg::Present(eval) = val_option { + if objtype::isinstance(&eval, &vm.ctx.str_type()) + && objtype::isinstance(&enc, &vm.ctx.str_type()) + { + //return Ok(PyBytes::new(vec![1]).into_ref_with_type(vm, cls.clone())); + return Err(vm.new_type_error("Ok les 2 sont sstr".to_string())); + } else { + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; + } - PyBytes::new(value).into_ref_with_type(vm, cls) + let value = if let OptionalArg::Present(ival) = val_option { + println!("{:?}", ival); + match_class!(ival.clone(), + _i @ PyInt => { + let size = objint::get_value(&ival).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _l @ PyString=> load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _obj => {return Err(vm.new_type_error(format!( + "int() argument must be a string or a number, not " + )));} + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), + Err(err) => Err(err), + } } #[pymethod(name = "__repr__")] From be55562457fbd361e3478035cdaeccef6151b8dd Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 6 Apr 2019 12:56:09 -0700 Subject: [PATCH 099/223] Allow first argument to be a shared ref to a python value --- vm/src/function.rs | 27 ++++++++++++++++++++++++--- vm/src/obj/objfloat.rs | 14 ++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index aae4eefe8..305c4de5e 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use std::ops::RangeInclusive; use crate::obj::objtype::{isinstance, PyClassRef}; -use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use crate::pyobject::{ + IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use self::OptionalArg::*; @@ -403,6 +405,7 @@ tuple_from_py_func_args!(A, B); tuple_from_py_func_args!(A, B, C); tuple_from_py_func_args!(A, B, C, D); tuple_from_py_func_args!(A, B, C, D, E); +tuple_from_py_func_args!(A, B, C, D, E, F); /// A built-in Python function. pub type PyNativeFunc = Box PyResult + 'static>; @@ -440,17 +443,19 @@ impl IntoPyNativeFunc for PyNativeFunc { } } +pub struct OwnedParam(std::marker::PhantomData); +pub struct RefParam(std::marker::PhantomData); + // This is the "magic" that allows rust functions of varying signatures to // generate native python functions. // // Note that this could be done without a macro - it is simply to avoid repetition. macro_rules! into_py_native_func_tuple { ($(($n:tt, $T:ident)),*) => { - impl IntoPyNativeFunc<($($T,)*), R> for F + impl IntoPyNativeFunc<($(OwnedParam<$T>,)*), R> for F where F: Fn($($T,)* &VirtualMachine) -> R + 'static, $($T: FromArgs,)* - ($($T,)*): FromArgs, R: IntoPyObject, { fn into_func(self) -> PyNativeFunc { @@ -461,6 +466,22 @@ macro_rules! into_py_native_func_tuple { }) } } + + impl IntoPyNativeFunc<(RefParam, $(OwnedParam<$T>,)*), R> for F + where + F: Fn(&S, $($T,)* &VirtualMachine) -> R + 'static, + S: PyValue, + $($T: FromArgs,)* + R: IntoPyObject, + { + fn into_func(self) -> PyNativeFunc { + Box::new(move |vm, args| { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)* vm).into_pyobject(vm) + }) + } + } }; } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index ca6bad9d2..de7c465d3 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -34,6 +34,13 @@ impl From for PyFloat { } } +impl PyFloat { + fn is_integer(&self, _vm: &VirtualMachine) -> bool { + let v = self.value; + (v - v.round()).abs() < std::f64::EPSILON + } +} + pub type PyFloatRef = PyRef; impl PyFloatRef { @@ -311,11 +318,6 @@ impl PyFloatRef { } } - fn is_integer(self, _vm: &VirtualMachine) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON - } - fn real(self, _vm: &VirtualMachine) -> Self { self } @@ -385,7 +387,7 @@ pub fn init(context: &PyContext) { "__mul__" => context.new_rustfunc(PyFloatRef::mul), "__rmul__" => context.new_rustfunc(PyFloatRef::mul), "real" => context.new_property(PyFloatRef::real), - "is_integer" => context.new_rustfunc(PyFloatRef::is_integer), + "is_integer" => context.new_rustfunc(PyFloat::is_integer), "as_integer_ratio" => context.new_rustfunc(PyFloatRef::as_integer_ratio) }); } From c2716040d2419311837d9ef025e476b68f1310e0 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 6 Apr 2019 19:05:18 -0700 Subject: [PATCH 100/223] float: take self by shared ref --- vm/src/obj/objfloat.rs | 112 ++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index de7c465d3..796da71ae 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -35,16 +35,7 @@ impl From for PyFloat { } impl PyFloat { - fn is_integer(&self, _vm: &VirtualMachine) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON - } -} - -pub type PyFloatRef = PyRef; - -impl PyFloatRef { - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let value = self.value; let result = if objtype::isinstance(&other, &vm.ctx.float_type()) { let other = get_value(&other); @@ -63,7 +54,7 @@ impl PyFloatRef { vm.ctx.new_bool(result) } - fn lt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn lt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 < get_value(&i2)) @@ -75,7 +66,7 @@ impl PyFloatRef { } } - fn le(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn le(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 <= get_value(&i2)) @@ -87,7 +78,7 @@ impl PyFloatRef { } } - fn gt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn gt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 > get_value(&i2)) @@ -99,7 +90,7 @@ impl PyFloatRef { } } - fn ge(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ge(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 >= get_value(&i2)) @@ -111,11 +102,11 @@ impl PyFloatRef { } } - fn abs(self, _vm: &VirtualMachine) -> f64 { + fn abs(&self, _vm: &VirtualMachine) -> f64 { self.value.abs() } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1 + get_value(&other)) @@ -127,23 +118,23 @@ impl PyFloatRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { self.value != 0.0 } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.float_type()) || objtype::isinstance(&other, &vm.ctx.int_type()) { - let r1 = PyFloatRef::floordiv(self.clone(), other.clone(), vm)?; - let r2 = PyFloatRef::mod_(self, other, vm)?; + let r1 = self.floordiv(other.clone(), vm)?; + let r2 = self.mod_(other, vm)?; Ok(vm.ctx.new_tuple(vec![r1, r2])) } else { Ok(vm.ctx.not_implemented()) } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -202,7 +193,7 @@ impl PyFloatRef { PyFloat { value }.into_ref_with_type(vm, cls) } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -221,11 +212,11 @@ impl PyFloatRef { } } - fn neg(self, _vm: &VirtualMachine) -> f64 { + fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1.powf(get_value(&other))) @@ -237,7 +228,7 @@ impl PyFloatRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(v1 - get_value(&other))) @@ -250,7 +241,7 @@ impl PyFloatRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(get_value(&other) - v1)) @@ -263,11 +254,11 @@ impl PyFloatRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -286,7 +277,7 @@ impl PyFloatRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -305,7 +296,7 @@ impl PyFloatRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type) { Ok(vm.ctx.new_float(v1 * get_value(&other))) @@ -318,11 +309,16 @@ impl PyFloatRef { } } - fn real(self, _vm: &VirtualMachine) -> Self { - self + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf } - fn as_integer_ratio(self, vm: &VirtualMachine) -> PyResult { + fn is_integer(&self, _vm: &VirtualMachine) -> bool { + let v = self.value; + (v - v.round()).abs() < std::f64::EPSILON + } + + fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult { let value = self.value; if value.is_infinite() { return Err( @@ -340,6 +336,8 @@ impl PyFloatRef { } } +pub type PyFloatRef = PyRef; + // Retrieve inner float value: pub fn get_value(obj: &PyObjectRef) -> f64 { obj.payload::().unwrap().value @@ -363,31 +361,31 @@ pub fn init(context: &PyContext) { let float_doc = "Convert a string or number to a floating point number, if possible."; extend_class!(context, float_type, { - "__eq__" => context.new_rustfunc(PyFloatRef::eq), - "__lt__" => context.new_rustfunc(PyFloatRef::lt), - "__le__" => context.new_rustfunc(PyFloatRef::le), - "__gt__" => context.new_rustfunc(PyFloatRef::gt), - "__ge__" => context.new_rustfunc(PyFloatRef::ge), - "__abs__" => context.new_rustfunc(PyFloatRef::abs), - "__add__" => context.new_rustfunc(PyFloatRef::add), - "__radd__" => context.new_rustfunc(PyFloatRef::add), - "__bool__" => context.new_rustfunc(PyFloatRef::bool), - "__divmod__" => context.new_rustfunc(PyFloatRef::divmod), - "__floordiv__" => context.new_rustfunc(PyFloatRef::floordiv), - "__new__" => context.new_rustfunc(PyFloatRef::new_float), - "__mod__" => context.new_rustfunc(PyFloatRef::mod_), - "__neg__" => context.new_rustfunc(PyFloatRef::neg), - "__pow__" => context.new_rustfunc(PyFloatRef::pow), - "__sub__" => context.new_rustfunc(PyFloatRef::sub), - "__rsub__" => context.new_rustfunc(PyFloatRef::rsub), - "__repr__" => context.new_rustfunc(PyFloatRef::repr), + "__eq__" => context.new_rustfunc(PyFloat::eq), + "__lt__" => context.new_rustfunc(PyFloat::lt), + "__le__" => context.new_rustfunc(PyFloat::le), + "__gt__" => context.new_rustfunc(PyFloat::gt), + "__ge__" => context.new_rustfunc(PyFloat::ge), + "__abs__" => context.new_rustfunc(PyFloat::abs), + "__add__" => context.new_rustfunc(PyFloat::add), + "__radd__" => context.new_rustfunc(PyFloat::add), + "__bool__" => context.new_rustfunc(PyFloat::bool), + "__divmod__" => context.new_rustfunc(PyFloat::divmod), + "__floordiv__" => context.new_rustfunc(PyFloat::floordiv), + "__new__" => context.new_rustfunc(PyFloat::new_float), + "__mod__" => context.new_rustfunc(PyFloat::mod_), + "__neg__" => context.new_rustfunc(PyFloat::neg), + "__pow__" => context.new_rustfunc(PyFloat::pow), + "__sub__" => context.new_rustfunc(PyFloat::sub), + "__rsub__" => context.new_rustfunc(PyFloat::rsub), + "__repr__" => context.new_rustfunc(PyFloat::repr), "__doc__" => context.new_str(float_doc.to_string()), - "__truediv__" => context.new_rustfunc(PyFloatRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyFloatRef::rtruediv), - "__mul__" => context.new_rustfunc(PyFloatRef::mul), - "__rmul__" => context.new_rustfunc(PyFloatRef::mul), - "real" => context.new_property(PyFloatRef::real), + "__truediv__" => context.new_rustfunc(PyFloat::truediv), + "__rtruediv__" => context.new_rustfunc(PyFloat::rtruediv), + "__mul__" => context.new_rustfunc(PyFloat::mul), + "__rmul__" => context.new_rustfunc(PyFloat::mul), + "real" => context.new_property(PyFloat::real), "is_integer" => context.new_rustfunc(PyFloat::is_integer), - "as_integer_ratio" => context.new_rustfunc(PyFloatRef::as_integer_ratio) + "as_integer_ratio" => context.new_rustfunc(PyFloat::as_integer_ratio) }); } From 5afc5585648b8faf8661269a8bc7442a2adf0245 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 6 Apr 2019 23:47:01 -0500 Subject: [PATCH 101/223] Add id() method --- wasm/lib/src/vm_class.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 3f7bdef8a..c159f60e1 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -37,6 +37,13 @@ impl StoredVirtualMachine { held_objects: RefCell::new(Vec::new()), } } + + pub fn id(&self) -> &str { + self.vm + .wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") + } } // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! From 7ca3b77fcdeaf0349cab7071cee3ea1ca3f8783d Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 6 Apr 2019 23:55:30 -0500 Subject: [PATCH 102/223] Just a fn --- wasm/lib/src/vm_class.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c159f60e1..5efa7c849 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -37,13 +37,6 @@ impl StoredVirtualMachine { held_objects: RefCell::new(Vec::new()), } } - - pub fn id(&self) -> &str { - self.vm - .wasm_id - .as_ref() - .expect("VirtualMachine inside of WASM crate should have wasm_id set") - } } // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! @@ -53,6 +46,11 @@ thread_local! { static STORED_VMS: RefCell>> = RefCell::default(); } +pub fn get_vm_id(vm: &VirtualMachine) -> &str { + vm.wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") +} pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { STORED_VMS.with(|cell| { cell.borrow() @@ -62,10 +60,7 @@ pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc Weak { - let id = vm - .wasm_id - .as_ref() - .expect("VirtualMachine inside of WASM crate should have wasm_id set"); + let id = get_vm_id(vm); STORED_VMS .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } From 02a21cd07262ceadcefaa5be814aee5a57bdab03 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 09:46:45 +1200 Subject: [PATCH 103/223] Add builtin warning classes --- vm/src/builtins.rs | 13 +++++++++++++ vm/src/exceptions.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 210d12b49..27e024aaf 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -767,6 +767,19 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), + + // Warnings + "Warning" => ctx.exceptions.warning.clone(), + "BytesWarning" => ctx.exceptions.bytes_warning.clone(), + "UnicodeWarning" => ctx.exceptions.unicode_warning.clone(), + "DeprecationWarning" => ctx.exceptions.deprecation_warning.clone(), + "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.clone(), + "FutureWarning" => ctx.exceptions.future_warning.clone(), + "ImportWarning" => ctx.exceptions.import_warning.clone(), + "SyntaxWarning" => ctx.exceptions.syntax_warning.clone(), + "ResourceWarning" => ctx.exceptions.resource_warning.clone(), + "RuntimeWarning" => ctx.exceptions.runtime_warning.clone(), + "UserWarning" => ctx.exceptions.user_warning.clone(), }); } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index cec4b3125..d81764e25 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -103,6 +103,18 @@ pub struct ExceptionZoo { pub type_error: PyClassRef, pub value_error: PyClassRef, pub zero_division_error: PyClassRef, + + pub warning: PyClassRef, + pub bytes_warning: PyClassRef, + pub unicode_warning: PyClassRef, + pub deprecation_warning: PyClassRef, + pub pending_deprecation_warning: PyClassRef, + pub future_warning: PyClassRef, + pub import_warning: PyClassRef, + pub syntax_warning: PyClassRef, + pub resource_warning: PyClassRef, + pub runtime_warning: PyClassRef, + pub user_warning: PyClassRef, } impl ExceptionZoo { @@ -130,6 +142,19 @@ impl ExceptionZoo { let file_not_found_error = create_type("FileNotFoundError", &type_type, &os_error); let permission_error = create_type("PermissionError", &type_type, &os_error); + let warning = create_type("Warning", &type_type, &exception_type); + let bytes_warning = create_type("BytesWarning", &type_type, &warning); + let unicode_warning = create_type("UnicodeWarning", &type_type, &warning); + let deprecation_warning = create_type("DeprecationWarning", &type_type, &warning); + let pending_deprecation_warning = + create_type("PendingDeprecationWarning", &type_type, &warning); + let future_warning = create_type("FutureWarning", &type_type, &warning); + let import_warning = create_type("ImportWarning", &type_type, &warning); + let syntax_warning = create_type("SyntaxWarning", &type_type, &warning); + let resource_warning = create_type("ResourceWarning", &type_type, &warning); + let runtime_warning = create_type("RuntimeWarning", &type_type, &warning); + let user_warning = create_type("UserWarning", &type_type, &warning); + ExceptionZoo { arithmetic_error, assertion_error, @@ -152,6 +177,17 @@ impl ExceptionZoo { type_error, value_error, zero_division_error, + warning, + bytes_warning, + unicode_warning, + deprecation_warning, + pending_deprecation_warning, + future_warning, + import_warning, + syntax_warning, + resource_warning, + runtime_warning, + user_warning, } } } From 32051c9d961dce2488c79541ba43b5ddd7e61598 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 7 Apr 2019 19:19:39 +1200 Subject: [PATCH 104/223] Implement copy and update dict methods --- tests/snippets/dict.py | 16 ++++++++++++ vm/src/dictdatatype.rs | 9 +++++-- vm/src/obj/objdict.rs | 58 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 69040d624..f924bb0e7 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -107,3 +107,19 @@ x = {} x['a'] = 2 x['b'] = 10 assert list(x) == ['a', 'b'] + +y = x.copy() +x['c'] = 12 +assert list(y) == ['a', 'b'] +assert y['a'] == 2 +assert y['b'] == 10 + +y.update({'c': 19, "d": -1, 'b': 12}) +assert list(y) == ['a', 'b', 'c', 'd'] +assert y['a'] == 2 +assert y['b'] == 12 +assert y['c'] == 19 +assert y['d'] == -1 + +y.update(y) +assert list(y) == ['a', 'b', 'c', 'd'] diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6641a1884..8a6a8df1b 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,6 +9,7 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; +#[derive(Clone)] pub struct Dict { size: usize, indices: HashMap, @@ -25,6 +26,7 @@ impl Default for Dict { } } +#[derive(Clone)] struct DictEntry { hash: usize, key: PyObjectRef, @@ -118,13 +120,16 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { + pub fn iter_items(&self) -> impl Iterator + '_ { self.entries .iter() .filter(|e| e.is_some()) .map(|e| e.as_ref().unwrap()) .map(|e| (e.key.clone(), e.value.clone())) - .collect() + } + + pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { + self.iter_items().collect() } /// Lookup the index for the given key. diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index d54f62bb7..e84f7b178 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,7 +3,8 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, + PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -44,13 +45,28 @@ impl PyDictRef { kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult { - let mut dict = DictContentType::default(); + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + // it's unfortunate that we can't abstract over RefCall, as we should be able to use dict + // directly here, but that would require generic associated types + PyDictRef::merge(&entries, dict_obj, kwargs, vm)?; + + PyDict { entries }.into_ref_with_type(vm, class) + } + + fn merge( + dict: &RefCell, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - for (key, value) in dict_obj.get_key_value_pairs() { - dict.insert(vm, &key, value)?; + let mut dict_borrowed = dict.borrow_mut(); + for (key, value) in dict_obj.entries.borrow().iter_items() { + dict_borrowed.insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -68,17 +84,15 @@ impl PyDictRef { if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - dict.insert(vm, &key, value)?; + dict.borrow_mut().insert(vm, &key, value)?; } } } + let mut dict_borrowed = dict.borrow_mut(); for (key, value) in kwargs.into_iter() { - dict.insert(vm, &vm.new_str(key), value)?; + dict_borrowed.insert(vm, &vm.new_str(key), value)?; } - PyDict { - entries: RefCell::new(dict), - } - .into_ref_with_type(vm, class) + Ok(()) } fn bool(self, _vm: &VirtualMachine) -> bool { @@ -209,6 +223,28 @@ impl PyDictRef { } } + fn copy(self, _vm: &VirtualMachine) -> PyDict { + PyDict { + entries: self.entries.clone(), + } + } + + fn update( + self, + mut dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let OptionalArg::Present(ref other) = dict_obj { + if self.is(other) { + // Updating yourself is a noop, and this avoids a borrow error + dict_obj = OptionalArg::Missing; + } + } + + PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) + } + /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); @@ -266,5 +302,7 @@ pub fn init(context: &PyContext) { // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::iter), "get" => context.new_rustfunc(PyDictRef::get), + "copy" => context.new_rustfunc(PyDictRef::copy), + "update" => context.new_rustfunc(PyDictRef::update), }); } From 7e965c7cc8b331c11a339537161aee841b5a6ad0 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 10:12:57 +0200 Subject: [PATCH 105/223] check encoding in new --- vm/src/obj/objbytes.rs | 71 +++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 26de25010..1b06a9641 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -21,7 +21,9 @@ use crate::vm::VirtualMachine; use super::objbyteinner::PyByteInner; use super::objint; use super::objiter; +use super::objstr; use super::objtype::PyClassRef; +use std::clone::Clone; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ @@ -125,13 +127,24 @@ impl PyBytesRef { vm: &VirtualMachine, ) -> PyResult { // Create bytes data: + if let OptionalArg::Present(enc) = enc_option { if let OptionalArg::Present(eval) = val_option { - if objtype::isinstance(&eval, &vm.ctx.str_type()) - && objtype::isinstance(&enc, &vm.ctx.str_type()) - { - //return Ok(PyBytes::new(vec![1]).into_ref_with_type(vm, cls.clone())); - return Err(vm.new_type_error("Ok les 2 sont sstr".to_string())); + if let Ok(input) = eval.downcast::() { + if let Ok(encoding) = enc.downcast::() { + if encoding.value.to_lowercase() == "utf8".to_string() + || encoding.value.to_lowercase() == "utf-8".to_string() + { + return PyBytes::new(input.value.as_bytes().to_vec()) + .into_ref_with_type(vm, cls.clone()); + } else { + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error + ); + } + } else { + return Err(vm.new_type_error("encodin is not string".to_string())); + } } else { return Err(vm.new_type_error(format!( "bytes() argument 2 must be str, not {}", @@ -141,31 +154,31 @@ impl PyBytesRef { } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } - } - - let value = if let OptionalArg::Present(ival) = val_option { - println!("{:?}", ival); - match_class!(ival.clone(), - _i @ PyInt => { - let size = objint::get_value(&ival).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, - _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _l @ PyString=> load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _obj => {return Err(vm.new_type_error(format!( - "int() argument must be a string or a number, not " - )));} - ) } else { - Ok(vec![]) - }; - match value { - Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), - Err(err) => Err(err), + let value = if let OptionalArg::Present(ival) = val_option { + println!("{:?}", ival); + match_class!(ival.clone(), + _i @ PyInt => { + let size = objint::get_value(&ival).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _l @ PyString=> {return Err(vm.new_type_error(format!( + "string argument without an encoding" + )));}, + _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), + _obj => {} + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), + Err(err) => Err(err), + } } } From f03a164c146d15b3016c2a8e08d14f3df6b16d5f Mon Sep 17 00:00:00 2001 From: andrew Date: Sun, 7 Apr 2019 12:22:10 +0200 Subject: [PATCH 106/223] use a code generator for not implemented tests --- tests/README.md | 6 +- tests/generator/not_impl_footer.txt | 12 + tests/generator/not_impl_header.txt | 2 + tests/not_impl_gen.py | 33 + tests/snippets/whats_left_to_implement.py | 873 ---------------------- 5 files changed, 51 insertions(+), 875 deletions(-) create mode 100644 tests/generator/not_impl_footer.txt create mode 100644 tests/generator/not_impl_header.txt create mode 100644 tests/not_impl_gen.py delete mode 100644 tests/snippets/whats_left_to_implement.py diff --git a/tests/README.md b/tests/README.md index a225ba3b6..ba24ab8bc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,3 @@ - # Test snippets This directory contains two sets of test snippets which can be run in @@ -6,6 +5,10 @@ Python. The `snippets/` directory contains functional tests, and the `benchmarks/` directory contains snippets for use in benchmarking RustPython's performance. +## Generates the test for not implemented methods + +run using cpython not_impl_gen.py it automatically generate a +test snippet to check not yet implemented methods ## Running with CPython + RustPython @@ -15,4 +18,3 @@ compilation to bytecode. When this is done, run the bytecode with rustpython. ## Running with RustPython The other option is to run all snippets with RustPython. - diff --git a/tests/generator/not_impl_footer.txt b/tests/generator/not_impl_footer.txt new file mode 100644 index 000000000..5cd500999 --- /dev/null +++ b/tests/generator/not_impl_footer.txt @@ -0,0 +1,12 @@ +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] + +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: + print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_header.txt new file mode 100644 index 000000000..cd6f8f5a1 --- /dev/null +++ b/tests/generator/not_impl_header.txt @@ -0,0 +1,2 @@ +import platform + diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py new file mode 100644 index 000000000..2e20f5609 --- /dev/null +++ b/tests/not_impl_gen.py @@ -0,0 +1,33 @@ +objects = [ + bool, + bytearray, + bytes, + complex, + dict, + float, + frozenset, + int, + iter, + list, + memoryview, + range, + set, + str, + tuple, + object +] + +header = open("generator/not_impl_header.txt") +footer = open("generator/not_impl_footer.txt") +output = open("snippets/whats_left_to_implement.py", "w") + +output.write(header.read()) +output.write("expected_methods = {\n") + +for obj in objects: + output.write(f" '{obj.__name__}': ({obj.__name__}, [\n") + output.write("\n".join(f" '{attr}'," for attr in dir(obj))) + output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n")) + +output.write("}\n\n") +output.write(footer.read()) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py deleted file mode 100644 index 6abade9df..000000000 --- a/tests/snippets/whats_left_to_implement.py +++ /dev/null @@ -1,873 +0,0 @@ -import platform - -expected_methods = { - 'bool': (bool, [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', - ]), - 'bytearray': (bytearray, [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', - ]), - 'bytes': (bytes, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', - ]), - 'complex': (complex, [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', - ]), - 'dict': (dict, [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', - ]), - 'float': (float, [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', - ]), - 'frozenset': (frozenset, [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', - ]), - 'int': (int, [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', - ]), - 'iter': ([].__iter__(), [ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', - ]), - 'list': (list, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', - ]), - 'memoryview': (memoryview, [ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', - ]), - 'range': (range, [ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', - ]), - 'set': (set, [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', - ]), - 'str': (str, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' - ]), - 'tuple': (tuple, [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - ]), - 'object': (object, [ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' - ]), -} - -not_implemented = [(name, method) - for name, (val, methods) in expected_methods.items() - for method in methods - if not hasattr(val, method)] - -for r in not_implemented: - print(r[0], ".", r[1], sep="") -if not not_implemented: - print("Not much \\o/") - -if platform.python_implementation() == "CPython": - assert len(not_implemented) == 0, "CPython should have all the methods" From 2eb8e7bf2b49ba578f0180588a5e8592d38c5fd0 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 18:55:35 +0200 Subject: [PATCH 107/223] new ok for pybytes --- tests/snippets/bytes.py | 18 ++++++++++++ vm/src/obj/objbytes.rs | 61 ++++++++++++++++++----------------------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 19ebef68a..1a4b43dfb 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -1,3 +1,21 @@ +from testutils import assertRaises + +# new +assert bytes([1,2,3]) +assert bytes((1,2,3)) +assert bytes(range(4)) +assert b'bla' +assert bytes(3) +assert bytes("bla", "utf8") +try: + bytes("bla") +except TypeError: + assert True +else: + assert False + +# + assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented assert b'foobar'.__gt__(2) == NotImplemented diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 1b06a9641..74795088f 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -97,26 +97,6 @@ extend_class!(context, bytesiterator_type, { });*/ //} -fn load_byte( - elements: PyResult>, - vm: &VirtualMachine, -) -> Result, PyObjectRef> { - if let Ok(value) = elements { - let mut data_bytes = vec![]; - for elem in value.iter() { - let v = objint::to_int(vm, &elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); - } - } - Ok(data_bytes) - } else { - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) - } -} - #[pyimpl(__inside_vm)] impl PyBytesRef { #[pymethod(name = "__new__")] @@ -126,14 +106,14 @@ impl PyBytesRef { enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - // Create bytes data: - + // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = enc_option { if let OptionalArg::Present(eval) = val_option { if let Ok(input) = eval.downcast::() { - if let Ok(encoding) = enc.downcast::() { + if let Ok(encoding) = enc.clone().downcast::() { if encoding.value.to_lowercase() == "utf8".to_string() || encoding.value.to_lowercase() == "utf-8".to_string() + // TODO: different encoding { return PyBytes::new(input.value.as_bytes().to_vec()) .into_ref_with_type(vm, cls.clone()); @@ -143,23 +123,23 @@ impl PyBytesRef { ); } } else { - return Err(vm.new_type_error("encodin is not string".to_string())); + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); } } else { - return Err(vm.new_type_error(format!( - "bytes() argument 2 must be str, not {}", - enc.class().name - ))); + return Err(vm.new_type_error("encoding without a string argument".to_string())); } } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } + // On ly one argument } else { let value = if let OptionalArg::Present(ival) = val_option { - println!("{:?}", ival); match_class!(ival.clone(), - _i @ PyInt => { - let size = objint::get_value(&ival).to_usize().unwrap(); + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); let mut res: Vec = Vec::with_capacity(size); for _ in 0..size { res.push(0) @@ -168,9 +148,22 @@ impl PyBytesRef { _l @ PyString=> {return Err(vm.new_type_error(format!( "string argument without an encoding" )));}, - _j @ PyList => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _k @ PyTuple => load_byte(vm.extract_elements(&ival), vm).or_else(|x| {return Err(x) }), - _obj => {} + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)));}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } ) } else { Ok(vec![]) From a0aa88d2fb3aba561e0a085a14f618f7008f1f92 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 20:02:27 +0300 Subject: [PATCH 108/223] clear return implicit None --- vm/src/obj/objset.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 6c67312fb..c3c580bc4 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -290,9 +290,8 @@ impl PySetInner { perform_action_with_hash(vm, &mut self.elements, &item, &discard) } - fn clear(&mut self, vm: &VirtualMachine) -> PyResult { + fn clear(&mut self) -> () { self.elements.clear(); - Ok(vm.get_none()) } fn pop(&mut self, vm: &VirtualMachine) -> PyResult { @@ -498,8 +497,8 @@ impl PySetRef { self.inner.borrow_mut().discard(&item, vm) } - fn clear(self, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().clear(vm) + fn clear(self, _vm: &VirtualMachine) -> () { + self.inner.borrow_mut().clear() } fn pop(self, vm: &VirtualMachine) -> PyResult { From aec33634ca1323b68fbbc8ae75deda857b0e10b2 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:27:03 -0700 Subject: [PATCH 109/223] int: take self by ref --- vm/src/obj/objint.rs | 198 +++++++++++++++++++++++++------------------ 1 file changed, 115 insertions(+), 83 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 3c0c3b707..706b4ecfa 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -95,12 +95,8 @@ impl_try_from_object_int!( (u64, to_u64), ); -impl PyIntRef { - fn pass_value(self, _vm: &VirtualMachine) -> Self { - self - } - - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +impl PyInt { + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) } else { @@ -108,7 +104,7 @@ impl PyIntRef { } } - fn ne(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) } else { @@ -116,7 +112,7 @@ impl PyIntRef { } } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) } else { @@ -124,7 +120,7 @@ impl PyIntRef { } } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) } else { @@ -132,7 +128,7 @@ impl PyIntRef { } } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) } else { @@ -140,7 +136,7 @@ impl PyIntRef { } } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) } else { @@ -148,7 +144,7 @@ impl PyIntRef { } } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) } else { @@ -156,7 +152,7 @@ impl PyIntRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) } else { @@ -164,7 +160,7 @@ impl PyIntRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) } else { @@ -172,7 +168,7 @@ impl PyIntRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) } else { @@ -180,7 +176,7 @@ impl PyIntRef { } } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) } else { @@ -188,7 +184,7 @@ impl PyIntRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) } else { @@ -196,7 +192,7 @@ impl PyIntRef { } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -209,7 +205,7 @@ impl PyIntRef { } } - fn lshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -228,7 +224,7 @@ impl PyIntRef { } } - fn rshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -247,7 +243,7 @@ impl PyIntRef { } } - fn xor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) } else { @@ -255,7 +251,7 @@ impl PyIntRef { } } - fn rxor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) ^ (&self.value)) } else { @@ -263,7 +259,7 @@ impl PyIntRef { } } - fn or(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) } else { @@ -271,7 +267,7 @@ impl PyIntRef { } } - fn and(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); vm.ctx.new_int((&self.value) & v2) @@ -280,7 +276,7 @@ impl PyIntRef { } } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); vm.ctx.new_int(self.value.pow(v2)) @@ -292,7 +288,7 @@ impl PyIntRef { } } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -305,7 +301,7 @@ impl PyIntRef { } } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -321,37 +317,65 @@ impl PyIntRef { } } - fn neg(self, _vm: &VirtualMachine) -> BigInt { + fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } - fn hash(self, _vm: &VirtualMachine) -> u64 { + fn hash(&self, _vm: &VirtualMachine) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() } - fn abs(self, _vm: &VirtualMachine) -> BigInt { + fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } - fn round(self, _precision: OptionalArg, _vm: &VirtualMachine) -> Self { - self + fn round( + zelf: PyRef, + _precision: OptionalArg, + _vm: &VirtualMachine, + ) -> PyIntRef { + zelf } - fn float(self, _vm: &VirtualMachine) -> f64 { - self.value.to_f64().unwrap() + fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf } - fn invert(self, _vm: &VirtualMachine) -> BigInt { + fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn format(self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { + fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -359,15 +383,23 @@ impl PyIntRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } - fn bit_length(self, _vm: &VirtualMachine) -> usize { + fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } - fn imag(self, _vm: &VirtualMachine) -> usize { + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } } @@ -484,49 +516,49 @@ Base 0 means to interpret the base from the string as an integer literal. let int_type = &context.int_type; extend_class!(context, int_type, { "__doc__" => context.new_str(int_doc.to_string()), - "__eq__" => context.new_rustfunc(PyIntRef::eq), - "__ne__" => context.new_rustfunc(PyIntRef::ne), - "__lt__" => context.new_rustfunc(PyIntRef::lt), - "__le__" => context.new_rustfunc(PyIntRef::le), - "__gt__" => context.new_rustfunc(PyIntRef::gt), - "__ge__" => context.new_rustfunc(PyIntRef::ge), - "__abs__" => context.new_rustfunc(PyIntRef::abs), - "__add__" => context.new_rustfunc(PyIntRef::add), - "__radd__" => context.new_rustfunc(PyIntRef::add), - "__and__" => context.new_rustfunc(PyIntRef::and), - "__divmod__" => context.new_rustfunc(PyIntRef::divmod), - "__float__" => context.new_rustfunc(PyIntRef::float), - "__round__" => context.new_rustfunc(PyIntRef::round), - "__ceil__" => context.new_rustfunc(PyIntRef::pass_value), - "__floor__" => context.new_rustfunc(PyIntRef::pass_value), - "__index__" => context.new_rustfunc(PyIntRef::pass_value), - "__trunc__" => context.new_rustfunc(PyIntRef::pass_value), - "__int__" => context.new_rustfunc(PyIntRef::pass_value), - "__floordiv__" => context.new_rustfunc(PyIntRef::floordiv), - "__hash__" => context.new_rustfunc(PyIntRef::hash), - "__lshift__" => context.new_rustfunc(PyIntRef::lshift), - "__rshift__" => context.new_rustfunc(PyIntRef::rshift), + "__eq__" => context.new_rustfunc(PyInt::eq), + "__ne__" => context.new_rustfunc(PyInt::ne), + "__lt__" => context.new_rustfunc(PyInt::lt), + "__le__" => context.new_rustfunc(PyInt::le), + "__gt__" => context.new_rustfunc(PyInt::gt), + "__ge__" => context.new_rustfunc(PyInt::ge), + "__abs__" => context.new_rustfunc(PyInt::abs), + "__add__" => context.new_rustfunc(PyInt::add), + "__radd__" => context.new_rustfunc(PyInt::add), + "__and__" => context.new_rustfunc(PyInt::and), + "__divmod__" => context.new_rustfunc(PyInt::divmod), + "__float__" => context.new_rustfunc(PyInt::float), + "__round__" => context.new_rustfunc(PyInt::round), + "__ceil__" => context.new_rustfunc(PyInt::ceil), + "__floor__" => context.new_rustfunc(PyInt::floor), + "__index__" => context.new_rustfunc(PyInt::index), + "__trunc__" => context.new_rustfunc(PyInt::trunc), + "__int__" => context.new_rustfunc(PyInt::int), + "__floordiv__" => context.new_rustfunc(PyInt::floordiv), + "__hash__" => context.new_rustfunc(PyInt::hash), + "__lshift__" => context.new_rustfunc(PyInt::lshift), + "__rshift__" => context.new_rustfunc(PyInt::rshift), "__new__" => context.new_rustfunc(int_new), - "__mod__" => context.new_rustfunc(PyIntRef::mod_), - "__mul__" => context.new_rustfunc(PyIntRef::mul), - "__rmul__" => context.new_rustfunc(PyIntRef::mul), - "__or__" => context.new_rustfunc(PyIntRef::or), - "__neg__" => context.new_rustfunc(PyIntRef::neg), - "__pos__" => context.new_rustfunc(PyIntRef::pass_value), - "__pow__" => context.new_rustfunc(PyIntRef::pow), - "__repr__" => context.new_rustfunc(PyIntRef::repr), - "__sub__" => context.new_rustfunc(PyIntRef::sub), - "__rsub__" => context.new_rustfunc(PyIntRef::rsub), - "__format__" => context.new_rustfunc(PyIntRef::format), - "__truediv__" => context.new_rustfunc(PyIntRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyIntRef::rtruediv), - "__xor__" => context.new_rustfunc(PyIntRef::xor), - "__rxor__" => context.new_rustfunc(PyIntRef::rxor), - "__bool__" => context.new_rustfunc(PyIntRef::bool), - "__invert__" => context.new_rustfunc(PyIntRef::invert), - "bit_length" => context.new_rustfunc(PyIntRef::bit_length), - "conjugate" => context.new_rustfunc(PyIntRef::pass_value), - "real" => context.new_property(PyIntRef::pass_value), - "imag" => context.new_property(PyIntRef::imag) + "__mod__" => context.new_rustfunc(PyInt::mod_), + "__mul__" => context.new_rustfunc(PyInt::mul), + "__rmul__" => context.new_rustfunc(PyInt::mul), + "__or__" => context.new_rustfunc(PyInt::or), + "__neg__" => context.new_rustfunc(PyInt::neg), + "__pos__" => context.new_rustfunc(PyInt::pos), + "__pow__" => context.new_rustfunc(PyInt::pow), + "__repr__" => context.new_rustfunc(PyInt::repr), + "__sub__" => context.new_rustfunc(PyInt::sub), + "__rsub__" => context.new_rustfunc(PyInt::rsub), + "__format__" => context.new_rustfunc(PyInt::format), + "__truediv__" => context.new_rustfunc(PyInt::truediv), + "__rtruediv__" => context.new_rustfunc(PyInt::rtruediv), + "__xor__" => context.new_rustfunc(PyInt::xor), + "__rxor__" => context.new_rustfunc(PyInt::rxor), + "__bool__" => context.new_rustfunc(PyInt::bool), + "__invert__" => context.new_rustfunc(PyInt::invert), + "bit_length" => context.new_rustfunc(PyInt::bit_length), + "conjugate" => context.new_rustfunc(PyInt::conjugate), + "real" => context.new_property(PyInt::real), + "imag" => context.new_property(PyInt::imag) }); } From e70f174eb6f2802f5fb935a82a8b387228ffd4ae Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:35:39 -0700 Subject: [PATCH 110/223] str: take self by ref --- vm/src/obj/objstr.rs | 117 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 62cc923b1..0e16a8b3b 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -67,7 +67,7 @@ impl TryIntoRef for &str { } #[pyimpl(__inside_vm)] -impl PyStringRef { +impl PyString { // TODO: should with following format // class str(object='') // class str(object=b'', encoding='utf-8', errors='strict') @@ -89,7 +89,7 @@ impl PyStringRef { } } #[pymethod(name = "__add__")] - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn add(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) } else { @@ -98,12 +98,12 @@ impl PyStringRef { } #[pymethod(name = "__bool__")] - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() } #[pymethod(name = "__eq__")] - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) } else { @@ -112,17 +112,17 @@ impl PyStringRef { } #[pymethod(name = "__contains__")] - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + fn contains(&self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } #[pymethod(name = "__getitem__")] - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } #[pymethod(name = "__gt__")] - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn gt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) } else { @@ -131,7 +131,7 @@ impl PyStringRef { } #[pymethod(name = "__ge__")] - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ge(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) } else { @@ -140,7 +140,7 @@ impl PyStringRef { } #[pymethod(name = "__lt__")] - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn lt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) } else { @@ -149,7 +149,7 @@ impl PyStringRef { } #[pymethod(name = "__le__")] - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn le(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) } else { @@ -158,19 +158,19 @@ impl PyStringRef { } #[pymethod(name = "__hash__")] - fn hash(self, _vm: &VirtualMachine) -> usize { + fn hash(&self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() as usize } #[pymethod(name = "__len__")] - fn len(self, _vm: &VirtualMachine) -> usize { + fn len(&self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } #[pymethod(name = "__mul__")] - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); @@ -185,12 +185,12 @@ impl PyStringRef { } #[pymethod(name = "__str__")] - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + fn str(zelf: PyRef, _vm: &VirtualMachine) -> PyStringRef { + zelf } #[pymethod(name = "__repr__")] - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' @@ -221,30 +221,30 @@ impl PyStringRef { } #[pymethod] - fn lower(self, _vm: &VirtualMachine) -> String { + fn lower(&self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower #[pymethod] - fn casefold(self, _vm: &VirtualMachine) -> String { + fn casefold(&self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } #[pymethod] - fn upper(self, _vm: &VirtualMachine) -> String { + fn upper(&self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } #[pymethod] - fn capitalize(self, _vm: &VirtualMachine) -> String { + fn capitalize(&self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } #[pymethod] fn split( - self, + &self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, @@ -266,7 +266,7 @@ impl PyStringRef { #[pymethod] fn rsplit( - self, + &self, pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, @@ -287,23 +287,23 @@ impl PyStringRef { } #[pymethod] - fn strip(self, _vm: &VirtualMachine) -> String { + fn strip(&self, _vm: &VirtualMachine) -> String { self.value.trim().to_string() } #[pymethod] - fn lstrip(self, _vm: &VirtualMachine) -> String { + fn lstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_start().to_string() } #[pymethod] - fn rstrip(self, _vm: &VirtualMachine) -> String { + fn rstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_end().to_string() } #[pymethod] fn endswith( - self, + &self, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -318,7 +318,7 @@ impl PyStringRef { #[pymethod] fn startswith( - self, + &self, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -332,17 +332,17 @@ impl PyStringRef { } #[pymethod] - fn isalnum(self, _vm: &VirtualMachine) -> bool { + fn isalnum(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } #[pymethod] - fn isnumeric(self, _vm: &VirtualMachine) -> bool { + fn isnumeric(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } #[pymethod] - fn isdigit(self, _vm: &VirtualMachine) -> bool { + fn isdigit(&self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, @@ -359,7 +359,7 @@ impl PyStringRef { } #[pymethod] - fn isdecimal(self, _vm: &VirtualMachine) -> bool { + fn isdecimal(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -397,12 +397,12 @@ impl PyStringRef { } #[pymethod] - fn title(self, _vm: &VirtualMachine) -> String { + fn title(&self, _vm: &VirtualMachine) -> String { make_title(&self.value) } #[pymethod] - fn swapcase(self, _vm: &VirtualMachine) -> String { + fn swapcase(&self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char @@ -418,13 +418,13 @@ impl PyStringRef { } #[pymethod] - fn isalpha(self, _vm: &VirtualMachine) -> bool { + fn isalpha(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } #[pymethod] fn replace( - self, + &self, old: PyStringRef, new: PyStringRef, num: OptionalArg, @@ -439,12 +439,12 @@ impl PyStringRef { // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower #[pymethod] - fn isspace(self, _vm: &VirtualMachine) -> bool { + fn isspace(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } #[pymethod] - fn isupper(self, _vm: &VirtualMachine) -> bool { + fn isupper(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -454,7 +454,7 @@ impl PyStringRef { } #[pymethod] - fn islower(self, _vm: &VirtualMachine) -> bool { + fn islower(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -464,13 +464,13 @@ impl PyStringRef { } #[pymethod] - fn isascii(self, _vm: &VirtualMachine) -> bool { + fn isascii(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet #[pymethod] - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + fn splitlines(&self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value .split('\n') @@ -480,7 +480,7 @@ impl PyStringRef { } #[pymethod] - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn join(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { @@ -496,7 +496,7 @@ impl PyStringRef { #[pymethod] fn find( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -515,7 +515,7 @@ impl PyStringRef { #[pymethod] fn rfind( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -534,7 +534,7 @@ impl PyStringRef { #[pymethod] fn index( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -553,7 +553,7 @@ impl PyStringRef { #[pymethod] fn rindex( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -571,7 +571,7 @@ impl PyStringRef { } #[pymethod] - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + fn partition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -590,7 +590,7 @@ impl PyStringRef { } #[pymethod] - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + fn rpartition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -610,7 +610,7 @@ impl PyStringRef { } #[pymethod] - fn istitle(self, _vm: &VirtualMachine) -> bool { + fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -620,7 +620,7 @@ impl PyStringRef { #[pymethod] fn count( - self, + &self, sub: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -635,7 +635,7 @@ impl PyStringRef { } #[pymethod] - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + fn zfill(&self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { value.to_string() @@ -644,7 +644,10 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + fn get_fill_char<'a>( + rep: &'a OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<&'a str> { let rep_str = match rep { OptionalArg::Present(ref st) => &st.value, OptionalArg::Missing => " ", @@ -660,7 +663,7 @@ impl PyStringRef { #[pymethod] fn ljust( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -672,7 +675,7 @@ impl PyStringRef { #[pymethod] fn rjust( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -684,7 +687,7 @@ impl PyStringRef { #[pymethod] fn center( - self, + &self, len: usize, rep: OptionalArg, vm: &VirtualMachine, @@ -702,7 +705,7 @@ impl PyStringRef { } #[pymethod] - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + fn expandtabs(&self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; @@ -726,7 +729,7 @@ impl PyStringRef { } #[pymethod] - fn isidentifier(self, _vm: &VirtualMachine) -> bool { + fn isidentifier(&self, _vm: &VirtualMachine) -> bool { let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) @@ -769,7 +772,7 @@ impl IntoPyObject for &String { } pub fn init(ctx: &PyContext) { - PyStringRef::extend_class(ctx, &ctx.str_type); + PyString::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { From ea61599da46f302079e633ee80ea8acb1016c6b0 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 09:41:52 -0700 Subject: [PATCH 111/223] range: take self by ref --- vm/src/obj/objrange.rs | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 71adb4118..2820dd9b4 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -101,20 +101,20 @@ pub fn init(context: &PyContext) { When step is given, it specifies the increment (or decrement)."; extend_class!(context, range_type, { - "__bool__" => context.new_rustfunc(PyRangeRef::bool), - "__contains__" => context.new_rustfunc(PyRangeRef::contains), + "__bool__" => context.new_rustfunc(PyRange::bool), + "__contains__" => context.new_rustfunc(PyRange::contains), "__doc__" => context.new_str(range_doc.to_string()), - "__getitem__" => context.new_rustfunc(PyRangeRef::getitem), - "__iter__" => context.new_rustfunc(PyRangeRef::iter), - "__len__" => context.new_rustfunc(PyRangeRef::len), + "__getitem__" => context.new_rustfunc(PyRange::getitem), + "__iter__" => context.new_rustfunc(PyRange::iter), + "__len__" => context.new_rustfunc(PyRange::len), "__new__" => context.new_rustfunc(range_new), - "__repr__" => context.new_rustfunc(PyRangeRef::repr), - "__reversed__" => context.new_rustfunc(PyRangeRef::reversed), - "count" => context.new_rustfunc(PyRangeRef::count), - "index" => context.new_rustfunc(PyRangeRef::index), - "start" => context.new_property(PyRangeRef::start), - "stop" => context.new_property(PyRangeRef::stop), - "step" => context.new_property(PyRangeRef::step), + "__repr__" => context.new_rustfunc(PyRange::repr), + "__reversed__" => context.new_rustfunc(PyRange::reversed), + "count" => context.new_rustfunc(PyRange::count), + "index" => context.new_rustfunc(PyRange::index), + "start" => context.new_property(PyRange::start), + "stop" => context.new_property(PyRange::stop), + "step" => context.new_property(PyRange::step), }); let rangeiterator_type = &context.rangeiterator_type; @@ -126,7 +126,7 @@ pub fn init(context: &PyContext) { type PyRangeRef = PyRef; -impl PyRangeRef { +impl PyRange { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { start: PyInt::new(BigInt::zero()).into_ref(vm), @@ -153,26 +153,26 @@ impl PyRangeRef { .into_ref_with_type(vm, cls) } - fn start(self, _vm: &VirtualMachine) -> PyIntRef { + fn start(&self, _vm: &VirtualMachine) -> PyIntRef { self.start.clone() } - fn stop(self, _vm: &VirtualMachine) -> PyIntRef { + fn stop(&self, _vm: &VirtualMachine) -> PyIntRef { self.stop.clone() } - fn step(self, _vm: &VirtualMachine) -> PyIntRef { + fn step(&self, _vm: &VirtualMachine) -> PyIntRef { self.step.clone() } - fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyRangeIterator { + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIterator { PyRangeIterator { position: Cell::new(0), - range: self, + range: zelf, } } - fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyRangeIterator { + fn reversed(&self, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -204,7 +204,7 @@ impl PyRangeRef { } } - fn len(self, _vm: &VirtualMachine) -> PyInt { + fn len(&self, _vm: &VirtualMachine) -> PyInt { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -217,7 +217,7 @@ impl PyRangeRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { if self.step.as_bigint().is_one() { format!("range({}, {})", self.start, self.stop) } else { @@ -225,11 +225,11 @@ impl PyRangeRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.is_empty() } - fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { + fn contains(&self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { match self.offset(int.as_bigint()) { Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), @@ -240,7 +240,7 @@ impl PyRangeRef { } } - fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { match self.index_of(int.as_bigint()) { Some(idx) => Ok(PyInt::new(idx)), @@ -251,7 +251,7 @@ impl PyRangeRef { } } - fn count(self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { + fn count(&self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { if let Ok(int) = item.downcast::() { if self.index_of(int.as_bigint()).is_some() { PyInt::new(1) @@ -263,7 +263,7 @@ impl PyRangeRef { } } - fn getitem(self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { + fn getitem(&self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { RangeIndex::Int(index) => { if let Some(value) = self.get(index.as_bigint()) { @@ -314,10 +314,10 @@ impl PyRangeRef { fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let range = if args.args.len() <= 2 { let (cls, stop) = args.bind(vm)?; - PyRangeRef::new(cls, stop, vm) + PyRange::new(cls, stop, vm) } else { let (cls, start, stop, step) = args.bind(vm)?; - PyRangeRef::new_from(cls, start, stop, step, vm) + PyRange::new_from(cls, start, stop, step, vm) }?; Ok(range.into_object()) From 3bc1e3598c9999a4f1aca5ea72a3d4f749702317 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 21:26:20 +0300 Subject: [PATCH 112/223] Use PyIterable --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 48 +++++++++++++++++++------------------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 011716335..11d62944a 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -49,6 +49,7 @@ assert len(a) == 0 assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) +assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) @@ -181,6 +182,7 @@ assert b == a assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index c3c580bc4..46ba1db66 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,7 +8,9 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{ + PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, +}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -61,14 +63,13 @@ struct PySetInner { } impl PySetInner { - fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { let elements: HashMap = match iterable { OptionalArg::Missing => HashMap::new(), OptionalArg::Present(iterable) => { let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; } elements } @@ -175,9 +176,11 @@ impl PySetInner { ) } - fn union(&self, other: &PySetInner, _vm: &VirtualMachine) -> PyResult { + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.clone(); - elements.extend(other.elements.clone()); + for item in other.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; + } Ok(PySetInner { elements }) } @@ -302,10 +305,9 @@ impl PySetInner { } } - fn update(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut self.elements, &v)?; + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut self.elements, &item?)?; } Ok(vm.get_none()) } @@ -350,7 +352,7 @@ impl PySetInner { impl PySetRef { fn new( cls: PyClassRef, - iterable: OptionalArg, + iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { PySet { @@ -413,14 +415,10 @@ impl PySetRef { ) } - fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().union(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().union(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().union(other, vm)?), }, PySet::class(vm), None, @@ -505,12 +503,12 @@ impl PySetRef { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn ior(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(self.as_object().clone()) } - fn update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(vm.get_none()) } @@ -565,7 +563,7 @@ impl PySetRef { impl PyFrozenSetRef { fn new( cls: PyClassRef, - iterable: OptionalArg, + iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { PyFrozenSet { @@ -628,14 +626,10 @@ impl PyFrozenSetRef { ) } - fn union(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.union(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.union(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.union(other, vm)?, }, PyFrozenSet::class(vm), None, From f538a92007f375c07dd018f295fd34b101be4d4b Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 20:34:27 +0200 Subject: [PATCH 113/223] pybytes.__new__ ok --- vm/src/obj/objbyteinner.rs | 91 +++++++++++++++++++++++++++++++++++++- vm/src/obj/objbytes.rs | 69 ++--------------------------- 2 files changed, 92 insertions(+), 68 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 0607621f1..fd08e4b93 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,9 +1,96 @@ +use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyObjectRef; + +use crate::function::OptionalArg; + +use crate::vm::VirtualMachine; + +use crate::pyobject::{PyResult, TypeProtocol}; + +use crate::obj::objstr::PyString; + +use super::objint; +use crate::obj::objint::PyInt; +use num_traits::ToPrimitive; + #[derive(Debug, Default, Clone)] pub struct PyByteInner { pub elements: Vec, } + impl PyByteInner { - pub fn new(data: Vec) -> Self { - PyByteInner { elements: data } + pub fn new( + val_option: OptionalArg, + enc_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // First handle bytes(string, encoding[, errors]) + if let OptionalArg::Present(enc) = enc_option { + if let OptionalArg::Present(eval) = val_option { + if let Ok(input) = eval.downcast::() { + if let Ok(encoding) = enc.clone().downcast::() { + if encoding.value.to_lowercase() == "utf8".to_string() + || encoding.value.to_lowercase() == "utf-8".to_string() + // TODO: different encoding + { + return Ok(PyByteInner { + elements: input.value.as_bytes().to_vec(), + }); + } else { + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error + ); + } + } else { + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + // On ly one argument + } else { + let value = if let OptionalArg::Present(ival) = val_option { + match_class!(ival.clone(), + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); + let mut res: Vec = Vec::with_capacity(size); + for _ in 0..size { + res.push(0) + } + Ok(res)}, + _l @ PyString=> {return Err(vm.new_type_error(format!( + "string argument without an encoding" + )));}, + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)));}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => Ok(PyByteInner { elements: val }), + Err(err) => Err(err), + } + } } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 74795088f..07e71e23c 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -106,73 +106,10 @@ impl PyBytesRef { enc_option: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - // First handle bytes(string, encoding[, errors]) - if let OptionalArg::Present(enc) = enc_option { - if let OptionalArg::Present(eval) = val_option { - if let Ok(input) = eval.downcast::() { - if let Ok(encoding) = enc.clone().downcast::() { - if encoding.value.to_lowercase() == "utf8".to_string() - || encoding.value.to_lowercase() == "utf-8".to_string() - // TODO: different encoding - { - return PyBytes::new(input.value.as_bytes().to_vec()) - .into_ref_with_type(vm, cls.clone()); - } else { - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error - ); - } - } else { - return Err(vm.new_type_error(format!( - "bytes() argument 2 must be str, not {}", - enc.class().name - ))); - } - } else { - return Err(vm.new_type_error("encoding without a string argument".to_string())); - } - } else { - return Err(vm.new_type_error("encoding without a string argument".to_string())); - } - // On ly one argument - } else { - let value = if let OptionalArg::Present(ival) = val_option { - match_class!(ival.clone(), - i @ PyInt => { - let size = objint::get_value(&i.into_object()).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, - _l @ PyString=> {return Err(vm.new_type_error(format!( - "string argument without an encoding" - )));}, - obj => { - let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( - "cannot convert {} object to bytes", obj.class().name)));}); - - let mut data_bytes = vec![]; - for elem in elements.unwrap(){ - let v = objint::to_int(vm, &elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); - } - - } - Ok(data_bytes) - } - ) - } else { - Ok(vec![]) - }; - match value { - Ok(val) => PyBytes::new(val).into_ref_with_type(vm, cls.clone()), - Err(err) => Err(err), - } + PyBytes { + inner: PyByteInner::new(val_option, enc_option, vm)?, } + .into_ref_with_type(vm, cls) } #[pymethod(name = "__repr__")] From c2e03ed34287ab9b9cd827b894b3d8fffa82efd1 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 7 Apr 2019 11:55:13 -0700 Subject: [PATCH 114/223] fix int.__float__ --- vm/src/obj/objint.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 706b4ecfa..de211668f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -347,8 +347,11 @@ impl PyInt { zelf } - fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { - zelf + fn float(&self, vm: &VirtualMachine) -> PyResult { + self.value + .to_f64() + .map(PyFloat::from) + .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) } fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { From fa369ff77964b53d34a40a3a481bc728c2e0be47 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 21:58:37 +0300 Subject: [PATCH 115/223] Limit non-operator versions to set and frozenset --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 11d62944a..174426aa2 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -53,6 +53,7 @@ assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) @@ -186,6 +187,7 @@ assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 46ba1db66..2aa00431b 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -9,7 +9,8 @@ use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -17,6 +18,7 @@ use super::objbool; use super::objint; use super::objiter; use super::objlist::PyListIterator; +use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] @@ -467,6 +469,10 @@ impl PySetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -636,6 +642,10 @@ impl PyFrozenSetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { @@ -732,6 +742,27 @@ enum SetCombineOperation { Difference, } +struct SetIterable { + iterable: PyIterable, +} + +impl TryFromObject for SetIterable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if objtype::issubclass(&obj.class(), &vm.ctx.set_type()) + || objtype::issubclass(&obj.class(), &vm.ctx.frozenset_type()) + { + Ok(SetIterable { + iterable: PyIterable::try_from_object(vm, obj)?, + }) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) + } + } +} + fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } @@ -757,7 +788,7 @@ pub fn init(context: &PyContext) { "issubset" => context.new_rustfunc(PySetRef::le), "issuperset" => context.new_rustfunc(PySetRef::ge), "union" => context.new_rustfunc(PySetRef::union), - "__or__" => context.new_rustfunc(PySetRef::union), + "__or__" => context.new_rustfunc(PySetRef::or), "intersection" => context.new_rustfunc(PySetRef::intersection), "__and__" => context.new_rustfunc(PySetRef::intersection), "difference" => context.new_rustfunc(PySetRef::difference), @@ -798,7 +829,7 @@ pub fn init(context: &PyContext) { "issubset" => context.new_rustfunc(PyFrozenSetRef::le), "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), "union" => context.new_rustfunc(PyFrozenSetRef::union), - "__or__" => context.new_rustfunc(PyFrozenSetRef::union), + "__or__" => context.new_rustfunc(PyFrozenSetRef::or), "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), "difference" => context.new_rustfunc(PyFrozenSetRef::difference), From e011e3f32721fa1e7b3cd749152956cde336a5ee Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 22:36:56 +0300 Subject: [PATCH 116/223] intersection and difference support iterable --- tests/snippets/set.py | 8 +++ vm/src/obj/objset.rs | 142 +++++++++++++++++++++--------------------- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 174426aa2..a7977f35d 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -57,15 +57,19 @@ assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) +assert set([1,2,3]).intersection([1,2]) == set([1,2]) assert set([1,2,3]) & set([4,5]) == set([]) assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) +assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) assert set([1,2,3]).difference(set([1,2])) == set([3]) assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) +assert set([1,2,3]).difference([1,2]) == set([3]) assert set([1,2,3]) - set([4,5]) == set([1,2,3]) assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) +assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) @@ -191,15 +195,19 @@ assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) +assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) +assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 2aa00431b..508400694 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -187,33 +187,27 @@ impl PySetInner { Ok(PySetInner { elements }) } - fn intersection(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { - self._combine_inner(other, vm, SetCombineOperation::Intersection) - } - - fn difference(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { - self._combine_inner(other, vm, SetCombineOperation::Difference) - } - - fn _combine_inner( - &self, - other: &PySetInner, - vm: &VirtualMachine, - op: SetCombineOperation, - ) -> PyResult { + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = HashMap::new(); - - for element in self.elements.iter() { - let value = other.contains(element.1.clone(), vm)?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), - }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); + for item in other.iter(vm)? { + if let Ok(obj) = item { + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + insert_into_set(vm, &mut elements, &obj)?; + } } } + Ok(PySetInner { elements }) + } + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + for item in other.iter(vm)? { + if let Ok(obj) = item { + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + remove_from_set(vm, &mut elements, &obj)?; + } + } + } Ok(PySetInner { elements }) } @@ -265,21 +259,7 @@ impl PySetInner { } fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut self.elements, &item, &remove) + remove_from_set(vm, &mut self.elements, &item) } fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -427,28 +407,20 @@ impl PySetRef { )) } - fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().intersection(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().intersection(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), }, PySet::class(vm), None, )) } - fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), }, PySet::class(vm), None, @@ -473,6 +445,14 @@ impl PySetRef { self.union(other.iterable, vm) } + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -642,32 +622,20 @@ impl PyFrozenSetRef { )) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { - self.union(other.iterable, vm) - } - - fn intersection(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.intersection(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.intersection(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.intersection(other, vm)?, }, PyFrozenSet::class(vm), None, )) } - fn difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.difference(other, vm)?, }, PyFrozenSet::class(vm), None, @@ -688,6 +656,18 @@ impl PyFrozenSetRef { )) } + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } @@ -737,6 +717,28 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } +fn remove_from_set( + vm: &VirtualMachine, + elements: &mut HashMap, + item: &PyObjectRef, +) -> PyResult { + fn remove( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + value: &PyObjectRef, + ) -> PyResult { + match elements.remove(&key) { + None => { + let item_str = format!("{:?}", value); + Err(vm.new_key_error(item_str)) + } + Some(_) => Ok(vm.get_none()), + } + } + perform_action_with_hash(vm, elements, item, &remove) +} + enum SetCombineOperation { Intersection, Difference, @@ -790,9 +792,9 @@ pub fn init(context: &PyContext) { "union" => context.new_rustfunc(PySetRef::union), "__or__" => context.new_rustfunc(PySetRef::or), "intersection" => context.new_rustfunc(PySetRef::intersection), - "__and__" => context.new_rustfunc(PySetRef::intersection), + "__and__" => context.new_rustfunc(PySetRef::and), "difference" => context.new_rustfunc(PySetRef::difference), - "__sub__" => context.new_rustfunc(PySetRef::difference), + "__sub__" => context.new_rustfunc(PySetRef::sub), "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), "__doc__" => context.new_str(set_doc.to_string()), @@ -831,9 +833,9 @@ pub fn init(context: &PyContext) { "union" => context.new_rustfunc(PyFrozenSetRef::union), "__or__" => context.new_rustfunc(PyFrozenSetRef::or), "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), - "__and__" => context.new_rustfunc(PyFrozenSetRef::intersection), + "__and__" => context.new_rustfunc(PyFrozenSetRef::and), "difference" => context.new_rustfunc(PyFrozenSetRef::difference), - "__sub__" => context.new_rustfunc(PyFrozenSetRef::difference), + "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), From 3ed8727ee5b8252941d19e6c7e318510826ae903 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:01:49 +0300 Subject: [PATCH 117/223] symmetric_difference support iterable --- tests/snippets/set.py | 4 +++ vm/src/obj/objset.rs | 69 ++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a7977f35d..42237dd2e 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -73,9 +73,11 @@ assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) +assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -211,9 +213,11 @@ assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) +assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: frozenset([[]])) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 508400694..f8c37fdd6 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -190,10 +190,9 @@ impl PySetInner { fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = HashMap::new(); for item in other.iter(vm)? { - if let Ok(obj) = item { - if objbool::get_value(&self.contains(obj.clone(), vm)?) { - insert_into_set(vm, &mut elements, &obj)?; - } + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + insert_into_set(vm, &mut elements, &obj)?; } } Ok(PySetInner { elements }) @@ -202,37 +201,27 @@ impl PySetInner { fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut elements = self.elements.clone(); for item in other.iter(vm)? { - if let Ok(obj) = item { - if objbool::get_value(&self.contains(obj.clone(), vm)?) { - remove_from_set(vm, &mut elements, &obj)?; - } + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + remove_from_set(vm, &mut elements, &obj)?; } } Ok(PySetInner { elements }) } - fn symmetric_difference( - &self, - other: &PySetInner, - vm: &VirtualMachine, - ) -> PyResult { - let mut elements = HashMap::new(); + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut new_inner = self.clone(); - for element in self.elements.iter() { - let value = other.contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); + for item in other.iter(vm)? { + let obj = item?; + if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + new_inner.add(&obj, vm)?; + } else { + new_inner.remove(&obj, vm)?; } } - for element in other.elements.iter() { - let value = self.contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - Ok(PySetInner { elements }) + Ok(new_inner) } fn iter(&self, vm: &VirtualMachine) -> PyListIterator { @@ -427,14 +416,10 @@ impl PySetRef { )) } - fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - inner: RefCell::new(match_class!(other, - set @ PySet => self.inner.borrow().symmetric_difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.borrow().symmetric_difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - )), + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), }, PySet::class(vm), None, @@ -453,6 +438,10 @@ impl PySetRef { self.difference(other.iterable, vm) } + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } @@ -642,14 +631,10 @@ impl PyFrozenSetRef { )) } - fn symmetric_difference(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PyFrozenSet { - inner: match_class!(other, - set @ PySet => self.inner.symmetric_difference(&set.inner.borrow(), vm)?, - frozen @ PyFrozenSet => self.inner.symmetric_difference(&frozen.inner, vm)?, - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ), + inner: self.inner.symmetric_difference(other, vm)?, }, PyFrozenSet::class(vm), None, @@ -668,6 +653,10 @@ impl PyFrozenSetRef { self.difference(other.iterable, vm) } + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + fn iter(self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } @@ -796,7 +785,7 @@ pub fn init(context: &PyContext) { "difference" => context.new_rustfunc(PySetRef::difference), "__sub__" => context.new_rustfunc(PySetRef::sub), "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PySetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PySetRef::xor), "__doc__" => context.new_str(set_doc.to_string()), "add" => context.new_rustfunc(PySetRef::add), "remove" => context.new_rustfunc(PySetRef::remove), @@ -837,7 +826,7 @@ pub fn init(context: &PyContext) { "difference" => context.new_rustfunc(PyFrozenSetRef::difference), "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PyFrozenSetRef::xor), "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), "__len__" => context.new_rustfunc(PyFrozenSetRef::len), "__doc__" => context.new_str(frozenset_doc.to_string()), From 848350d3347bac1bc404816a1dfefe0c908b7b3c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:06:42 +0300 Subject: [PATCH 118/223] ior support only set and frozenset --- tests/snippets/set.py | 2 ++ vm/src/obj/objset.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 42237dd2e..a44847f9b 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -119,6 +119,8 @@ a |= set([3,4,5]) assert a == set([1,2,3,4,5]) with assertRaises(TypeError): a |= 1 +with assertRaises(TypeError): + a |= [1,2,3] a = set([1,2,3]) a.intersection_update([2,3,4,5]) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index f8c37fdd6..888ffae53 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -478,8 +478,8 @@ impl PySetRef { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().update(iterable, vm)?; + fn ior(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } From 5ae921dc571273343d981a20f469ce2b64bc4303 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:26:59 +0300 Subject: [PATCH 119/223] update function support iterable --- tests/snippets/set.py | 6 +++ vm/src/obj/objset.rs | 106 +++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 59 deletions(-) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a44847f9b..393db0754 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -132,6 +132,8 @@ a &= set([2,3,4,5]) assert a == set([2,3]) with assertRaises(TypeError): a &= 1 +with assertRaises(TypeError): + a &= [1,2,3] a = set([1,2,3]) a.difference_update([3,4,5]) @@ -143,6 +145,8 @@ a -= set([3,4,5]) assert a == set([1,2]) with assertRaises(TypeError): a -= 1 +with assertRaises(TypeError): + a -= [1,2,3] a = set([1,2,3]) a.symmetric_difference_update([3,4,5]) @@ -154,6 +158,8 @@ a ^= set([3,4,5]) assert a == set([1,2,4,5]) with assertRaises(TypeError): a ^= 1 +with assertRaises(TypeError): + a ^= [1,2,3] # frozen set diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 888ffae53..4b60b5b5c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -16,7 +16,6 @@ use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter; use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; @@ -283,39 +282,41 @@ impl PySetInner { Ok(vm.get_none()) } - fn combine_update_inner( - &mut self, - iterable: &PyObjectRef, - vm: &VirtualMachine, - op: SetCombineOperation, - ) -> PyResult { - let elements = &mut self.elements; - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + let temp_inner = self.copy(); + self.clear(); + for item in iterable.iter(vm)? { + let obj = item?; + if objbool::get_value(&temp_inner.contains(obj.clone(), vm)?) { + self.add(&obj, vm)?; } } Ok(vm.get_none()) } - fn ixor(&mut self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let elements_original = self.elements.clone(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut self.elements, &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(&iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - self.elements.remove(&element.0.clone()); + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if objbool::get_value(&self.contains(obj.clone(), vm)?) { + self.remove(&obj, vm)?; } } + Ok(vm.get_none()) + } + fn symmetric_difference_update( + &mut self, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + self.add(&obj, vm)?; + } else { + self.remove(&obj, vm)?; + } + } Ok(vm.get_none()) } } @@ -488,49 +489,41 @@ impl PySetRef { Ok(vm.get_none()) } - fn intersection_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Intersection, - )?; + fn intersection_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().intersection_update(iterable, vm)?; Ok(vm.get_none()) } - fn iand(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Intersection, - )?; + fn iand(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .intersection_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } - fn difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Difference, - )?; + fn difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn isub(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().combine_update_inner( - &iterable, - vm, - SetCombineOperation::Difference, - )?; + fn isub(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .difference_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } - fn symmetric_difference_update(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().ixor(iterable, vm)?; + fn symmetric_difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn ixor(self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().ixor(iterable, vm)?; + fn ixor(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable.iterable, vm)?; Ok(self.as_object().clone()) } } @@ -728,11 +721,6 @@ fn remove_from_set( perform_action_with_hash(vm, elements, item, &remove) } -enum SetCombineOperation { - Intersection, - Difference, -} - struct SetIterable { iterable: PyIterable, } From ec98b4d6bf509abffe66d47ef43ef60f3bdc7248 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 22:36:13 +0200 Subject: [PATCH 120/223] repr len eq --- tests/snippets/bytes.py | 23 ++++++++++++++++++++++- vm/src/obj/objbyteinner.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 1a4b43dfb..85469b756 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -14,7 +14,28 @@ except TypeError: else: assert False -# +a = b"abcd" +b = b"ab" +c = b"abcd" + +# +# repr +assert repr(bytes([0, 1, 2])) == repr(b'\x00\x01\x02') +assert ( +repr(bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) +== "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'") +) +assert repr(b"abcd") == "b'abcd'" + +#len +assert len(bytes("abcdé", "utf8")) == 6 + +# +assert a == b"abcd" +# assert a > b +# assert a >= b +# assert b < a +# assert b <= a assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index fd08e4b93..8d32aeabf 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,4 +1,3 @@ -use crate::obj::objtype::PyClassRef; use crate::pyobject::PyObjectRef; use crate::function::OptionalArg; @@ -93,4 +92,33 @@ impl PyByteInner { } } } + + pub fn repr(&self) -> PyResult { + let mut res = String::with_capacity(self.elements.len()); + for i in self.elements.iter() { + match i { + 0..=8 => res.push_str(&format!("\\x0{}", i)), + 9 => res.push_str("\\t"), + 10 => res.push_str("\\n"), + 13 => res.push_str("\\r"), + 32..=126 => res.push(*(i) as char), + _ => res.push_str(&format!("\\x{:x}", i)), + } + } + Ok(res) + } + + pub fn len(&self) -> usize { + self.elements.len() + } + + pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements == other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } } +// TODO +// fix b"é" not allowed should be bytes("é", "utf8") From dddf9fee3906348063b9298b4621ae32adfd0e72 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sun, 7 Apr 2019 22:42:47 +0200 Subject: [PATCH 121/223] gt lt ge le --- tests/snippets/bytes.py | 8 +- vm/src/obj/objbyteinner.rs | 32 +++++++ vm/src/obj/objbytes.rs | 169 +++++++++++++++---------------------- 3 files changed, 104 insertions(+), 105 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 85469b756..a0371b789 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -32,10 +32,10 @@ assert len(bytes("abcdé", "utf8")) == 6 # assert a == b"abcd" -# assert a > b -# assert a >= b -# assert b < a -# assert b <= a +assert a > b +assert a >= b +assert b < a +assert b <= a assert b'foobar'.__eq__(2) == NotImplemented assert b'foobar'.__ne__(2) == NotImplemented diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 8d32aeabf..81c70d440 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -119,6 +119,38 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } + + pub fn ge(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements >= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn le(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements <= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn gt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements > other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn lt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements < other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } } // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 07e71e23c..54427fc57 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,30 +1,11 @@ -use crate::obj::objint::PyInt; -use crate::obj::objlist; -use crate::obj::objlist::PyList; -use crate::obj::objstr::PyString; -use crate::obj::objtuple::PyTuple; -use crate::obj::objtype; -use std::cell::Cell; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use crate::vm::VirtualMachine; use std::ops::Deref; -use num_traits::ToPrimitive; - use crate::function::OptionalArg; -use crate::pyobject::{ - IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, -}; -use crate::vm::VirtualMachine; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::PyByteInner; -use super::objint; -use super::objiter; -use super::objstr; use super::objtype::PyClassRef; -use std::clone::Clone; - /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -35,7 +16,7 @@ use std::clone::Clone; /// - any object implementing the buffer API.\n \ /// - an integer"; #[pyclass(name = "bytes", __inside_vm)] -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct PyBytes { inner: PyByteInner, } @@ -113,102 +94,88 @@ impl PyBytesRef { } #[pymethod(name = "__repr__")] - fn repr(self, _vm: &VirtualMachine) -> String { - // TODO: don't just unwrap - let data = self.inner.elements.clone(); - format!("b'{:?}'", data) + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("b'{}'", self.inner.repr()?))) } #[pymethod(name = "__len__")] fn len(self, _vm: &VirtualMachine) -> usize { - self.inner.elements.len() - } -} -/* - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value == other.value) - } else { - vm.ctx.not_implemented() - } + self.inner.len() } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value >= other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.eq(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value > other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.ge(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value <= other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.le(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if let Ok(other) = other.downcast::() { - vm.ctx.new_bool(self.value < other.value) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.gt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } - - - fn hash(self, _vm: &VirtualMachine) -> u64 { - let mut hasher = DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() - } - - - fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { - PyBytesIterator { - position: Cell::new(0), - bytes: self, - } + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) } } +// fn hash(self, _vm: &VirtualMachine) -> u64 { +// let mut hasher = DefaultHasher::new(); +// self.value.hash(&mut hasher); +// hasher.finish() +// } +// fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { +// PyBytesIterator { +// position: Cell::new(0), +// bytes: self, +// } +// } +// } -#[derive(Debug)] -pub struct PyBytesIterator { - position: Cell, - bytes: PyBytesRef, -} +// #[derive(Debug)] +// pub struct PyBytesIterator { +// position: Cell, +// bytes: PyBytesRef, +// } -impl PyValue for PyBytesIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.bytesiterator_type() - } -} +// impl PyValue for PyBytesIterator { +// fn class(vm: &VirtualMachine) -> PyClassRef { +// vm.ctx.bytesiterator_type() +// } +// } -type PyBytesIteratorRef = PyRef; +// type PyBytesIteratorRef = PyRef; -impl PyBytesIteratorRef { - fn next(self, vm: &VirtualMachine) -> PyResult { - if self.position.get() < self.bytes.value.len() { - let ret = self.bytes[self.position.get()]; - self.position.set(self.position.get() + 1); - Ok(ret) - } else { - Err(objiter::new_stop_iteration(vm)) - } - } +// impl PyBytesIteratorRef { +// fn next(self, vm: &VirtualMachine) -> PyResult { +// if self.position.get() < self.bytes.value.len() { +// let ret = self.bytes[self.position.get()]; +// self.position.set(self.position.get() + 1); +// Ok(ret) +// } else { +// Err(objiter::new_stop_iteration(vm)) +// } +// } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -}*/ +// fn iter(self, _vm: &VirtualMachine) -> Self { +// self +// } +// } From abc72e999217e0de9318a04904562182a79ff62b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 7 Apr 2019 23:54:42 +0300 Subject: [PATCH 122/223] contains return bool --- vm/src/obj/objset.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 4b60b5b5c..5ec4fca46 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -88,14 +88,14 @@ impl PySetInner { } } - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { for element in self.elements.iter() { let value = vm._eq(needle.clone(), element.1.clone())?; if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); + return Ok(true); } } - Ok(vm.new_bool(false)) + Ok(false) } fn _compare_inner( @@ -124,8 +124,7 @@ impl PySetInner { return Ok(vm.new_bool(false)); } for element in get_other(swap).elements.iter() { - let value = get_zelf(swap).contains(element.1.clone(), vm)?; - if !objbool::get_value(&value) { + if !get_zelf(swap).contains(element.1.clone(), vm)? { return Ok(vm.new_bool(false)); } } @@ -190,7 +189,7 @@ impl PySetInner { let mut elements = HashMap::new(); for item in other.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { insert_into_set(vm, &mut elements, &obj)?; } } @@ -201,7 +200,7 @@ impl PySetInner { let mut elements = self.elements.clone(); for item in other.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { remove_from_set(vm, &mut elements, &obj)?; } } @@ -213,7 +212,7 @@ impl PySetInner { for item in other.iter(vm)? { let obj = item?; - if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + if !self.contains(obj.clone(), vm)? { new_inner.add(&obj, vm)?; } else { new_inner.remove(&obj, vm)?; @@ -287,7 +286,7 @@ impl PySetInner { self.clear(); for item in iterable.iter(vm)? { let obj = item?; - if objbool::get_value(&temp_inner.contains(obj.clone(), vm)?) { + if temp_inner.contains(obj.clone(), vm)? { self.add(&obj, vm)?; } } @@ -297,7 +296,7 @@ impl PySetInner { fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { for item in iterable.iter(vm)? { let obj = item?; - if objbool::get_value(&self.contains(obj.clone(), vm)?) { + if self.contains(obj.clone(), vm)? { self.remove(&obj, vm)?; } } @@ -311,7 +310,7 @@ impl PySetInner { ) -> PyResult { for item in iterable.iter(vm)? { let obj = item?; - if !objbool::get_value(&self.contains(obj.clone(), vm)?) { + if !self.contains(obj.clone(), vm)? { self.add(&obj, vm)?; } else { self.remove(&obj, vm)?; @@ -343,7 +342,7 @@ impl PySetRef { } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.borrow().contains(needle, vm) } @@ -550,7 +549,7 @@ impl PyFrozenSetRef { } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner.contains(needle, vm) } From 3c736c1f940f65e95275512e87d3ea28f46238e7 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 00:11:20 +0200 Subject: [PATCH 123/223] iter --- tests/snippets/bytes.py | 8 ++- vm/src/obj/objbyteinner.rs | 9 +++ vm/src/obj/objbytes.rs | 114 +++++++++++++++---------------------- 3 files changed, 63 insertions(+), 68 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index a0371b789..bb72847b6 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -30,7 +30,7 @@ assert repr(b"abcd") == "b'abcd'" #len assert len(bytes("abcdé", "utf8")) == 6 -# +#comp assert a == b"abcd" assert a > b assert a >= b @@ -43,3 +43,9 @@ assert b'foobar'.__gt__(2) == NotImplemented assert b'foobar'.__ge__(2) == NotImplemented assert b'foobar'.__lt__(2) == NotImplemented assert b'foobar'.__le__(2) == NotImplemented + +#hash +hash(a) == hash(b"abcd") + +#iter +[i for i in b"abcd"] == ["a", "b", "c", "d"] diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 81c70d440..93293e627 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -7,6 +7,8 @@ use crate::vm::VirtualMachine; use crate::pyobject::{PyResult, TypeProtocol}; use crate::obj::objstr::PyString; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use super::objint; use crate::obj::objint::PyInt; @@ -151,6 +153,13 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } + + pub fn hash(&self) -> usize { + let mut hasher = DefaultHasher::new(); + self.elements.hash(&mut hasher); + hasher.finish() as usize + } } + // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 54427fc57..8c64944ca 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,11 +1,14 @@ use crate::vm::VirtualMachine; +use core::cell::Cell; use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use super::objbyteinner::PyByteInner; +use super::objiter; use super::objtype::PyClassRef; + /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ /// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ @@ -44,39 +47,18 @@ impl PyValue for PyBytes { } } -// Binary data support - -// Fill bytes class methods: - pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { &obj.payload::().unwrap().inner.elements } -// pub fn init(context: &PyContext) { -// let bytes_doc = -pub fn init(ctx: &PyContext) { - PyBytesRef::extend_class(ctx, &ctx.bytes_type); +pub fn init(context: &PyContext) { + PyBytesRef::extend_class(context, &context.bytes_type); + let bytesiterator_type = &context.bytesiterator_type; + extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } -//extend_class!(context, &context.bytes_type, { -//"__new__" => context.new_rustfunc(bytes_new), -/* "__eq__" => context.new_rustfunc(PyBytesRef::eq), -"__lt__" => context.new_rustfunc(PyBytesRef::lt), -"__le__" => context.new_rustfunc(PyBytesRef::le), -"__gt__" => context.new_rustfunc(PyBytesRef::gt), -"__ge__" => context.new_rustfunc(PyBytesRef::ge), -"__hash__" => context.new_rustfunc(PyBytesRef::hash), -"__repr__" => context.new_rustfunc(PyBytesRef::repr), -"__len__" => context.new_rustfunc(PyBytesRef::len), -"__iter__" => context.new_rustfunc(PyBytesRef::iter), -"__doc__" => context.new_str(bytes_doc.to_string())*/ -// }); - -/* let bytesiterator_type = &context.bytesiterator_type; -extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), -});*/ -//} #[pyimpl(__inside_vm)] impl PyBytesRef { @@ -134,48 +116,46 @@ impl PyBytesRef { bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), _ => Ok(vm.ctx.not_implemented())) } + #[pymethod(name = "__hash__")] + fn hash(self, _vm: &VirtualMachine) -> usize { + self.inner.hash() + } + + #[pymethod(name = "__iter__")] + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { + position: Cell::new(0), + bytes: self, + } + } } -// fn hash(self, _vm: &VirtualMachine) -> u64 { -// let mut hasher = DefaultHasher::new(); -// self.value.hash(&mut hasher); -// hasher.finish() -// } +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, +} -// fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { -// PyBytesIterator { -// position: Cell::new(0), -// bytes: self, -// } -// } -// } +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } +} -// #[derive(Debug)] -// pub struct PyBytesIterator { -// position: Cell, -// bytes: PyBytesRef, -// } +type PyBytesIteratorRef = PyRef; -// impl PyValue for PyBytesIterator { -// fn class(vm: &VirtualMachine) -> PyClassRef { -// vm.ctx.bytesiterator_type() -// } -// } +impl PyBytesIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.inner.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } -// type PyBytesIteratorRef = PyRef; - -// impl PyBytesIteratorRef { -// fn next(self, vm: &VirtualMachine) -> PyResult { -// if self.position.get() < self.bytes.value.len() { -// let ret = self.bytes[self.position.get()]; -// self.position.set(self.position.get() + 1); -// Ok(ret) -// } else { -// Err(objiter::new_stop_iteration(vm)) -// } -// } - -// fn iter(self, _vm: &VirtualMachine) -> Self { -// self -// } -// } + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} From ec65b8480fe35ece7182544eaf75d0582a257266 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 00:45:53 +0200 Subject: [PATCH 124/223] add contains --- tests/snippets/bytes.py | 14 ++++++++++++++ vm/src/obj/objbyteinner.rs | 26 ++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index bb72847b6..2fd32153d 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -49,3 +49,17 @@ hash(a) == hash(b"abcd") #iter [i for i in b"abcd"] == ["a", "b", "c", "d"] + +#add +assert a + b == b"abcdab" + +#contains +# contains +assert b"ab" in b"abcd" +assert b"cd" in b"abcd" +assert b"abcd" in b"abcd" +assert b"a" in b"abcd" +assert b"d" in b"abcd" +assert b"dc" not in b"abcd" +# assert 97 in b"abcd" +# assert 150 not in b"abcd" \ No newline at end of file diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 93293e627..03876095c 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -159,6 +159,32 @@ impl PyByteInner { self.elements.hash(&mut hasher); hasher.finish() as usize } + + pub fn add(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + let elements: Vec = self + .elements + .iter() + .chain(other.elements.iter()) + .cloned() + .collect(); + Ok(vm.ctx.new_bytes(elements)) + } + + pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + for (n, i) in self.elements.iter().enumerate() { + if n + other.len() <= self.len() && *i == other.elements[0] { + if &self.elements[n..n + other.len()] == other.elements.as_slice() { + return Ok(vm.new_bool(true)); + } + } + } + Ok(vm.new_bool(false)) + } + + pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + self.elements.contains(int); + Ok(vm.new_bool(false)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8c64944ca..e052654bf 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -128,6 +128,33 @@ impl PyBytesRef { bytes: self, } } + + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.add(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + + #[pymethod(name = "__contains__")] + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // no new style since objint is not. + match_class!(needle, + bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), + int @ PyInt => self.inner.contains_int(&int, vm), + _ => Ok(vm.ctx.not_implemented())) + // if objtype::isinstance(&needle, &vm.ctx.bytes_type()) { + // let result = vec_contains(&self.value, &get_value(&needle)); + // vm.ctx.new_bool(result) + // } else if objtype::isinstance(&needle, &vm.ctx.int_type()) { + // let result = self + // .value + // .contains(&objint::get_value(&needle).to_u8().unwrap()); + // vm.ctx.new_bool(result) + // } else { + // vm.new_type_error(format!("Cannot add {:?} and {:?}", self, needle)) + // } + } } #[derive(Debug)] From 4d8a56aea1d86d861c488e1eb4fd8ee7669090b4 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:09:44 -0600 Subject: [PATCH 125/223] `list.__setitem__` with slice and int indexing optimisation is uncertain correctness is uncertain but it does appear to work --- vm/src/obj/objlist.rs | 173 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 159 insertions(+), 14 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 60c14b107..3a4035409 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -11,11 +11,11 @@ use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyVal use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; +//use super::objint; use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, SequenceIndex, + SequenceIndex, }; use super::objslice::PySliceRef; use super::objtype; @@ -181,22 +181,167 @@ impl PyListRef { } } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.borrow_mut(); + fn setitem( + self, + subscript: SequenceIndex, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.setindex(index, value, vm), + SequenceIndex::Slice(slice) => { + // TODO check special case of if a == b, may need copy of b first? + self.setslice(slice, value, vm) + } + } + } - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(pos_index) = elements.get_pos(idx) { - elements[pos_index] = value; - Ok(vm.get_none()) + fn setindex(self, index: i32, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut()[pos_index] = value; + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list assignment index out of range".to_string())) + } + } + + fn setslice(self, slice: PySliceRef, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&slice.start, &slice.stop); + if range.start < range.end { + match step.to_i32() { + Some(1) => self._set_slice(range, sec, vm), + Some(num) => { + // assign to extended slice + self._set_stepped_slice(range, num as usize, sec, vm) + } + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list, no question + #[allow(clippy::range_plus_one)] + self._set_stepped_slice(range.start..(range.start + 1), 1, sec, vm) + } + } } else { - Err(vm.new_index_error("list index out of range".to_string())) + // this functions as an insert of sec before range.start + self._set_slice(range.start..range.start, sec, vm) + // TODO this will take some special processing } } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = &slice.start.as_ref().map(|x| x + 1); + let stop = &slice.stop.as_ref().map(|x| x + 1); + let range = self.get_slice_range(&stop, &start); + match (-step).to_i32() { + Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list no question + self._set_stepped_slice_reverse(range.end - 1..range.end, 1, sec, vm) + } + } + } + } + + fn _set_slice(self, range: Range, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // consume the iter, we don't need it's size + // but if it's going to fail we want that to happen *before* we start modifing + if let Ok(items) = vm.extract_elements(&sec) { + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _set_stepped_slice( + self, + range: Range, + step: usize, + sec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let slicelen = ((range.end - range.start - 1) / step) + 1; + if let Ok(items) = vm.extract_elements(&sec) { + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _set_stepped_slice_reverse( + self, + range: Range, + step: usize, + sec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let slicelen = ((range.end - range.start - 1) / step) + 1; + if let Ok(items) = vm.extract_elements(&sec) { + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } else { + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + + fn _replace_indexes(self, indexes: I, items: &[PyObjectRef]) + where + I: Iterator, + { + let mut elements = self.elements.borrow_mut(); + + for (i, value) in indexes.zip(items) { + // clone for refrence count + elements[i] = value.clone(); } } From 118c98ccc76977945e78495b28124b42290162f6 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:12:03 -0600 Subject: [PATCH 126/223] exhaustive (hopefully) test snippets I attempted to cover every case with the snippets but I easily could of missed some. --- tests/snippets/list.py | 198 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 3d9743c0b..71f7dea06 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -201,3 +201,201 @@ assert_raises(TypeError, bad_del_1) def bad_del_2(): del ['a', 'b'][2] assert_raises(IndexError, bad_del_2) + +# __setitem__ + +# simple index +x = [1, 2, 3, 4, 5] +x[0] = 'a' +assert x == ['a', 2, 3, 4, 5] +x[-1] = 'b' +assert x == ['a', 2, 3, 4, 'b'] +# make sure refrences are assigned correctly +y = [] +x[1] = y +y.append(100) +assert x[1] == y +assert x[1] == [100] + +#index bounds +def set_index_out_of_bounds_high(): + x = [0, 1, 2, 3, 4] + x[5] = 'a' + +def set_index_out_of_bounds_low(): + x = [0, 1, 2, 3, 4] + x[-6] = 'a' + +assert_raises(IndexError, set_index_out_of_bounds_high) +assert_raises(IndexError, set_index_out_of_bounds_low) + +# non stepped slice index +a = list(range(10)) +x = a[:] +y = a[:] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +# replace whole list +x[:] = ['a', 'b', 'c'] +y[::1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c'] +assert x == y +# splice list start +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[:1] = ['a', 'b', 'c'] +y[0:1] = ['a', 'b', 'c'] +z[:1:1] = ['a', 'b', 'c'] +zz[0:1:1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# splice list end +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[5:] = ['a', 'b', 'c'] +y[5::1] = ['a', 'b', 'c'] +z[5:10] = ['a', 'b', 'c'] +zz[5:10:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +assert x == y +assert x == z +assert x == zz +# insert sec +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[1:1] = ['a', 'b', 'c'] +y[1:0] = ['a', 'b', 'c'] +z[1:1:1] = ['a', 'b', 'c'] +zz[1:0:1] = ['a', 'b', 'c'] +assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# same but negative indexes? +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[-1:-1] = ['a', 'b', 'c'] +y[-1:9] = ['a', 'b', 'c'] +z[-1:-1:1] = ['a', 'b', 'c'] +zz[-1:9:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +assert x == y +assert x == z +assert x == zz +# splice mid +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', 'e'] +y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +assert x == y +x = a[:] +x[3:5] = ['a'] +assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +# assign empty to non stepped empty slice does nothing +x = a[:] +y = a[:] +x[5:2] = [] +y[5:2:1] = [] +assert x == a +assert y == a +# assign empty to non stepped slice removes elems +x = a[:] +y = a[:] +x[2:8] = [] +y[2:8:1] = [] +assert x == [0, 1, 8, 9] +assert x == y +# make sure refrences are assigned correctly +yy = [] +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', yy] +y[3:5:1] = ['a', 'b', 'c', 'd', yy] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +assert x == y +yy.append(100) +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == y +assert x[7] == yy +assert x[7] == [100] +assert y[7] == yy +assert y[7] == [100] + +# no zero step +def no_zero_step_set(): + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] +assert_raises(ValueError, no_zero_step_set) + +# stepped slice index +# forward slice +x = a[:] +x[2:8:2] = ['a', 'b', 'c'] +assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e'] +x[::2] = c +y[-10::2] = c +z[0:10:2] = c +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +assert x == y +assert x == z +assert x == zz +# backward slice +x = a[:] +x[8:2:-2] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +x[::-1] = c +y[9:-11:-1] = c +z[9::-1] = c +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +assert x == y +assert x == z +assert x == zz +# step size bigger than len +x = a[:] +x[::200] = ['a'] +assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x = a[:] +x[5::200] = ['a'] +assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] + +# bad stepped slices +def stepped_slice_assign_too_big(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b', 'c', 'd'] + +assert_raises(ValueError, stepped_slice_assign_too_big) + +def stepped_slice_assign_too_small(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b'] + +assert_raises(ValueError, stepped_slice_assign_too_small) + +# must assign iter t0 slice +def must_assign_iter_to_slice(): + x = [0, 1, 2, 3, 4] + x[::2] = 42 + +assert_raises(TypeError, must_assign_iter_to_slice) From a23b5bc470f0196079529b11351169cc543a362f Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Sun, 7 Apr 2019 19:32:11 -0600 Subject: [PATCH 127/223] cleanup and cargo fmt --- vm/src/obj/objlist.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 3a4035409..1ef01ca3e 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -189,10 +189,7 @@ impl PyListRef { ) -> PyResult { match subscript { SequenceIndex::Int(index) => self.setindex(index, value, vm), - SequenceIndex::Slice(slice) => { - // TODO check special case of if a == b, may need copy of b first? - self.setslice(slice, value, vm) - } + SequenceIndex::Slice(slice) => self.setslice(slice, value, vm), } } @@ -229,7 +226,6 @@ impl PyListRef { } else { // this functions as an insert of sec before range.start self._set_slice(range.start..range.start, sec, vm) - // TODO this will take some special processing } } else { // calculate the range for the reverse slice, first the bounds needs to be made From 0e264426c9b876c4706042eff8cfd50415d81849 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Mon, 8 Apr 2019 09:12:22 +0200 Subject: [PATCH 128/223] fix import --- tests/snippets/builtin_slice.py | 16 ++++++++-------- tests/test_snippets.py | 22 +++++----------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index d61698034..b12ad8021 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -79,13 +79,13 @@ def test_all_slices(): test all possible slices except big number """ - import resources - from resources.cpython_generated_slices import SLICES_RES, START, END, STEP, LL + mod = __import__('cpython_generated_slices') - ll = LL - start = START - end = END - step = STEP + ll = mod.LL + start = mod.START + end = mod.END + step = mod.STEP + slices_res = mod.SLICES_RES count = 0 failures = [] @@ -94,11 +94,11 @@ def test_all_slices(): for t in step: lhs = ll[s:e:t] try: - assert lhs == SLICES_RES[count] + assert lhs == slices_res[count] except AssertionError: failures.append( "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( - s, e, t, lhs, SLICES_RES[count] + s, e, t, lhs, slices_res[count] ) ) count += 1 diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7a888f093..9322485ce 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -26,10 +26,6 @@ TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) -RESOURCES_DIR = os.path.abspath( - os.path.join(TEST_DIRS[_TestType.functional], "resources") -) - @contextlib.contextmanager def pushd(path): @@ -149,17 +145,11 @@ def generate_slices(path): class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - # setup resource dir - cls.resources = Path(RESOURCES_DIR) - if cls.resources.exists(): - shutil.rmtree( - RESOURCES_DIR, ignore_errors=True - ) # we don't care if dir doesnt exist - cls.resources.mkdir() - (cls.resources / "__init__.py").touch() - # Here add resource files - cls.slices_resource_path = Path(RESOURCES_DIR) / "cpython_generated_slices.py" + cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + if cls.slices_resource_path.exists(): + cls.slices_resource_path.unlink() + generate_slices(cls.slices_resource_path) # cargo stuff @@ -168,6 +158,4 @@ class SampleTestCase(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree( - RESOURCES_DIR, ignore_errors=True - ) # we don't care if dir doesnt exist + cls.slices_resource_path.unlink() From 86928114fa1505ea6dadfc39cb096e73249ac0ff Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Mon, 8 Apr 2019 10:33:56 +0200 Subject: [PATCH 129/223] resolve conflict --- vm/src/obj/objsequence.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index 1e389bafd..32c723a26 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -2,9 +2,6 @@ use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; - use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; use crate::vm::VirtualMachine; From 9c32de954bf586cd2425bc27014ca6f85e6bab6b Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Sat, 6 Apr 2019 17:09:41 +0100 Subject: [PATCH 130/223] Add iterator for dictionary keys. --- vm/src/dictdatatype.rs | 12 +++++++++++ vm/src/obj/objdict.rs | 49 +++++++++++++++++++++++++++++++----------- vm/src/pyobject.rs | 3 +++ 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 8a6a8df1b..dc25e6eeb 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -120,6 +120,18 @@ impl Dict { self.len() == 0 } + pub fn next_entry(&self, position: usize) -> Option<(usize, &PyObjectRef, &T)> { + let mut new_position = position; + while position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[position] { + return Some((new_position + 1, key, value)); + } else { + new_position += 1; + } + } + None + } + pub fn iter_items(&self) -> impl Iterator + '_ { self.entries .iter() diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e84f7b178..4a1b6bbb1 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -132,20 +132,11 @@ impl PyDictRef { } /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + fn iter(self, _vm: &VirtualMachine) -> PyDictIterator { // TODO: separate type, not a list iterator - let keys = self - .entries - .borrow() - .get_items() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); - let key_list = vm.ctx.new_list(keys); - - PyListIterator { + PyDictIterator { position: Cell::new(0), - list: key_list.downcast().unwrap(), + dict: self, } } @@ -284,6 +275,35 @@ impl ItemProtocol for PyDictRef { } } +#[derive(Debug)] +struct PyDictIterator { + pub dict: PyDictRef, + pub position: Cell, +} +type PyDictIteratorRef = PyRef; + +impl PyDictIteratorRef { + fn next(self: PyDictIteratorRef, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, _value)) => { + self.position.set(new_position); + Ok(key.clone()) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictiterator_type.clone() + } +} + pub fn init(context: &PyContext) { extend_class!(context, &context.dict_type, { "__bool__" => context.new_rustfunc(PyDictRef::bool), @@ -305,4 +325,9 @@ pub fn init(context: &PyContext) { "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), }); + + extend_class!(context, &context.dictiterator_type, { + "__next__" => context.new_rustfunc(PyDictIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictIteratorRef::iter), + }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 4d2c8353b..2c2371f21 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,6 +129,7 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, + pub dictiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -254,6 +255,7 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); + let dictiterator_type = create_type("dict_iterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -313,6 +315,7 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, + dictiterator_type, set_type, frozenset_type, true_value, From 5c4755ffd77f66371e102efee16528086da8c883 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 11:08:45 +0100 Subject: [PATCH 131/223] Separate iterators for items/keys/values. --- tests/snippets/dict.py | 9 ++- vm/src/dictdatatype.rs | 7 +- vm/src/obj/objdict.rs | 155 ++++++++++++++++++++++++++++------------- vm/src/pyobject.rs | 12 +++- 4 files changed, 128 insertions(+), 55 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f924bb0e7..f57fc37b6 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -37,11 +37,18 @@ for key in a.keys(): res.add(key) assert res == set(['a','b']) +x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} +del x['c'] +it = iter(x.items()) +assert ('a', 1) == next(it) +assert ('b', 2) == next(it) +assert ('d', 3) == next(it) + + x = {} x[1] = 1 assert x[1] == 1 - x[7] = 7 x[2] = 2 x[(5, 6)] = 5 diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index dc25e6eeb..0612bc8a3 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -120,13 +120,12 @@ impl Dict { self.len() == 0 } - pub fn next_entry(&self, position: usize) -> Option<(usize, &PyObjectRef, &T)> { - let mut new_position = position; + pub fn next_entry(&self, mut position: usize) -> Option<(usize, &PyObjectRef, &T)> { while position < self.entries.len() { if let Some(DictEntry { key, value, .. }) = &self.entries[position] { - return Some((new_position + 1, key, value)); + return Some((position + 1, key, value)); } else { - new_position += 1; + position += 1; } } None diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4a1b6bbb1..a0d9210b5 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -11,7 +11,6 @@ use crate::vm::{ReprGuard, VirtualMachine}; use crate::dictdatatype; use super::objiter; -use super::objlist::PyListIterator; use super::objstr; use crate::obj::objtype::PyClassRef; @@ -131,47 +130,16 @@ impl PyDictRef { self.entries.borrow_mut().clear() } - /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, _vm: &VirtualMachine) -> PyDictIterator { - // TODO: separate type, not a list iterator - PyDictIterator { - position: Cell::new(0), - dict: self, - } + fn iter(self, vm: &VirtualMachine) -> PyDictKeysIteratorRef { + PyDictKeysIteratorRef::new(self, vm) } - fn values(self, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type. `values` should be a live view over the collection, not an iterator. - let values = self - .entries - .borrow() - .get_items() - .iter() - .map(|(_k, v)| v.clone()) - .collect(); - let values_list = vm.ctx.new_list(values); - - PyListIterator { - position: Cell::new(0), - list: values_list.downcast().unwrap(), - } + fn values(self, vm: &VirtualMachine) -> PyDictValuesIteratorRef { + PyDictValuesIteratorRef::new(self, vm) } - fn items(self, vm: &VirtualMachine) -> PyListIterator { - // TODO: separate type. `items` should be a live view over the collection, not an iterator. - let items = self - .entries - .borrow() - .get_items() - .iter() - .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) - .collect(); - let items_list = vm.ctx.new_list(items); - - PyListIterator { - position: Cell::new(0), - list: items_list.downcast().unwrap(), - } + fn items(self, vm: &VirtualMachine) -> PyDictItemsIteratorRef { + PyDictItemsIteratorRef::new(self, vm) } pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -276,14 +244,22 @@ impl ItemProtocol for PyDictRef { } #[derive(Debug)] -struct PyDictIterator { +struct PyDictKeysIterator { pub dict: PyDictRef, pub position: Cell, } -type PyDictIteratorRef = PyRef; +type PyDictKeysIteratorRef = PyRef; -impl PyDictIteratorRef { - fn next(self: PyDictIteratorRef, vm: &VirtualMachine) -> PyResult { +impl PyDictKeysIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictKeysIteratorRef { + PyDictKeysIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self, vm: &VirtualMachine) -> PyResult { match self.dict.entries.borrow().next_entry(self.position.get()) { Some((new_position, key, _value)) => { self.position.set(new_position); @@ -298,9 +274,84 @@ impl PyDictIteratorRef { } } -impl PyValue for PyDictIterator { +impl PyValue for PyDictKeysIterator { fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictiterator_type.clone() + vm.ctx.dictkeysiterator_type.clone() + } +} + +#[derive(Debug)] +struct PyDictValuesIterator { + pub dict: PyDictRef, + pub position: Cell, +} +type PyDictValuesIteratorRef = PyRef; + +impl PyDictValuesIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictValuesIteratorRef { + PyDictValuesIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, _key, value)) => { + self.position.set(new_position); + Ok(value.clone()) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictValuesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictvaluesiterator_type.clone() + } +} + +#[derive(Debug)] +struct PyDictItemsIterator { + pub dict: PyDictRef, + pub position: Cell, +} + +type PyDictItemsIteratorRef = PyRef; + +impl PyDictItemsIteratorRef { + fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictItemsIteratorRef { + PyDictItemsIterator { + position: Cell::new(0), + dict, + } + .into_ref(vm) + } + + fn next(self: PyDictItemsIteratorRef, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, value)) => { + self.position.set(new_position); + Ok(vm.ctx.new_tuple(vec![key.clone(), value.clone()])) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +impl PyValue for PyDictItemsIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.dictitemsiterator_type.clone() } } @@ -326,8 +377,18 @@ pub fn init(context: &PyContext) { "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictiterator_type, { - "__next__" => context.new_rustfunc(PyDictIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictIteratorRef::iter), + extend_class!(context, &context.dictkeysiterator_type, { + "__next__" => context.new_rustfunc(PyDictKeysIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictKeysIteratorRef::iter), + }); + + extend_class!(context, &context.dictvaluesiterator_type, { + "__next__" => context.new_rustfunc(PyDictValuesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictValuesIteratorRef::iter), + }); + + extend_class!(context, &context.dictitemsiterator_type, { + "__next__" => context.new_rustfunc(PyDictItemsIteratorRef::next), + "__iter__" => context.new_rustfunc(PyDictItemsIteratorRef::iter), }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 2c2371f21..8c405c5b1 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,7 +129,9 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, - pub dictiterator_type: PyClassRef, + pub dictkeysiterator_type: PyClassRef, + pub dictvaluesiterator_type: PyClassRef, + pub dictitemsiterator_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -255,7 +257,9 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); - let dictiterator_type = create_type("dict_iterator", &type_type, &object_type); + let dictkeysiterator_type = create_type("dict_keys", &type_type, &object_type); + let dictvaluesiterator_type = create_type("dict_values", &type_type, &object_type); + let dictitemsiterator_type = create_type("dict_items", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -315,7 +319,9 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, - dictiterator_type, + dictkeysiterator_type, + dictvaluesiterator_type, + dictitemsiterator_type, set_type, frozenset_type, true_value, From 2d127b983422e68a9d9fe26b509107fbaaf2bfce Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 11:58:11 +0100 Subject: [PATCH 132/223] Use macro to generate full set of views and iterators. --- vm/src/obj/objdict.rs | 217 ++++++++++++++++++++---------------------- vm/src/pyobject.rs | 27 ++++-- 2 files changed, 121 insertions(+), 123 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index a0d9210b5..e80fa7d8a 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -130,16 +130,20 @@ impl PyDictRef { self.entries.borrow_mut().clear() } - fn iter(self, vm: &VirtualMachine) -> PyDictKeysIteratorRef { - PyDictKeysIteratorRef::new(self, vm) + fn iter(self, _vm: &VirtualMachine) -> PyDictKeyIterator { + PyDictKeyIterator::new(self) } - fn values(self, vm: &VirtualMachine) -> PyDictValuesIteratorRef { - PyDictValuesIteratorRef::new(self, vm) + fn keys(self, _vm: &VirtualMachine) -> PyDictKeys { + PyDictKeys::new(self) } - fn items(self, vm: &VirtualMachine) -> PyDictItemsIteratorRef { - PyDictItemsIteratorRef::new(self, vm) + fn values(self, _vm: &VirtualMachine) -> PyDictValues { + PyDictValues::new(self) + } + + fn items(self, _vm: &VirtualMachine) -> PyDictItems { + PyDictItems::new(self) } pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { @@ -243,116 +247,89 @@ impl ItemProtocol for PyDictRef { } } -#[derive(Debug)] -struct PyDictKeysIterator { - pub dict: PyDictRef, - pub position: Cell, -} -type PyDictKeysIteratorRef = PyRef; - -impl PyDictKeysIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictKeysIteratorRef { - PyDictKeysIterator { - position: Cell::new(0), - dict, +macro_rules! dict_iterator { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $result_fn: expr) => { + #[derive(Debug)] + struct $name { + pub dict: PyDictRef, } - .into_ref(vm) - } - fn next(self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, _value)) => { - self.position.set(new_position); - Ok(key.clone()) + impl $name { + fn new(dict: PyDictRef) -> Self { + $name { dict: dict } } - None => Err(objiter::new_stop_iteration(vm)), - } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -} - -impl PyValue for PyDictKeysIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictkeysiterator_type.clone() - } -} - -#[derive(Debug)] -struct PyDictValuesIterator { - pub dict: PyDictRef, - pub position: Cell, -} -type PyDictValuesIteratorRef = PyRef; - -impl PyDictValuesIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictValuesIteratorRef { - PyDictValuesIterator { - position: Cell::new(0), - dict, - } - .into_ref(vm) - } - - fn next(self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, _key, value)) => { - self.position.set(new_position); - Ok(value.clone()) + fn iter(&self, _vm: &VirtualMachine) -> $iter_name { + $iter_name::new(self.dict.clone()) } - None => Err(objiter::new_stop_iteration(vm)), } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } -} - -impl PyValue for PyDictValuesIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictvaluesiterator_type.clone() - } -} - -#[derive(Debug)] -struct PyDictItemsIterator { - pub dict: PyDictRef, - pub position: Cell, -} - -type PyDictItemsIteratorRef = PyRef; - -impl PyDictItemsIteratorRef { - fn new(dict: PyDictRef, vm: &VirtualMachine) -> PyDictItemsIteratorRef { - PyDictItemsIterator { - position: Cell::new(0), - dict, - } - .into_ref(vm) - } - - fn next(self: PyDictItemsIteratorRef, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, value)) => { - self.position.set(new_position); - Ok(vm.ctx.new_tuple(vec![key.clone(), value.clone()])) + impl PyValue for $name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$class.clone() } - None => Err(objiter::new_stop_iteration(vm)), } - } - fn iter(self, _vm: &VirtualMachine) -> Self { - self - } + #[derive(Debug)] + struct $iter_name { + pub dict: PyDictRef, + pub position: Cell, + } + + impl $iter_name { + fn new(dict: PyDictRef) -> Self { + $iter_name { + position: Cell::new(0), + dict, + } + } + + fn next(&self, vm: &VirtualMachine) -> PyResult { + match self.dict.entries.borrow().next_entry(self.position.get()) { + Some((new_position, key, value)) => { + self.position.set(new_position); + Ok($result_fn(vm, key, value)) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + } + + impl PyValue for $iter_name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$iter_class.clone() + } + } + }; } -impl PyValue for PyDictItemsIterator { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.dictitemsiterator_type.clone() - } +dict_iterator! { + PyDictKeys, + PyDictKeyIterator, + dictkeys_type, + dictkeyiterator_type, + |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() +} + +dict_iterator! { + PyDictValues, + PyDictValueIterator, + dictvalues_type, + dictvalueiterator_type, + |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() +} + +dict_iterator! { + PyDictItems, + PyDictItemIterator, + dictitems_type, + dictitemiterator_type, + |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| + vm.ctx.new_tuple(vec![key.clone(), value.clone()]) } pub fn init(context: &PyContext) { @@ -371,24 +348,36 @@ pub fn init(context: &PyContext) { "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), // TODO: separate type. `keys` should be a live view over the collection, not an iterator. - "keys" => context.new_rustfunc(PyDictRef::iter), + "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), "copy" => context.new_rustfunc(PyDictRef::copy), "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictkeysiterator_type, { - "__next__" => context.new_rustfunc(PyDictKeysIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictKeysIteratorRef::iter), + extend_class!(context, &context.dictkeys_type, { + "__iter__" => context.new_rustfunc(PyDictKeys::iter), }); - extend_class!(context, &context.dictvaluesiterator_type, { - "__next__" => context.new_rustfunc(PyDictValuesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictValuesIteratorRef::iter), + extend_class!(context, &context.dictkeyiterator_type, { + "__next__" => context.new_rustfunc(PyDictKeyIterator::next), + "__iter__" => context.new_rustfunc(PyDictKeyIterator::iter), }); - extend_class!(context, &context.dictitemsiterator_type, { - "__next__" => context.new_rustfunc(PyDictItemsIteratorRef::next), - "__iter__" => context.new_rustfunc(PyDictItemsIteratorRef::iter), + extend_class!(context, &context.dictvalues_type, { + "__iter__" => context.new_rustfunc(PyDictValues::iter), + }); + + extend_class!(context, &context.dictvalueiterator_type, { + "__next__" => context.new_rustfunc(PyDictValueIterator::next), + "__iter__" => context.new_rustfunc(PyDictValueIterator::iter), + }); + + extend_class!(context, &context.dictitems_type, { + "__iter__" => context.new_rustfunc(PyDictItems::iter), + }); + + extend_class!(context, &context.dictitemiterator_type, { + "__next__" => context.new_rustfunc(PyDictItemIterator::next), + "__iter__" => context.new_rustfunc(PyDictItemIterator::iter), }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 8c405c5b1..434693ea6 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -129,9 +129,12 @@ pub struct PyContext { pub false_value: PyIntRef, pub list_type: PyClassRef, pub listiterator_type: PyClassRef, - pub dictkeysiterator_type: PyClassRef, - pub dictvaluesiterator_type: PyClassRef, - pub dictitemsiterator_type: PyClassRef, + pub dictkeyiterator_type: PyClassRef, + pub dictvalueiterator_type: PyClassRef, + pub dictitemiterator_type: PyClassRef, + pub dictkeys_type: PyClassRef, + pub dictvalues_type: PyClassRef, + pub dictitems_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, @@ -257,9 +260,12 @@ impl PyContext { let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); let listiterator_type = create_type("list_iterator", &type_type, &object_type); - let dictkeysiterator_type = create_type("dict_keys", &type_type, &object_type); - let dictvaluesiterator_type = create_type("dict_values", &type_type, &object_type); - let dictitemsiterator_type = create_type("dict_items", &type_type, &object_type); + let dictkeys_type = create_type("dict_keys", &type_type, &object_type); + let dictvalues_type = create_type("dict_values", &type_type, &object_type); + let dictitems_type = create_type("dict_items", &type_type, &object_type); + let dictkeyiterator_type = create_type("dict_keyiterator", &type_type, &object_type); + let dictvalueiterator_type = create_type("dict_valueiterator", &type_type, &object_type); + let dictitemiterator_type = create_type("dict_itemiterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -319,9 +325,12 @@ impl PyContext { staticmethod_type, list_type, listiterator_type, - dictkeysiterator_type, - dictvaluesiterator_type, - dictitemsiterator_type, + dictkeys_type, + dictvalues_type, + dictitems_type, + dictkeyiterator_type, + dictvalueiterator_type, + dictitemiterator_type, set_type, frozenset_type, true_value, From 29e9753e51fa127aec3efea7a68503cf02145b7b Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 8 Apr 2019 15:02:04 +0100 Subject: [PATCH 133/223] Even newer style class definitions. --- vm/src/obj/objdict.rs | 51 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index e80fa7d8a..612fa2cb7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -8,11 +8,11 @@ use crate::pyobject::{ }; use crate::vm::{ReprGuard, VirtualMachine}; -use crate::dictdatatype; - use super::objiter; use super::objstr; +use crate::dictdatatype; use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyClassImpl; pub type DictContentType = dictdatatype::Dict; @@ -248,17 +248,20 @@ impl ItemProtocol for PyDictRef { } macro_rules! dict_iterator { - ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $result_fn: expr) => { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { + #[pyclass(name = $class_name, __inside_vm)] #[derive(Debug)] struct $name { pub dict: PyDictRef, } + #[pyimpl(__inside_vm)] impl $name { fn new(dict: PyDictRef) -> Self { $name { dict: dict } } + #[pymethod(name = "__iter__")] fn iter(&self, _vm: &VirtualMachine) -> $iter_name { $iter_name::new(self.dict.clone()) } @@ -270,12 +273,14 @@ macro_rules! dict_iterator { } } + #[pyclass(name = $iter_class_name, __inside_vm)] #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, pub position: Cell, } + #[pyimpl(__inside_vm)] impl $iter_name { fn new(dict: PyDictRef) -> Self { $iter_name { @@ -284,6 +289,7 @@ macro_rules! dict_iterator { } } + #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { match self.dict.entries.borrow().next_entry(self.position.get()) { Some((new_position, key, value)) => { @@ -294,6 +300,7 @@ macro_rules! dict_iterator { } } + #[pymethod(name = "__iter__")] fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { zelf } @@ -312,6 +319,8 @@ dict_iterator! { PyDictKeyIterator, dictkeys_type, dictkeyiterator_type, + "dictkeys", + "dictkeyiterator", |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() } @@ -320,6 +329,8 @@ dict_iterator! { PyDictValueIterator, dictvalues_type, dictvalueiterator_type, + "dictvalues", + "dictvalueiterator", |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() } @@ -328,6 +339,8 @@ dict_iterator! { PyDictItemIterator, dictitems_type, dictitemiterator_type, + "dictitems", + "dictitemiterator", |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| vm.ctx.new_tuple(vec![key.clone(), value.clone()]) } @@ -354,30 +367,10 @@ pub fn init(context: &PyContext) { "update" => context.new_rustfunc(PyDictRef::update), }); - extend_class!(context, &context.dictkeys_type, { - "__iter__" => context.new_rustfunc(PyDictKeys::iter), - }); - - extend_class!(context, &context.dictkeyiterator_type, { - "__next__" => context.new_rustfunc(PyDictKeyIterator::next), - "__iter__" => context.new_rustfunc(PyDictKeyIterator::iter), - }); - - extend_class!(context, &context.dictvalues_type, { - "__iter__" => context.new_rustfunc(PyDictValues::iter), - }); - - extend_class!(context, &context.dictvalueiterator_type, { - "__next__" => context.new_rustfunc(PyDictValueIterator::next), - "__iter__" => context.new_rustfunc(PyDictValueIterator::iter), - }); - - extend_class!(context, &context.dictitems_type, { - "__iter__" => context.new_rustfunc(PyDictItems::iter), - }); - - extend_class!(context, &context.dictitemiterator_type, { - "__next__" => context.new_rustfunc(PyDictItemIterator::next), - "__iter__" => context.new_rustfunc(PyDictItemIterator::iter), - }); + PyDictKeys::extend_class(context, &context.dictkeys_type); + PyDictKeyIterator::extend_class(context, &context.dictkeyiterator_type); + PyDictValues::extend_class(context, &context.dictvalues_type); + PyDictValueIterator::extend_class(context, &context.dictvalueiterator_type); + PyDictItems::extend_class(context, &context.dictitems_type); + PyDictItemIterator::extend_class(context, &context.dictitemiterator_type); } From 2811f27bb546a811ea54628df5b73cf6e61a8d48 Mon Sep 17 00:00:00 2001 From: andrew Date: Mon, 8 Apr 2019 16:03:24 +0200 Subject: [PATCH 134/223] add a warning --- tests/generator/not_impl_header.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_header.txt index cd6f8f5a1..4ceefff4d 100644 --- a/tests/generator/not_impl_header.txt +++ b/tests/generator/not_impl_header.txt @@ -1,2 +1,7 @@ +# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE +# EDIT tests/not_impl_gen.py, NOT THIS FILE. +# RESULTS OF THIS TEST DEPEND ON THE CPYTHON +# VERSION USED TO RUN not_impl_gen.py + import platform From 3aa110a2960a7572852f98b8f97a544dcb379d1c Mon Sep 17 00:00:00 2001 From: andrew Date: Mon, 8 Apr 2019 16:03:35 +0200 Subject: [PATCH 135/223] add .gitignore --- tests/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/.gitignore diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 000000000..05ba24c43 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +snippets/whats_left_to_implement.py From b1a95f669f9a5fcb5a439ce240b7c767d36cd04b Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Mon, 8 Apr 2019 17:02:27 +0200 Subject: [PATCH 136/223] Initial version of symbol table builder. --- vm/src/compile.rs | 11 +- vm/src/lib.rs | 1 + vm/src/symboltable.rs | 272 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 vm/src/symboltable.rs diff --git a/vm/src/compile.rs b/vm/src/compile.rs index dd045c3bf..cc52661a4 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -10,6 +10,7 @@ use crate::error::CompileError; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; +use crate::symboltable::SymbolTable; use crate::VirtualMachine; use num_complex::Complex64; use rustpython_parser::{ast, parser}; @@ -37,6 +38,8 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; + let mut symbol_table = SymbolTable::new(); + symbol_table.scan_program(&ast); compiler.compile_program(&ast) } Mode::Eval => { @@ -194,11 +197,8 @@ impl Compiler { // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::Global { names } => { - unimplemented!("global {:?}", names); - } - ast::Statement::Nonlocal { names } => { - unimplemented!("nonlocal {:?}", names); + ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => { + // Handled during symbol table construction. } ast::Statement::If { test, body, orelse } => { let end_label = self.new_label(); @@ -1624,6 +1624,7 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option< (body, None) } + #[cfg(test)] mod tests { use super::Compiler; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 89b4dd872..578f1d99a 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -55,6 +55,7 @@ pub mod import; pub mod obj; pub mod pyobject; pub mod stdlib; +mod symboltable; mod sysmodule; mod traceback; pub mod util; diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs new file mode 100644 index 000000000..6525e71e1 --- /dev/null +++ b/vm/src/symboltable.rs @@ -0,0 +1,272 @@ +/* Python code is pre-scanned for symbols in the ast. + +This ensures that global and nonlocal keywords are picked up. +Then the compiler can use the symbol table to generate proper +load and store instructions for names. +*/ + +use rustpython_parser::ast; +use std::collections::HashMap; + +pub enum SymbolRole { + Global, + Nonlocal, + Used, + Assigned, +} + +pub struct SymbolTable { + // TODO: split-up into nested scopes. + symbols: HashMap, +} + +impl SymbolTable { + pub fn new() -> Self { + SymbolTable { + symbols: HashMap::new(), + } + } + + pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { + return self.symbols.get(name); + } + + pub fn scan_program(&mut self, program: &ast::Program) { + self.scan_statements(&program.statements); + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) { + for statement in statements { + self.scan_statement(statement) + } + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) { + match &statement.node { + ast::Statement::Global { names } => { + for name in names { + self.register_name(name, SymbolRole::Global); + } + } + ast::Statement::Nonlocal { names } => { + for name in names { + self.register_name(name, SymbolRole::Nonlocal); + } + } + ast::Statement::FunctionDef { + name, + body, + args, + decorator_list, + returns, + } => { + self.scan_expressions(decorator_list); + self.register_name(name, SymbolRole::Assigned); + for parameter in &args.args {} + for parameter in &args.kwonlyargs {} + + self.scan_expressions(&args.defaults); + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression); + } + } + + self.scan_statements(body); + if let Some(expression) = returns { + self.scan_expression(expression); + } + } + ast::Statement::ClassDef { + name, + body, + bases, + keywords, + decorator_list, + } => { + self.register_name(name, SymbolRole::Assigned); + self.scan_statements(body); + self.scan_expressions(bases); + for keyword in keywords { + self.scan_expression(&keyword.value); + } + self.scan_expressions(decorator_list); + } + ast::Statement::Expression { expression } => self.scan_expression(expression), + ast::Statement::If { test, body, orelse } => { + self.scan_expression(test); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::For { + target, + iter, + body, + orelse, + } => { + self.scan_expression(target); + self.scan_expression(iter); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::While { test, body, orelse } => { + self.scan_expression(test); + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + } + ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { + // No symbols here. + } + ast::Statement::Import { import_parts } => for part in import_parts {}, + ast::Statement::Return { value } => { + if let Some(expression) = value { + self.scan_expression(expression); + } + } + ast::Statement::Assert { test, msg } => { + self.scan_expression(test); + if let Some(expression) = msg { + self.scan_expression(expression); + } + } + ast::Statement::Delete { targets } => { + self.scan_expressions(targets); + } + ast::Statement::Assign { targets, value } => { + self.scan_expressions(targets); + self.scan_expression(value); + } + ast::Statement::AugAssign { + target, + op: _, + value, + } => { + self.scan_expression(target); + self.scan_expression(value); + } + ast::Statement::With { items, body } => { + for item in items { + self.scan_expression(&item.context_expr); + if let Some(expression) = &item.optional_vars { + self.scan_expression(expression); + } + } + self.scan_statements(body); + } + ast::Statement::Try { + body, + handlers, + orelse, + finalbody, + } => { + self.scan_statements(body); + if let Some(code) = orelse { + self.scan_statements(code); + } + if let Some(code) = finalbody { + self.scan_statements(code); + } + } + ast::Statement::Raise { exception, cause } => { + if let Some(expression) = exception { + self.scan_expression(expression); + } + if let Some(expression) = cause { + self.scan_expression(expression); + } + } + } + } + + fn scan_expressions(&mut self, expressions: &[ast::Expression]) { + for expression in expressions { + self.scan_expression(expression); + } + } + + fn scan_expression(&mut self, expression: &ast::Expression) { + match expression { + ast::Expression::Binop { a, op: _, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::BoolOp { a, op: _, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::Compare { vals, ops: _ } => { + self.scan_expressions(vals); + } + ast::Expression::Subscript { a, b } => { + self.scan_expression(a); + self.scan_expression(b); + } + ast::Expression::Attribute { value, name: _ } => { + self.scan_expression(value); + } + ast::Expression::Dict { elements } => { + for (key, value) in elements { + self.scan_expression(key); + self.scan_expression(value); + } + } + ast::Expression::Yield { value } => { + if let Some(expression) = value { + self.scan_expression(expression); + } + } + ast::Expression::YieldFrom { value } => { + self.scan_expression(value); + } + ast::Expression::Unop { op: _, a } => { + self.scan_expression(a); + } + ast::Expression::True + | ast::Expression::False + | ast::Expression::None + | ast::Expression::Ellipsis => {} + ast::Expression::Number { .. } => {} + ast::Expression::Starred { value } => { + self.scan_expression(value); + } + ast::Expression::Bytes { .. } => {} + ast::Expression::Tuple { elements } + | ast::Expression::Set { elements } + | ast::Expression::List { elements } + | ast::Expression::Slice { elements } => { + self.scan_expressions(elements); + } + ast::Expression::Comprehension { kind, generators } => {} + ast::Expression::Call { + function, + args, + keywords, + } => { + self.scan_expression(function); + self.scan_expressions(args); + } + ast::Expression::String { value } => {} + ast::Expression::Identifier { name } => { + self.register_name(name, SymbolRole::Used); + } + ast::Expression::Lambda { args, body } => { + self.scan_expression(body); + } + ast::Expression::IfExpression { test, body, orelse } => { + self.scan_expression(test); + self.scan_expression(body); + self.scan_expression(orelse); + } + } + } + + fn register_name(&mut self, name: &String, role: SymbolRole) { + self.symbols.insert(name.clone(), role); + } +} From e7e126e31ded0b1da77d945f0f92d631b0055d57 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 8 Apr 2019 12:30:53 -0500 Subject: [PATCH 137/223] Fix whats_left.sh --- whats_left.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/whats_left.sh b/whats_left.sh index c6f191bd5..e05051d28 100755 --- a/whats_left.sh +++ b/whats_left.sh @@ -1,5 +1,11 @@ #!/bin/sh +set -e -cd "$(dirname "$0")" || exit +cd "$(dirname "$0")" +cd tests + +python3 not_impl_gen.py + +cd .. cargo run -- tests/snippets/whats_left_to_implement.py From 4e277cfcc2d6eac7aa15cc493dacaa46fa8971d8 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 12:41:28 -0600 Subject: [PATCH 138/223] move to useing PyIterable --- vm/src/obj/objlist.rs | 108 ++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 1ef01ca3e..4c3c16383 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -7,7 +7,10 @@ use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{ + IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, +}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; @@ -189,7 +192,12 @@ impl PyListRef { ) -> PyResult { match subscript { SequenceIndex::Int(index) => self.setindex(index, value, vm), - SequenceIndex::Slice(slice) => self.setslice(slice, value, vm), + SequenceIndex::Slice(slice) => { + if let Ok(sec) = PyIterable::try_from_object(vm, value) { + return self.setslice(slice, sec, vm); + } + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } } } @@ -202,7 +210,7 @@ impl PyListRef { } } - fn setslice(self, slice: PySliceRef, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn setslice(self, slice: PySliceRef, sec: PyIterable, vm: &VirtualMachine) -> PyResult { let step = slice.step.clone().unwrap_or_else(BigInt::one); if step.is_zero() { @@ -244,52 +252,50 @@ impl PyListRef { } } - fn _set_slice(self, range: Range, sec: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // consume the iter, we don't need it's size - // but if it's going to fail we want that to happen *before* we start modifing - if let Ok(items) = vm.extract_elements(&sec) { - // replace the range of elements with the full sequence - self.elements.borrow_mut().splice(range, items); + fn _set_slice(self, range: Range, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - Ok(vm.get_none()) - } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) - } + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) } fn _set_stepped_slice( self, range: Range, step: usize, - sec: PyObjectRef, + sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { + let slicelen = ((range.end - range.start - 1) / step) + 1; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing - let slicelen = ((range.end - range.start - 1) / step) + 1; - if let Ok(items) = vm.extract_elements(&sec) { - let n = items.len(); + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - if range.start < range.end { - if n == slicelen { - let indexes = range.step_by(step); - self._replace_indexes(indexes, &items); - Ok(vm.get_none()) - } else { - Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size {}", - n, slicelen - ))) - } + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) } else { - // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size 0", - n + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen ))) } } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) } } @@ -297,35 +303,35 @@ impl PyListRef { self, range: Range, step: usize, - sec: PyObjectRef, + sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { + let slicelen = ((range.end - range.start - 1) / step) + 1; + // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing - let slicelen = ((range.end - range.start - 1) / step) + 1; - if let Ok(items) = vm.extract_elements(&sec) { - let n = items.len(); + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; - if range.start < range.end { - if n == slicelen { - let indexes = range.rev().step_by(step); - self._replace_indexes(indexes, &items); - Ok(vm.get_none()) - } else { - Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size {}", - n, slicelen - ))) - } + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) } else { - // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( - "attempt to assign sequence of size {} to extended slice of size 0", - n + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen ))) } } else { - Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) } } From d0e6d2fa323992ce1bce5ad998a55c1d04c7883c Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 13:18:37 -0600 Subject: [PATCH 139/223] add tests for custom iters --- tests/snippets/list.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 71f7dea06..55c84fdf9 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -399,3 +399,55 @@ def must_assign_iter_to_slice(): x[::2] = 42 assert_raises(TypeError, must_assign_iter_to_slice) + +# other iterables? +a = list(range(10)) + +# string +x = a[:] +x[3:8] = "abcdefghi" +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] + +# tuple +x = a[:] +x[3:8] = (11, 12, 13, 14, 15) +assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + +# class +class CIterNext: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + def __iter__(self): + return self + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + +x = list(range(10)) +x[3:8] = CIterNext() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +class CIter: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __iter__(self): + for n in self.sec: + yield n + +x = list(range(10)) +x[3:8] = CIter() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +class CGetItem: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __getitem__(self, sub): + return self.sec[sub] + +x = list(range(10)) +x[3:8] = CGetItem() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] From 8232a4d28512323125f853e3d448cc190c2c3a6f Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 21:27:44 +0200 Subject: [PATCH 140/223] finish contains --- tests/snippets/bytes.py | 45 ++++++++++++++++++++------------------ vm/src/obj/objbyteinner.rs | 14 ++++++++---- vm/src/obj/objbytes.rs | 12 +--------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 2fd32153d..b03eb3808 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -1,10 +1,10 @@ from testutils import assertRaises # new -assert bytes([1,2,3]) -assert bytes((1,2,3)) +assert bytes([1, 2, 3]) +assert bytes((1, 2, 3)) assert bytes(range(4)) -assert b'bla' +assert b"bla" assert bytes(3) assert bytes("bla", "utf8") try: @@ -20,40 +20,39 @@ c = b"abcd" # # repr -assert repr(bytes([0, 1, 2])) == repr(b'\x00\x01\x02') -assert ( -repr(bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) -== "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'") +assert repr(bytes([0, 1, 2])) == repr(b"\x00\x01\x02") +assert repr( + bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) + == "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'" ) assert repr(b"abcd") == "b'abcd'" -#len +# len assert len(bytes("abcdé", "utf8")) == 6 -#comp +# comp assert a == b"abcd" assert a > b assert a >= b assert b < a assert b <= a -assert b'foobar'.__eq__(2) == NotImplemented -assert b'foobar'.__ne__(2) == NotImplemented -assert b'foobar'.__gt__(2) == NotImplemented -assert b'foobar'.__ge__(2) == NotImplemented -assert b'foobar'.__lt__(2) == NotImplemented -assert b'foobar'.__le__(2) == NotImplemented +assert b"foobar".__eq__(2) == NotImplemented +assert b"foobar".__ne__(2) == NotImplemented +assert b"foobar".__gt__(2) == NotImplemented +assert b"foobar".__ge__(2) == NotImplemented +assert b"foobar".__lt__(2) == NotImplemented +assert b"foobar".__le__(2) == NotImplemented -#hash +# hash hash(a) == hash(b"abcd") -#iter +# iter [i for i in b"abcd"] == ["a", "b", "c", "d"] -#add +# add assert a + b == b"abcdab" -#contains # contains assert b"ab" in b"abcd" assert b"cd" in b"abcd" @@ -61,5 +60,9 @@ assert b"abcd" in b"abcd" assert b"a" in b"abcd" assert b"d" in b"abcd" assert b"dc" not in b"abcd" -# assert 97 in b"abcd" -# assert 150 not in b"abcd" \ No newline at end of file +assert 97 in b"abcd" +assert 150 not in b"abcd" +try: + 350 in b"abcd" +except ValueError: + pass diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 03876095c..95df84421 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -181,11 +181,17 @@ impl PyByteInner { Ok(vm.new_bool(false)) } - pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { - self.elements.contains(int); - Ok(vm.new_bool(false)) + pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(int) = int.as_bigint().to_u8() { + if self.elements.contains(&int) { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } else { + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } } } - // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index e052654bf..f376dbc67 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,3 +1,4 @@ +use crate::obj::objint::PyInt; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -143,17 +144,6 @@ impl PyBytesRef { bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), int @ PyInt => self.inner.contains_int(&int, vm), _ => Ok(vm.ctx.not_implemented())) - // if objtype::isinstance(&needle, &vm.ctx.bytes_type()) { - // let result = vec_contains(&self.value, &get_value(&needle)); - // vm.ctx.new_bool(result) - // } else if objtype::isinstance(&needle, &vm.ctx.int_type()) { - // let result = self - // .value - // .contains(&objint::get_value(&needle).to_u8().unwrap()); - // vm.ctx.new_bool(result) - // } else { - // vm.new_type_error(format!("Cannot add {:?} and {:?}", self, needle)) - // } } } From 4c50defa37d5a38e75247d6ac32fd3c680f9e7fc Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 13:46:23 -0600 Subject: [PATCH 141/223] extend PyIterable::try_from_object to work with sequences --- vm/src/pyobject.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index ee765f987..be51128ef 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,4 +1,5 @@ use std::any::Any; +use std::cell::Cell; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; @@ -1043,10 +1044,24 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - Ok(PyIterable { - method: vm.get_method(obj, "__iter__")?, - _item: std::marker::PhantomData, - }) + if let Ok(method) = vm.get_method(obj.clone(), "__iter__") { + Ok(PyIterable { + method: method, + _item: std::marker::PhantomData, + }) + } else if vm.get_method(obj.clone(), "__getitem__").is_ok() { + Self::try_from_object( + vm, + objiter::PySequenceIterator { + position: Cell::new(0), + obj: obj.clone(), + } + .into_ref(vm) + .into_object(), + ) + } else { + Err(vm.new_type_error(format!("'{}' object is not iterable", obj.class().name))) + } } } From 4d53ddedb020c100eeb6e45675aebd572909f9dd Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 14:45:34 -0600 Subject: [PATCH 142/223] test for if iter raises error --- tests/snippets/list.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 55c84fdf9..217e79089 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -414,6 +414,7 @@ x[3:8] = (11, 12, 13, 14, 15) assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] # class +# __next__ class CIterNext: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -431,6 +432,7 @@ x = list(range(10)) x[3:8] = CIterNext() assert x == [0, 1, 2, 1, 2, 3, 8, 9] +# __iter__ yield class CIter: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -442,6 +444,7 @@ x = list(range(10)) x[3:8] = CIter() assert x == [0, 1, 2, 1, 2, 3, 8, 9] +# __getitem but no __iter__ sequence class CGetItem: def __init__(self, sec=(1, 2, 3)): self.sec = sec @@ -451,3 +454,17 @@ class CGetItem: x = list(range(10)) x[3:8] = CGetItem() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# iter raises error +class CIterError: + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + +def bad_iter_assign(): + x = list(range(10)) + x[3:8] = CIterError() + +assert_raises(RuntimeError, bad_iter_assign) From b53595831005023bdfeb6e5c93eb40b3286775f9 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 8 Apr 2019 23:12:40 +0200 Subject: [PATCH 143/223] add getitem, fix contain error_message --- tests/snippets/bytes.py | 11 +++++++++++ vm/src/obj/objbyteinner.rs | 18 +++++++++++++++++- vm/src/obj/objbytes.rs | 12 ++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index b03eb3808..770f4f373 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -66,3 +66,14 @@ try: 350 in b"abcd" except ValueError: pass + + +# getitem +d = b"abcdefghij" + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == b"cdef" +assert d[-6:] == b"efghij" +assert d[1:8:2] == b"bdfh" +assert d[8:1:-2] == b"igec" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 95df84421..bb9966ebf 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -11,6 +11,7 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use super::objint; +use super::objsequence::PySliceableSequence; use crate::obj::objint::PyInt; use num_traits::ToPrimitive; @@ -189,9 +190,24 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } else { - Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + Err(vm.new_value_error("byte mu st be in range(0, 256)".to_string())) } } + + pub fn getitem_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + Ok(vm.new_int(self.elements[idx])) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + pub fn getitem_slice(&self, slice: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) + } } + // TODO // fix b"é" not allowed should be bytes("é", "utf8") diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index f376dbc67..2c15e60cd 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -8,6 +8,7 @@ use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyVa use super::objbyteinner::PyByteInner; use super::objiter; +use super::objslice::PySlice; use super::objtype::PyClassRef; /// "bytes(iterable_of_ints) -> bytes\n\ @@ -139,11 +140,18 @@ impl PyBytesRef { #[pymethod(name = "__contains__")] fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // no new style since objint is not. match_class!(needle, bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), int @ PyInt => self.inner.contains_int(&int, vm), - _ => Ok(vm.ctx.not_implemented())) + obj => Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))) + } + + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(needle, + int @ PyInt => self.inner.getitem_int(&int, vm), + slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), + obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) } } From b2a1f6580b6dcf3eec299ec1423513c682aa18eb Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 15:56:06 -0600 Subject: [PATCH 144/223] fix error when start or stop of slice is -1 and step != 1 --- tests/snippets/list.py | 10 ++++++++++ vm/src/obj/objlist.rs | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 217e79089..dc0582d4c 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -468,3 +468,13 @@ def bad_iter_assign(): x[3:8] = CIterError() assert_raises(RuntimeError, bad_iter_assign) + +# slice assign when step or stop is -1 + +a = list(range(10)) +x = a[:] +x[-1:-5:-1] = ['a', 'b', 'c', 'd'] +assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x = a[:] +x[-5:-1:-1] = [] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 4c3c16383..6de08dfef 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -3,7 +3,7 @@ use std::fmt; use std::ops::Range; -use num_bigint::BigInt; +use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; @@ -238,8 +238,20 @@ impl PyListRef { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = &slice.start.as_ref().map(|x| x + 1); - let stop = &slice.stop.as_ref().map(|x| x + 1); + let start = &slice.start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = &slice.stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); match (-step).to_i32() { Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), @@ -271,7 +283,11 @@ impl PyListRef { sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { - let slicelen = ((range.end - range.start - 1) / step) + 1; + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing let items: Result, _> = sec.iter(vm)?.collect(); @@ -290,6 +306,9 @@ impl PyListRef { n, slicelen ))) } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) } else { // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( @@ -306,7 +325,11 @@ impl PyListRef { sec: PyIterable, vm: &VirtualMachine, ) -> PyResult { - let slicelen = ((range.end - range.start - 1) / step) + 1; + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; // consume the iter, we need it's size // and if it's going to fail we want that to happen *before* we start modifing @@ -326,6 +349,9 @@ impl PyListRef { n, slicelen ))) } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) } else { // empty slice but this is an error because stepped slice Err(vm.new_value_error(format!( From 1101b6571ccded8b05e252b47658448d04382783 Mon Sep 17 00:00:00 2001 From: Rachel Powers Date: Mon, 8 Apr 2019 16:17:57 -0600 Subject: [PATCH 145/223] fix error related to #746 but for list.__delitem__ --- tests/snippets/list.py | 8 ++++++++ vm/src/obj/objlist.rs | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 3d9743c0b..41de1b5fd 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -201,3 +201,11 @@ assert_raises(TypeError, bad_del_1) def bad_del_2(): del ['a', 'b'][2] assert_raises(IndexError, bad_del_2) + +# is step != 1 and start or stop of slice == -1 +x = list(range(10)) +del x[-1:-5:-1] +assert x == [0, 1, 2, 3, 4, 5] +x = list(range(10)) +del x[-5:-1:-1] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 60c14b107..c5278e1fb 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -3,7 +3,7 @@ use std::fmt; use std::ops::Range; -use num_bigint::BigInt; +use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; @@ -411,8 +411,20 @@ impl PyListRef { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { From 422ab69b99af4611cd8d3f416cfe24e94894782c Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 8 Apr 2019 21:50:38 -0500 Subject: [PATCH 146/223] Use impl style classes --- wasm/lib/src/browser_module.rs | 54 ++++++++++++++++------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 607cfc886..d302ec99a 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -10,7 +10,7 @@ use rustpython_vm::obj::{ objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; @@ -161,6 +161,7 @@ fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult Ok(vm.get_none()) } +#[pyclass(name = "Promise")] #[derive(Debug)] pub struct PyPromise { value: Promise, @@ -173,6 +174,7 @@ impl PyValue for PyPromise { } } +#[pyimpl] impl PyPromise { pub fn new(value: Promise) -> PyPromise { PyPromise { value } @@ -187,15 +189,16 @@ impl PyPromise { self.value.clone() } + #[pymethod] fn then( - zelf: PyPromiseRef, + &self, on_fulfill: PyFunctionRef, on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); @@ -220,10 +223,11 @@ impl PyPromise { PyPromise::from_future(ret_future).into_ref(vm) } - fn catch(zelf: PyPromiseRef, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + #[pymethod] + fn catch(&self, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { res.or_else(|err| { let stored_vm = weak_vm .upgrade() @@ -239,11 +243,11 @@ impl PyPromise { } } +#[pyclass] #[derive(Debug)] struct Document { doc: web_sys::Document, } -type DocumentRef = PyRef; impl PyValue for Document { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -251,9 +255,11 @@ impl PyValue for Document { } } +#[pyimpl] impl Document { - fn query(zelf: DocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { - let elem = zelf + #[pymethod] + fn query(&self, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = self .doc .query_selector(&query.value) .map_err(|err| convert::js_py_typeerror(vm, err))?; @@ -265,11 +271,11 @@ impl Document { } } +#[pyclass] #[derive(Debug)] struct Element { elem: web_sys::Element, } -type ElementRef = PyRef; impl PyValue for Element { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -277,26 +283,24 @@ impl PyValue for Element { } } +#[pyimpl] impl Element { + #[pymethod] fn get_attr( - zelf: ElementRef, + &self, attr: PyStringRef, default: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - match zelf.elem.get_attribute(&attr.value) { + match self.elem.get_attribute(&attr.value) { Some(s) => vm.new_str(s), None => default.into_option().unwrap_or_else(|| vm.get_none()), } } - fn set_attr( - zelf: ElementRef, - attr: PyStringRef, - value: PyStringRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.elem + #[pymethod] + fn set_attr(&self, attr: PyStringRef, value: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + self.elem .set_attribute(&attr.value, &value.value) .map_err(|err| convert::js_to_py(vm, err)) } @@ -340,14 +344,9 @@ fn browser_prompt( pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let promise = py_class!(ctx, "Promise", ctx.object(), { - "then" => ctx.new_rustfunc(PyPromise::then), - "catch" => ctx.new_rustfunc(PyPromise::catch), - }); + let promise = PyPromise::make_class(ctx); - let document_class = py_class!(ctx, "Document", ctx.object(), { - "query" => ctx.new_rustfunc(Document::query), - }); + let document_class = Document::make_class(ctx); let document = PyObject::new( Document { @@ -357,10 +356,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { None, ); - let element = py_class!(ctx, "Element", ctx.object(), { - "get_attr" => ctx.new_rustfunc(Element::get_attr), - "set_attr" => ctx.new_rustfunc(Element::set_attr), - }); + let element = Element::make_class(ctx); py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), From 046e4b3fe37648e86ad69436fb7cee26b2a2c3f9 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 9 Apr 2019 20:43:59 +1200 Subject: [PATCH 147/223] Cleanup dict test snippet --- tests/snippets/dict.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f924bb0e7..23d6e933b 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -110,16 +110,10 @@ assert list(x) == ['a', 'b'] y = x.copy() x['c'] = 12 -assert list(y) == ['a', 'b'] -assert y['a'] == 2 -assert y['b'] == 10 +assert dict_eq(y, {'a': 2, 'b': 10}) y.update({'c': 19, "d": -1, 'b': 12}) -assert list(y) == ['a', 'b', 'c', 'd'] -assert y['a'] == 2 -assert y['b'] == 12 -assert y['c'] == 19 -assert y['d'] == -1 +assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) y.update(y) -assert list(y) == ['a', 'b', 'c', 'd'] +assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) # hasn't changed From 23a4b1e9d86bed3acd95ed3917023ace2456fad3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:13:44 +0100 Subject: [PATCH 148/223] Additional tests for dictionary iteration. --- tests/snippets/dict.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index f57fc37b6..44d1c382f 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -37,13 +37,36 @@ for key in a.keys(): res.add(key) assert res == set(['a','b']) +# Deleted values are correctly skipped over: x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} del x['c'] it = iter(x.items()) assert ('a', 1) == next(it) assert ('b', 2) == next(it) assert ('d', 3) == next(it) +with assertRaises(StopIteration): + next(it) +# Iterating a dictionary is just its keys: +assert ['a', 'b', 'd'] == list(x) + +# Iterating view captures dictionary when iterated. +data = {1: 2, 3: 4} +items = data.items() +assert list(items) == [(1, 2), (3, 4)] +data[5] = 6 +assert list(items) == [(1, 2), (3, 4), (5, 6)] + +# Values can be changed during iteration. +data = {1: 2, 3: 4} +items = iter(data.items()) +assert (1, 2) == next(items) +data[3] = "changed" +assert (3, "changed") == next(items) + +# View isn't itself an iterator. +with assertRaises(TypeError): + next(data.keys()) x = {} x[1] = 1 From d9216c87f1acde0f0a7f092d943f0ffe8bdebd41 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:17:17 +0100 Subject: [PATCH 149/223] Remove done todo. --- vm/src/obj/objdict.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 612fa2cb7..53445e01f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -360,7 +360,6 @@ pub fn init(context: &PyContext) { "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), - // TODO: separate type. `keys` should be a live view over the collection, not an iterator. "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), "copy" => context.new_rustfunc(PyDictRef::copy), From 25d3f83e4df7548010c2d090e8fc5193c3d75b08 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 10:44:04 +0100 Subject: [PATCH 150/223] Add __len__ to dictionary views. --- tests/snippets/dict.py | 2 ++ vm/src/obj/objdict.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 091b03405..45c6c3b43 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -68,6 +68,8 @@ assert (3, "changed") == next(items) with assertRaises(TypeError): next(data.keys()) +assert len(data.keys()) == 2 + x = {} x[1] = 1 assert x[1] == 1 diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 53445e01f..6d2aefd78 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -265,6 +265,11 @@ macro_rules! dict_iterator { fn iter(&self, _vm: &VirtualMachine) -> $iter_name { $iter_name::new(self.dict.clone()) } + + #[pymethod(name = "__len__")] + fn len(&self, vm: &VirtualMachine) -> usize { + self.dict.clone().len(vm) + } } impl PyValue for $name { From ec5fd550debd3c2bbdc646215484dfc81fb3de3f Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 11:25:18 +0100 Subject: [PATCH 151/223] Implement IntoIterator for PyDictRef. --- vm/src/dictdatatype.rs | 12 ------- vm/src/frame.rs | 15 ++++----- vm/src/obj/objdict.rs | 73 ++++++++++++++++++++++++++++++----------- vm/src/obj/objmodule.rs | 6 +--- vm/src/obj/objobject.rs | 2 +- vm/src/stdlib/json.rs | 2 +- 6 files changed, 63 insertions(+), 47 deletions(-) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 0612bc8a3..6dc49125b 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -131,18 +131,6 @@ impl Dict { None } - pub fn iter_items(&self) -> impl Iterator + '_ { - self.entries - .iter() - .filter(|e| e.is_some()) - .map(|e| e.as_ref().unwrap()) - .map(|e| (e.key.clone(), e.value.clone())) - } - - pub fn get_items(&self) -> Vec<(PyObjectRef, T)> { - self.iter_items().collect() - } - /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { let hash_value = calc_hash(vm, key)?; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2570c0934..ff9baba35 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -393,9 +393,8 @@ impl Frame { // Take all key-value pairs from the dict: let dict: PyDictRef = obj.downcast().expect("Need a dictionary to build a map."); - let dict_elements = dict.get_key_value_pairs(); - for (key, value) in dict_elements.iter() { - map_obj.set_item(key.clone(), value.clone(), vm).unwrap(); + for (key, value) in dict { + map_obj.set_item(key, value, vm).unwrap(); } } } else { @@ -636,8 +635,7 @@ impl Frame { let kwargs = if *has_kwargs { let kw_dict: PyDictRef = self.pop_value().downcast().expect("Kwargs must be a dict."); - let dict_elements = kw_dict.get_key_value_pairs(); - dict_elements + kw_dict .into_iter() .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() @@ -862,8 +860,8 @@ impl Frame { // Grab all the names from the module and put them in the context if let Some(dict) = &module.dict { - for (k, v) in dict.get_key_value_pairs().iter() { - self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + for (k, v) in dict { + self.scope.store_name(&vm, &objstr::get_value(&k), v); } } Ok(None) @@ -1230,8 +1228,7 @@ impl fmt::Debug for Frame { .collect::(); let dict = self.scope.get_locals(); let local_str = dict - .get_key_value_pairs() - .iter() + .into_iter() .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) .collect::(); write!( diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 53445e01f..b0d99d4e7 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,8 +3,7 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, - PyValue, + IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -63,9 +62,8 @@ impl PyDictRef { if let OptionalArg::Present(dict_obj) = dict_obj { let dicted: PyResult = dict_obj.clone().downcast(); if let Ok(dict_obj) = dicted { - let mut dict_borrowed = dict.borrow_mut(); - for (key, value) in dict_obj.entries.borrow().iter_items() { - dict_borrowed.insert(vm, &key, value)?; + for (key, value) in dict_obj { + dict.borrow_mut().insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -105,7 +103,7 @@ impl PyDictRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; - for (key, value) in self.get_key_value_pairs() { + for (key, value) in self { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -146,10 +144,6 @@ impl PyDictRef { PyDictItems::new(self) } - pub fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - self.entries.borrow().get_items() - } - fn inner_setitem( self, key: PyObjectRef, @@ -194,24 +188,17 @@ impl PyDictRef { fn update( self, - mut dict_obj: OptionalArg, + dict_obj: OptionalArg, kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult<()> { - if let OptionalArg::Present(ref other) = dict_obj { - if self.is(other) { - // Updating yourself is a noop, and this avoids a borrow error - dict_obj = OptionalArg::Missing; - } - } - PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) } /// Take a python dictionary and convert it to attributes. pub fn to_attributes(self) -> PyAttributes { let mut attrs = PyAttributes::new(); - for (key, value) in self.get_key_value_pairs() { + for (key, value) in self { let key = objstr::get_value(&key); attrs.insert(key, value); } @@ -247,6 +234,54 @@ impl ItemProtocol for PyDictRef { } } +// Implement IntoIterator so that we can easily iterate dictionaries from rust code. +impl IntoIterator for PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIterator; + + fn into_iter(self) -> Self::IntoIter { + DictIterator::new(self) + } +} + +impl IntoIterator for &PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIterator; + + fn into_iter(self) -> Self::IntoIter { + DictIterator::new(self.clone()) + } +} + +pub struct DictIterator { + dict: PyDictRef, + position: Cell, +} + +impl DictIterator { + pub fn new(dict: PyDictRef) -> DictIterator { + DictIterator { + dict, + position: Cell::new(0), + } + } +} + +impl Iterator for DictIterator { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option { + self.dict + .entries + .borrow() + .next_entry(self.position.get()) + .map(|(new_position, key, value)| { + self.position.set(new_position); + (key.clone(), value.clone()) + }) + } +} + macro_rules! dict_iterator { ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { #[pyclass(name = $class_name, __inside_vm)] diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index 75749d557..904487e4d 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -17,11 +17,7 @@ impl PyValue for PyModule { impl PyModuleRef { fn dir(self: PyModuleRef, vm: &VirtualMachine) -> PyResult { if let Some(dict) = &self.into_object().dict { - let keys = dict - .get_key_value_pairs() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); + let keys = dict.into_iter().map(|(k, _v)| k.clone()).collect(); Ok(vm.ctx.new_list(keys)) } else { panic!("Modules should definitely have a dict."); diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index a3bc41103..f8e86d0a1 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -237,7 +237,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in dict.get_key_value_pairs() { + for (key, value) in dict.into_iter() { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index f4061bc2a..3fa30e16b 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -64,7 +64,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); - let pairs = dict.get_key_value_pairs(); + let pairs: Vec<(PyObjectRef, PyObjectRef)> = dict.into_iter().collect(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; From 2940c7cc280143de25777198e1bb3631b3f54104 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 14:33:29 +0200 Subject: [PATCH 152/223] add islpaha isalnum isdigit islower isupper isspace istitle --- tests/snippets/bytes.py | 35 +++++++++++++++ vm/src/obj/objbyteinner.rs | 90 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 33 ++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 770f4f373..ceb52921c 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -77,3 +77,38 @@ assert d[2:6] == b"cdef" assert d[-6:] == b"efghij" assert d[1:8:2] == b"bdfh" assert d[8:1:-2] == b"igec" + + +# is_xx methods + +assert bytes(b"1a23").isalnum() +assert not bytes(b"1%a23").isalnum() + +assert bytes(b"abc").isalpha() +assert not bytes(b"abc1").isalpha() + +# travis doesn't like this +# assert bytes(b'xyz').isascii() +# assert not bytes([128, 157, 32]).isascii() + +assert bytes(b"1234567890").isdigit() +assert not bytes(b"12ab").isdigit() + +l = bytes(b"lower") +b = bytes(b"UPPER") + +assert l.islower() +assert not l.isupper() +assert b.isupper() +assert not bytes(b"Super Friends").islower() + +assert bytes(b" \n\t").isspace() +assert not bytes(b"\td\n").isspace() + +assert b.isupper() +assert not b.islower() +assert l.islower() +assert not bytes(b"tuPpEr").isupper() + +assert bytes(b"Is Title Case").istitle() +assert not bytes(b"is Not title casE").istitle() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bb9966ebf..14da2521e 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -207,6 +207,96 @@ impl PyByteInner { .ctx .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) } + + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_alphanumeric()), + )) + } + + pub fn isalpha(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_alphabetic()), + )) + } + + pub fn isascii(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_ascii()), + )) + } + + pub fn isdigit(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_digit(10)), + )) + } + + pub fn islower(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_lowercase()), + )) + } + + pub fn isspace(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_whitespace()), + )) + } + + pub fn isupper(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_uppercase()), + )) + } + + pub fn istitle(&self, vm: &VirtualMachine) -> PyResult { + if self.elements.is_empty() { + return Ok(vm.new_bool(false)); + } + + let mut iter = self.elements.iter().peekable(); + let mut prev_cased = false; + + while let Some(c) = iter.next() { + let current = char::from(*c); + let next = if let Some(k) = iter.peek() { + char::from(**k) + } else if current.is_uppercase() { + return Ok(vm.new_bool(!prev_cased)); + } else { + return Ok(vm.new_bool(prev_cased)); + }; + + let is_cased = current.to_uppercase().next().unwrap() != current + || current.to_lowercase().next().unwrap() != current; + if (is_cased && next.is_uppercase() && !prev_cased) + || (!is_cased && next.is_lowercase()) + { + return Ok(vm.new_bool(false)); + } + + prev_cased = is_cased; + } + + Ok(vm.new_bool(true)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2c15e60cd..83ce3b8fb 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -153,6 +153,39 @@ impl PyBytesRef { slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) } + + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalnum(vm) + } + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalpha(vm) + } + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.isascii(vm) + } + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.isdigit(vm) + } + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.islower(vm) + } + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.isspace(vm) + } + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.isupper(vm) + } + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.istitle(vm) + } } #[derive(Debug)] From 6c745f68dd6d123fc491cc513429ede529fa3a2e Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 14:43:13 +0200 Subject: [PATCH 153/223] fix typo, fix bytesinner.add now return Vec[u8] --- vm/src/obj/objbyteinner.rs | 6 +++--- vm/src/obj/objbytes.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 14da2521e..bcb1f8207 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -55,7 +55,7 @@ impl PyByteInner { } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } - // On ly one argument + // Only one argument } else { let value = if let OptionalArg::Present(ival) = val_option { match_class!(ival.clone(), @@ -161,14 +161,14 @@ impl PyByteInner { hasher.finish() as usize } - pub fn add(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + pub fn add(&self, other: &PyByteInner, _vm: &VirtualMachine) -> Vec { let elements: Vec = self .elements .iter() .chain(other.elements.iter()) .cloned() .collect(); - Ok(vm.ctx.new_bytes(elements)) + elements } pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 83ce3b8fb..0b257cb77 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -134,7 +134,7 @@ impl PyBytesRef { #[pymethod(name = "__add__")] fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { match_class!(other, - bytes @ PyBytes => self.inner.add(&bytes.inner, vm), + bytes @ PyBytes => Ok(vm.ctx.new_bytes(self.inner.add(&bytes.inner, vm))), _ => Ok(vm.ctx.not_implemented())) } From 9b763072fe78d8b2dd9fcdeb31a3d1fafcbcc367 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 16:23:56 +0200 Subject: [PATCH 154/223] add upper lower --- tests/snippets/bytes.py | 6 ++++++ vm/src/obj/objbyteinner.rs | 8 ++++++++ vm/src/obj/objbytes.rs | 17 +++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index ceb52921c..3ccb2fc23 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -112,3 +112,9 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() + +# upper lower +l = bytes(b"lower") +b = bytes(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index bcb1f8207..4771a91bd 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -297,6 +297,14 @@ impl PyByteInner { Ok(vm.new_bool(true)) } + + pub fn lower(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_lowercase() + } + + pub fn upper(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_uppercase() + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0b257cb77..0be821500 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -158,34 +158,51 @@ impl PyBytesRef { fn isalnum(self, vm: &VirtualMachine) -> PyResult { self.inner.isalnum(vm) } + #[pymethod(name = "isalpha")] fn isalpha(self, vm: &VirtualMachine) -> PyResult { self.inner.isalpha(vm) } + #[pymethod(name = "isascii")] fn isascii(self, vm: &VirtualMachine) -> PyResult { self.inner.isascii(vm) } + #[pymethod(name = "isdigit")] fn isdigit(self, vm: &VirtualMachine) -> PyResult { self.inner.isdigit(vm) } + #[pymethod(name = "islower")] fn islower(self, vm: &VirtualMachine) -> PyResult { self.inner.islower(vm) } + #[pymethod(name = "isspace")] fn isspace(self, vm: &VirtualMachine) -> PyResult { self.inner.isspace(vm) } + #[pymethod(name = "isupper")] fn isupper(self, vm: &VirtualMachine) -> PyResult { self.inner.isupper(vm) } + #[pymethod(name = "istitle")] fn istitle(self, vm: &VirtualMachine) -> PyResult { self.inner.istitle(vm) } + + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.upper(vm))) + } } #[derive(Debug)] From 5cc83a35aa6d9430a8f0d7c793d6cbbad17dfd66 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Tue, 9 Apr 2019 17:08:38 +0200 Subject: [PATCH 155/223] hex --- tests/snippets/bytes.py | 3 ++- vm/src/obj/objbyteinner.rs | 9 +++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 3ccb2fc23..bf7906bc1 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -113,8 +113,9 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower hex l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() +assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 4771a91bd..6994a58ac 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -305,6 +305,15 @@ impl PyByteInner { pub fn upper(&self, _vm: &VirtualMachine) -> Vec { self.elements.to_ascii_uppercase() } + + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { + let bla = self + .elements + .iter() + .map(|x| format!("{:02x}", x)) + .collect::(); + Ok(vm.ctx.new_str(bla)) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 0be821500..5859b10b4 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -203,6 +203,11 @@ impl PyBytesRef { fn upper(self, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_bytes(self.inner.upper(vm))) } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.hex(vm) + } } #[derive(Debug)] From cbdab226d995fff81892af6eddd34b9e74f8cf48 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Tue, 9 Apr 2019 22:07:21 -0500 Subject: [PATCH 156/223] Change #[pymethod(property)] to #[pyproperty] --- derive/src/pyclass.rs | 246 ++++++++++++++++++++++++++++-------------- 1 file changed, 165 insertions(+), 81 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index e533e8009..710747291 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,91 +1,138 @@ use super::rustpython_path_attr; -use proc_macro2::{Span, TokenStream as TokenStream2}; +use proc_macro2::TokenStream as TokenStream2; use quote::quote; +use std::collections::HashMap; use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; -enum MethodKind { +#[derive(PartialEq, Clone, Copy)] +enum ClassItemKind { Method, - Property, + PropertyGetter, + PropertySetter, } -impl MethodKind { - fn to_ctx_constructor_fn(&self) -> Ident { - let f = match self { - MethodKind::Method => "new_rustfunc", - MethodKind::Property => "new_property", - }; - Ident::new(f, Span::call_site()) +struct ClassItem { + item_name: Ident, + py_name: String, + kind: ClassItemKind, +} + +fn meta_to_vec(meta: Meta) -> Result, Meta> { + match meta { + Meta::Word(_) => Ok(Vec::new()), + Meta::List(list) => Ok(list.nested.into_iter().collect()), + Meta::NameValue(_) => Err(meta), } } -struct Method { - fn_name: Ident, - py_name: String, - kind: MethodKind, -} - -impl Method { - fn from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { - let mut py_name = None; - let mut kind = MethodKind::Method; - let mut pymethod_to_remove = Vec::new(); - let metas = attrs +impl ClassItem { + fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + let mut item = None; + let mut attr_idx = None; + for (i, meta) in attrs .iter() + .filter_map(|attr| attr.parse_meta().ok()) .enumerate() - .filter_map(|(i, attr)| { - if attr.path.is_ident("pymethod") { - let meta = attr.parse_meta().expect("Invalid attribute"); - // remove #[pymethod] because there's no actual proc macro - // implementation for it - pymethod_to_remove.push(i); - match meta { - Meta::List(list) => Some(list), - Meta::Word(_) => None, - Meta::NameValue(_) => panic!( - "#[pymethod = ...] attribute on a method should be a list, like \ - #[pymethod(...)]" - ), - } - } else { - None + { + let name = meta.name(); + if name == "pymethod" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") } - }) - .flat_map(|attr| attr.nested); - for meta in metas { - if let NestedMeta::Meta(meta) = meta { - match meta { - Meta::NameValue(name_value) => { - if name_value.ident == "name" { - if let Lit::Str(s) = &name_value.lit { - py_name = Some(s.value()); - } else { - panic!("#[pymethod(name = ...)] must be a string"); + let nesteds = meta_to_vec(meta) + .expect("#[pymethod = ...] must be a list, e.g. #[pymethod(...)]"); + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pymethod(name = ...)] must be a string"); + } } } + _ => {} } - Meta::Word(ident) => { - if ident == "property" { - kind = MethodKind::Property - } - } - _ => {} } + item = Some(ClassItem { + item_name: sig.ident.clone(), + py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), + kind: ClassItemKind::Method, + }); + attr_idx = Some(i); + } else if name == "pyproperty" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") + } + let nesteds = meta_to_vec(meta) + .expect("#[pyproperty = ...] must be a list, e.g. #[pyproperty(...)]"); + let mut setter = false; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pyproperty(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => { + if ident == "setter" { + setter = true; + } + } + _ => {} + } + } + let kind = if setter { + ClassItemKind::PropertySetter + } else { + ClassItemKind::PropertyGetter + }; + let py_name = py_name.unwrap_or_else(|| { + let item_name = sig.ident.to_string(); + if item_name.starts_with("set_") { + let name = &item_name["set_".len()..]; + if name.is_empty() { + panic!( + "A #[pyproperty(setter)] fn with a set_* name have something \ + after \"set_\"" + ) + } else { + name.to_string() + } + } else { + panic!( + "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ + fn name along the lines of \"set_*\"" + ) + } + }); + item = Some(ClassItem { + py_name, + item_name: sig.ident.clone(), + kind, + }); + attr_idx = Some(i); } } - // if there are no #[pymethods]s, then it's not a method, so exclude it from - // the final result - if pymethod_to_remove.is_empty() { - return None; + if let Some(attr_idx) = attr_idx { + attrs.remove(attr_idx); } - for i in pymethod_to_remove { - attrs.remove(i); - } - let py_name = py_name.unwrap_or_else(|| sig.ident.to_string()); - Some(Method { - fn_name: sig.ident.clone(), - py_name, - kind, - }) + item } } @@ -98,30 +145,66 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let rp_path = rustpython_path_attr(&attr); - let methods = imp + let items = imp .items .iter_mut() .filter_map(|item| { if let ImplItem::Method(meth) = item { - Method::from_syn(&mut meth.attrs, &meth.sig) + ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig) } else { None } }) .collect::>(); let ty = &imp.self_ty; - let methods = methods.iter().map( - |Method { - py_name, - fn_name, - kind, - }| { - let constructor_fn = kind.to_ctx_constructor_fn(); - quote! { - class.set_str_attr(#py_name, ctx.#constructor_fn(Self::#fn_name)); + let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); + for item in items.iter() { + match item.kind { + ClassItemKind::PropertyGetter => { + let (ref mut getter, _) = properties.entry(&item.py_name).or_default(); + if getter.is_some() { + panic!("Multiple property getters with name {:?}", &item.py_name) + } + *getter = Some(&item.item_name); } - }, - ); + ClassItemKind::PropertySetter => { + let (_, ref mut setter) = properties.entry(&item.py_name).or_default(); + if setter.is_some() { + panic!("Multiple property getters with name {:?}", &item.py_name) + } + *setter = Some(&item.item_name); + } + ClassItemKind::Method => {} + } + } + let methods = items.iter().filter_map(|item| { + if let ClassItemKind::Method = item.kind { + let ClassItem { + py_name, item_name, .. + } = item; + Some(quote! { + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); + }) + } else { + None + } + }); + let properties = properties.iter().map(|(name, prop)| { + let getter = match prop.0 { + Some(getter) => getter, + None => panic!("Property {:?} is missing a getter", name), + }; + let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); + quote! { + class.set_str_attr( + #name, + #rp_path::obj::objproperty::PropertyBuilder + .add_getter(Self::#getter) + #add_setter + .create(), + ); + } + }); quote! { #imp @@ -131,6 +214,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { class: &#rp_path::obj::objtype::PyClassRef, ) { #(#methods)* + #(#properties)* } } } From 977f56ade1bc9f9aca692ff14dcb352b42508e10 Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 10 Apr 2019 10:08:10 +0200 Subject: [PATCH 157/223] add bytes.fromhex --- tests/snippets/bytes.py | 14 +++++++++++++- vm/src/obj/objbyteinner.rs | 33 +++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 22 +++++++++++++++++++--- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index bf7906bc1..acd8d9e50 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -113,9 +113,21 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower hex +# upper lower l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() + +# hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +try: + bytes.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytes.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 6994a58ac..7504f2b74 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -314,6 +314,39 @@ impl PyByteInner { .collect::(); Ok(vm.ctx.new_str(bla)) } + + pub fn fromhex(string: String, vm: &VirtualMachine) -> Result, PyObjectRef> { + // first check for invalid character + for (i, c) in string.char_indices() { + if !c.is_digit(16) && !c.is_whitespace() { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + i + ))); + } + } + + // strip white spaces + let stripped = string.split_whitespace().collect::(); + + // Hex is evaluated on 2 digits + if stripped.len() % 2 != 0 { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + stripped.len() - 1 + ))); + } + + // parse even string + Ok(stripped + .chars() + .collect::>() + .chunks(2) + .map(|x| x.to_vec().iter().collect::()) + .map(|x| u8::from_str_radix(&x, 16)) + .map(|x| x.unwrap()) + .collect::>()) + } } // TODO diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 5859b10b4..2c8edca6a 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,4 +1,5 @@ use crate::obj::objint::PyInt; +use crate::obj::objstr::PyString; use crate::vm::VirtualMachine; use core::cell::Cell; use std::ops::Deref; @@ -55,11 +56,15 @@ pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a pub fn init(context: &PyContext) { PyBytesRef::extend_class(context, &context.bytes_type); + let bytes_type = &context.bytes_type; + extend_class!(context, bytes_type, { +"fromhex" => context.new_rustfunc(PyBytesRef::fromhex), +}); let bytesiterator_type = &context.bytesiterator_type; extend_class!(context, bytesiterator_type, { - "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), - "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), - }); +"__next__" => context.new_rustfunc(PyBytesIteratorRef::next), +"__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), +}); } #[pyimpl(__inside_vm)] @@ -208,6 +213,17 @@ impl PyBytesRef { fn hex(self, vm: &VirtualMachine) -> PyResult { self.inner.hex(vm) } + + // #[pymethod(name = "fromhex")] + fn fromhex(string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(string, + s @ PyString => { + match PyByteInner::fromhex(s.to_string(), vm) { + Ok(x) => Ok(vm.ctx.new_bytes(x)), + Err(y) => Err(y)}}, + obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) + ) + } } #[derive(Debug)] From 9a92159413a9ef6de556e45e47bdea14fbe43dd0 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:26:03 +0100 Subject: [PATCH 158/223] Remove unneeded Cell on DictIter. --- vm/src/obj/objdict.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index b0d99d4e7..35925ff35 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -255,15 +255,12 @@ impl IntoIterator for &PyDictRef { pub struct DictIterator { dict: PyDictRef, - position: Cell, + position: usize, } impl DictIterator { pub fn new(dict: PyDictRef) -> DictIterator { - DictIterator { - dict, - position: Cell::new(0), - } + DictIterator { dict, position: 0 } } } @@ -271,14 +268,13 @@ impl Iterator for DictIterator { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { - self.dict - .entries - .borrow() - .next_entry(self.position.get()) - .map(|(new_position, key, value)| { - self.position.set(new_position); - (key.clone(), value.clone()) - }) + match self.dict.entries.borrow().next_entry(self.position) { + Some((new_position, key, value)) => { + self.position = new_position; + Some((key.clone(), value.clone())) + } + None => None, + } } } From 8f840d537610cb05e545a3545feb2011e230a095 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:26:18 +0100 Subject: [PATCH 159/223] Pick code review nits. --- vm/src/obj/objdict.rs | 18 +++++++++--------- vm/src/obj/objobject.rs | 2 +- vm/src/stdlib/json.rs | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 35925ff35..753baf786 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -237,34 +237,34 @@ impl ItemProtocol for PyDictRef { // Implement IntoIterator so that we can easily iterate dictionaries from rust code. impl IntoIterator for PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterator; + type IntoIter = DictIter; fn into_iter(self) -> Self::IntoIter { - DictIterator::new(self) + DictIter::new(self) } } impl IntoIterator for &PyDictRef { type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIterator; + type IntoIter = DictIter; fn into_iter(self) -> Self::IntoIter { - DictIterator::new(self.clone()) + DictIter::new(self.clone()) } } -pub struct DictIterator { +pub struct DictIter { dict: PyDictRef, position: usize, } -impl DictIterator { - pub fn new(dict: PyDictRef) -> DictIterator { - DictIterator { dict, position: 0 } +impl DictIter { + pub fn new(dict: PyDictRef) -> DictIter { + DictIter { dict, position: 0 } } } -impl Iterator for DictIterator { +impl Iterator for DictIter { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index f8e86d0a1..2be4ff4f4 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -237,7 +237,7 @@ pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { // Get instance attributes: if let Some(dict) = &obj.dict { - for (key, value) in dict.into_iter() { + for (key, value) in dict { attributes.insert(key.to_string(), value.clone()); } } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index 3fa30e16b..dceec8a7d 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -64,7 +64,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); - let pairs: Vec<(PyObjectRef, PyObjectRef)> = dict.into_iter().collect(); + let pairs: Vec<_> = dict.into_iter().collect(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; From e04275b1999d42749c92d029a109cfc2aa6699c7 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 09:48:53 +0100 Subject: [PATCH 160/223] Missed get_key_values_pairs removal in wasm. --- wasm/lib/src/browser_module.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index d302ec99a..478ebb9c5 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -83,7 +83,7 @@ fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyRe if let Some(headers) = headers { let h = request.headers(); - for (key, value) in headers.get_key_value_pairs() { + for (key, value) in headers { let key = vm.to_str(&key)?; let value = vm.to_str(&value)?; h.set(key.as_str(), value.as_str()) From 751c3d52e2a4deccd0443b84c0c70329a2b76f02 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Wed, 10 Apr 2019 13:24:31 +0200 Subject: [PATCH 161/223] use vec! for int arg in bytes new --- tests/snippets/bytes.py | 6 +++++- vm/src/obj/objbyteinner.rs | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index acd8d9e50..0a3521df3 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -4,8 +4,8 @@ from testutils import assertRaises assert bytes([1, 2, 3]) assert bytes((1, 2, 3)) assert bytes(range(4)) -assert b"bla" assert bytes(3) +assert b"bla" assert bytes("bla", "utf8") try: bytes("bla") @@ -14,6 +14,9 @@ except TypeError: else: assert False + + + a = b"abcd" b = b"ab" c = b"abcd" @@ -49,6 +52,7 @@ hash(a) == hash(b"abcd") # iter [i for i in b"abcd"] == ["a", "b", "c", "d"] +assert list(bytes(3)) == [0,0,0] # add assert a + b == b"abcdab" diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 7504f2b74..fac093762 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -61,11 +61,7 @@ impl PyByteInner { match_class!(ival.clone(), i @ PyInt => { let size = objint::get_value(&i.into_object()).to_usize().unwrap(); - let mut res: Vec = Vec::with_capacity(size); - for _ in 0..size { - res.push(0) - } - Ok(res)}, + Ok(vec![0; size])}, _l @ PyString=> {return Err(vm.new_type_error(format!( "string argument without an encoding" )));}, From 54c7335f1c32c0215b81f856cac26218a1223f5e Mon Sep 17 00:00:00 2001 From: Jimmy Girardet Date: Wed, 10 Apr 2019 18:05:28 +0200 Subject: [PATCH 162/223] SyntaxError for non ascii char --- parser/src/lexer.rs | 8 ++++++-- tests/snippets/bytes.py | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index edf21c973..561ea4d77 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -540,8 +540,12 @@ where let end_pos = self.get_pos(); let tok = if is_bytes { - Tok::Bytes { - value: string_content.as_bytes().to_vec(), + if string_content.is_ascii() { + Tok::Bytes { + value: string_content.as_bytes().to_vec(), + } + } else { + return Err(LexicalError::StringError); } } else { Tok::String { diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 0a3521df3..816eae495 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -11,10 +11,6 @@ try: bytes("bla") except TypeError: assert True -else: - assert False - - a = b"abcd" From 9d25a216de4d8240f72eb6f347228dc9b31269c1 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Wed, 10 Apr 2019 21:39:16 +0200 Subject: [PATCH 163/223] fix some tests, run clippy --- tests/snippets/bytes.py | 12 ++++-------- vm/src/obj/objbyteinner.rs | 30 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 816eae495..0ffbc29f8 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -7,10 +7,8 @@ assert bytes(range(4)) assert bytes(3) assert b"bla" assert bytes("bla", "utf8") -try: +with assertRaises(TypeError): bytes("bla") -except TypeError: - assert True a = b"abcd" @@ -48,7 +46,7 @@ hash(a) == hash(b"abcd") # iter [i for i in b"abcd"] == ["a", "b", "c", "d"] -assert list(bytes(3)) == [0,0,0] +assert list(bytes(3)) == [0, 0, 0] # add assert a + b == b"abcdab" @@ -62,10 +60,8 @@ assert b"d" in b"abcd" assert b"dc" not in b"abcd" assert 97 in b"abcd" assert 150 not in b"abcd" -try: +with assertRaises(ValueError): 350 in b"abcd" -except ValueError: - pass # getitem @@ -113,7 +109,7 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index fac093762..532504b02 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -31,8 +31,8 @@ impl PyByteInner { if let OptionalArg::Present(eval) = val_option { if let Ok(input) = eval.downcast::() { if let Ok(encoding) = enc.clone().downcast::() { - if encoding.value.to_lowercase() == "utf8".to_string() - || encoding.value.to_lowercase() == "utf-8".to_string() + if &encoding.value.to_lowercase() == "utf8" + || &encoding.value.to_lowercase() == "utf-8" // TODO: different encoding { return Ok(PyByteInner { @@ -62,12 +62,10 @@ impl PyByteInner { i @ PyInt => { let size = objint::get_value(&i.into_object()).to_usize().unwrap(); Ok(vec![0; size])}, - _l @ PyString=> {return Err(vm.new_type_error(format!( - "string argument without an encoding" - )));}, + _l @ PyString=> {return Err(vm.new_type_error("string argument without an encoding".to_string()));}, obj => { - let elements = vm.extract_elements(&obj).or_else(|_| {return Err(vm.new_type_error(format!( - "cannot convert {} object to bytes", obj.class().name)));}); + let elements = vm.extract_elements(&obj).or_else(|_| {Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)))}); let mut data_bytes = vec![]; for elem in elements.unwrap(){ @@ -111,6 +109,10 @@ impl PyByteInner { self.elements.len() } + pub fn is_empty(&self) -> bool { + self.elements.len() == 0 + } + pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { if self.elements == other.elements { Ok(vm.new_bool(true)) @@ -169,10 +171,11 @@ impl PyByteInner { pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { for (n, i) in self.elements.iter().enumerate() { - if n + other.len() <= self.len() && *i == other.elements[0] { - if &self.elements[n..n + other.len()] == other.elements.as_slice() { - return Ok(vm.new_bool(true)); - } + if n + other.len() <= self.len() + && *i == other.elements[0] + && &self.elements[n..n + other.len()] == other.elements.as_slice() + { + return Ok(vm.new_bool(true)); } } Ok(vm.new_bool(false)) @@ -186,7 +189,7 @@ impl PyByteInner { Ok(vm.new_bool(false)) } } else { - Err(vm.new_value_error("byte mu st be in range(0, 256)".to_string())) + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) } } @@ -344,6 +347,3 @@ impl PyByteInner { .collect::>()) } } - -// TODO -// fix b"é" not allowed should be bytes("é", "utf8") From 47bd9680f9c473129d331b22f549bc40e18a0a56 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:41:25 -0500 Subject: [PATCH 164/223] Change ClassItem to enum --- derive/src/pyclass.rs | 64 ++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 710747291..3415da181 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -4,17 +4,16 @@ use quote::quote; use std::collections::HashMap; use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; -#[derive(PartialEq, Clone, Copy)] -enum ClassItemKind { - Method, - PropertyGetter, - PropertySetter, -} - -struct ClassItem { - item_name: Ident, - py_name: String, - kind: ClassItemKind, +enum ClassItem { + Method { + item_name: Ident, + py_name: String, + }, + Property { + item_name: Ident, + py_name: String, + setter: bool, + }, } fn meta_to_vec(meta: Meta) -> Result, Meta> { @@ -60,10 +59,9 @@ impl ClassItem { _ => {} } } - item = Some(ClassItem { + item = Some(ClassItem::Method { item_name: sig.ident.clone(), py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), - kind: ClassItemKind::Method, }); attr_idx = Some(i); } else if name == "pyproperty" { @@ -97,11 +95,6 @@ impl ClassItem { _ => {} } } - let kind = if setter { - ClassItemKind::PropertySetter - } else { - ClassItemKind::PropertyGetter - }; let py_name = py_name.unwrap_or_else(|| { let item_name = sig.ident.to_string(); if item_name.starts_with("set_") { @@ -121,10 +114,10 @@ impl ClassItem { ) } }); - item = Some(ClassItem { + item = Some(ClassItem::Property { py_name, item_name: sig.ident.clone(), - kind, + setter, }); attr_idx = Some(i); } @@ -159,29 +152,24 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { let ty = &imp.self_ty; let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); for item in items.iter() { - match item.kind { - ClassItemKind::PropertyGetter => { - let (ref mut getter, _) = properties.entry(&item.py_name).or_default(); - if getter.is_some() { - panic!("Multiple property getters with name {:?}", &item.py_name) + match item { + ClassItem::Property { + item_name, + py_name, + setter, + } => { + let entry = properties.entry(py_name).or_default(); + let func = if *setter { &mut entry.1 } else { &mut entry.0 }; + if func.is_some() { + panic!("Multiple property accessors with name {:?}", py_name) } - *getter = Some(&item.item_name); + *func = Some(item_name); } - ClassItemKind::PropertySetter => { - let (_, ref mut setter) = properties.entry(&item.py_name).or_default(); - if setter.is_some() { - panic!("Multiple property getters with name {:?}", &item.py_name) - } - *setter = Some(&item.item_name); - } - ClassItemKind::Method => {} + _ => {} } } let methods = items.iter().filter_map(|item| { - if let ClassItemKind::Method = item.kind { - let ClassItem { - py_name, item_name, .. - } = item; + if let ClassItem::Method { item_name, py_name } = item { Some(quote! { class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); }) From d099165e3bf2b5e53bf41d087482bb05dd2800f6 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:41:45 -0500 Subject: [PATCH 165/223] Rename item_name -> item_ident --- derive/src/pyclass.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 3415da181..892b298b8 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -6,11 +6,11 @@ use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, enum ClassItem { Method { - item_name: Ident, + item_ident: Ident, py_name: String, }, Property { - item_name: Ident, + item_ident: Ident, py_name: String, setter: bool, }, @@ -60,7 +60,7 @@ impl ClassItem { } } item = Some(ClassItem::Method { - item_name: sig.ident.clone(), + item_ident: sig.ident.clone(), py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), }); attr_idx = Some(i); @@ -96,9 +96,9 @@ impl ClassItem { } } let py_name = py_name.unwrap_or_else(|| { - let item_name = sig.ident.to_string(); - if item_name.starts_with("set_") { - let name = &item_name["set_".len()..]; + let item_ident = sig.ident.to_string(); + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; if name.is_empty() { panic!( "A #[pyproperty(setter)] fn with a set_* name have something \ @@ -116,7 +116,7 @@ impl ClassItem { }); item = Some(ClassItem::Property { py_name, - item_name: sig.ident.clone(), + item_ident: sig.ident.clone(), setter, }); attr_idx = Some(i); @@ -154,7 +154,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { for item in items.iter() { match item { ClassItem::Property { - item_name, + item_ident, py_name, setter, } => { @@ -163,15 +163,19 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { if func.is_some() { panic!("Multiple property accessors with name {:?}", py_name) } - *func = Some(item_name); + *func = Some(item_ident); } _ => {} } } let methods = items.iter().filter_map(|item| { - if let ClassItem::Method { item_name, py_name } = item { + if let ClassItem::Method { + item_ident, + py_name, + } = item + { Some(quote! { - class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_name)); + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident)); }) } else { None From 2bdaa02700f88691826ad7edd666d632e1d1d26d Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:45:07 -0500 Subject: [PATCH 166/223] Improve error messages --- derive/src/pyclass.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 892b298b8..2387b4d79 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -28,6 +28,7 @@ impl ClassItem { fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { let mut item = None; let mut attr_idx = None; + // TODO: better error handling throughout this for (i, meta) in attrs .iter() .filter_map(|attr| attr.parse_meta().ok()) @@ -38,13 +39,15 @@ impl ClassItem { if item.is_some() { panic!("You can only have one #[py*] attribute on an impl item") } - let nesteds = meta_to_vec(meta) - .expect("#[pymethod = ...] must be a list, e.g. #[pymethod(...)]"); + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); let mut py_name = None; for meta in nesteds { let meta = match meta { NestedMeta::Meta(meta) => meta, - NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + NestedMeta::Literal(_) => continue, }; match meta { Meta::NameValue(name_value) => { @@ -68,14 +71,16 @@ impl ClassItem { if item.is_some() { panic!("You can only have one #[py*] attribute on an impl item") } - let nesteds = meta_to_vec(meta) - .expect("#[pyproperty = ...] must be a list, e.g. #[pyproperty(...)]"); + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); let mut setter = false; let mut py_name = None; for meta in nesteds { let meta = match meta { NestedMeta::Meta(meta) => meta, - NestedMeta::Literal(_) => panic!("Expected a meta, found a literal"), + NestedMeta::Literal(_) => continue, }; match meta { Meta::NameValue(name_value) => { From c5f4eab1424e4b88493a98bf904e350c55982ebe Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 17:55:24 -0500 Subject: [PATCH 167/223] Change objint to impl style with properties --- derive/src/pyclass.rs | 28 ++++++++++++++++------------ vm/src/obj/objint.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 2387b4d79..433d5911e 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -102,21 +102,25 @@ impl ClassItem { } let py_name = py_name.unwrap_or_else(|| { let item_ident = sig.ident.to_string(); - if item_ident.starts_with("set_") { - let name = &item_ident["set_".len()..]; - if name.is_empty() { - panic!( - "A #[pyproperty(setter)] fn with a set_* name have something \ - after \"set_\"" - ) + if setter { + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; + if name.is_empty() { + panic!( + "A #[pyproperty(setter)] fn with a set_* name have something \ + after \"set_\"" + ) + } else { + name.to_string() + } } else { - name.to_string() - } - } else { - panic!( + panic!( "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ fn name along the lines of \"set_*\"" ) + } + } else { + item_ident } }); item = Some(ClassItem::Property { @@ -195,7 +199,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { class.set_str_attr( #name, - #rp_path::obj::objproperty::PropertyBuilder + #rp_path::obj::objproperty::PropertyBuilder::new(ctx) .add_getter(Self::#getter) #add_setter .create(), diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index de211668f..789d140d6 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -17,6 +17,7 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; +#[pyclass(__inside_vm)] #[derive(Debug)] pub struct PyInt { value: BigInt, @@ -95,7 +96,9 @@ impl_try_from_object_int!( (u64, to_u64), ); +#[pyimpl(__inside_vm)] impl PyInt { + #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) @@ -104,6 +107,7 @@ impl PyInt { } } + #[pymethod(name = "__ne__")] fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) @@ -112,6 +116,7 @@ impl PyInt { } } + #[pymethod(name = "__lt__")] fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) @@ -120,6 +125,7 @@ impl PyInt { } } + #[pymethod(name = "__le__")] fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) @@ -128,6 +134,7 @@ impl PyInt { } } + #[pymethod(name = "__gt__")] fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) @@ -136,6 +143,7 @@ impl PyInt { } } + #[pymethod(name = "__ge__")] fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) @@ -144,6 +152,7 @@ impl PyInt { } } + #[pymethod(name = "__add__")] fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) @@ -152,6 +161,7 @@ impl PyInt { } } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) @@ -160,6 +170,7 @@ impl PyInt { } } + #[pymethod(name = "__rsub__")] fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) @@ -168,6 +179,7 @@ impl PyInt { } } + #[pymethod(name = "__mul__")] fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) @@ -176,6 +188,7 @@ impl PyInt { } } + #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) @@ -184,6 +197,7 @@ impl PyInt { } } + #[pymethod(name = "__rtruediv__")] fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) @@ -192,6 +206,7 @@ impl PyInt { } } + #[pymethod(name = "__floordiv__")] fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -205,6 +220,7 @@ impl PyInt { } } + #[pymethod(name = "__lshift__")] fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); @@ -224,6 +240,7 @@ impl PyInt { } } + #[pymethod(name = "__rshift__")] fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); @@ -243,6 +260,7 @@ impl PyInt { } } + #[pymethod(name = "__xor__")] fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) @@ -251,6 +269,7 @@ impl PyInt { } } + #[pymethod(name = "__rxor__")] fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) ^ (&self.value)) @@ -259,6 +278,7 @@ impl PyInt { } } + #[pymethod(name = "__or__")] fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) @@ -267,6 +287,7 @@ impl PyInt { } } + #[pymethod(name = "__and__")] fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -276,6 +297,7 @@ impl PyInt { } } + #[pymethod(name = "__pow__")] fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); @@ -288,6 +310,7 @@ impl PyInt { } } + #[pymethod(name = "__mod__")] fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -301,6 +324,7 @@ impl PyInt { } } + #[pymethod(name = "__divmod__")] fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); @@ -317,20 +341,24 @@ impl PyInt { } } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } + #[pymethod(name = "__hash__")] fn hash(&self, _vm: &VirtualMachine) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() } + #[pymethod(name = "__abs__")] fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } + #[pymethod(name = "__round__")] fn round( zelf: PyRef, _precision: OptionalArg, @@ -339,14 +367,17 @@ impl PyInt { zelf } + #[pymethod(name = "__int__")] fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__pos__")] fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__float__")] fn float(&self, vm: &VirtualMachine) -> PyResult { self.value .to_f64() @@ -354,30 +385,37 @@ impl PyInt { .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) } + #[pymethod(name = "__trunc__")] fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__floor__")] fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__ceil__")] fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__index__")] fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pymethod(name = "__invert__")] fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } + #[pymethod(name = "__repr__")] fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } + #[pymethod(name = "__format__")] fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { @@ -386,22 +424,27 @@ impl PyInt { } } + #[pymethod(name = "__bool__")] fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } + #[pymethod] fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } + #[pymethod] fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pyproperty] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf } + #[pyproperty] fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } From dd4539b1a601f33ac77b1fdfdc74b909423dff73 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 18:02:30 -0500 Subject: [PATCH 168/223] Change to use PyClassImpl::extend_class --- vm/src/obj/objint.rs | 80 ++++++++++---------------------------------- 1 file changed, 18 insertions(+), 62 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 789d140d6..a51908209 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -8,7 +8,8 @@ use num_traits::{Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::OptionalArg; use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; @@ -17,6 +18,20 @@ use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; +/// int(x=0) -> integer +/// int(x, base=10) -> integer +/// +/// Convert a number or string to an integer, or return 0 if no arguments +/// are given. If x is a number, return x.__int__(). For floating point +/// numbers, this truncates towards zero. +/// +/// If x is not a number or if base is given, then x must be a string, +/// bytes, or bytearray instance representing an integer literal in the +/// given base. The literal can be preceded by '+' or '-' and be surrounded +/// by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. +/// Base 0 means to interpret the base from the string as an integer literal. +/// >>> int('0b100', base=0) +/// 4 #[pyclass(__inside_vm)] #[derive(Debug)] pub struct PyInt { @@ -543,68 +558,9 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { } } -#[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let int_doc = "int(x=0) -> integer -int(x, base=10) -> integer - -Convert a number or string to an integer, or return 0 if no arguments -are given. If x is a number, return x.__int__(). For floating point -numbers, this truncates towards zero. - -If x is not a number or if base is given, then x must be a string, -bytes, or bytearray instance representing an integer literal in the -given base. The literal can be preceded by '+' or '-' and be surrounded -by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. -Base 0 means to interpret the base from the string as an integer literal. ->>> int('0b100', base=0) -4"; - let int_type = &context.int_type; - extend_class!(context, int_type, { - "__doc__" => context.new_str(int_doc.to_string()), - "__eq__" => context.new_rustfunc(PyInt::eq), - "__ne__" => context.new_rustfunc(PyInt::ne), - "__lt__" => context.new_rustfunc(PyInt::lt), - "__le__" => context.new_rustfunc(PyInt::le), - "__gt__" => context.new_rustfunc(PyInt::gt), - "__ge__" => context.new_rustfunc(PyInt::ge), - "__abs__" => context.new_rustfunc(PyInt::abs), - "__add__" => context.new_rustfunc(PyInt::add), - "__radd__" => context.new_rustfunc(PyInt::add), - "__and__" => context.new_rustfunc(PyInt::and), - "__divmod__" => context.new_rustfunc(PyInt::divmod), - "__float__" => context.new_rustfunc(PyInt::float), - "__round__" => context.new_rustfunc(PyInt::round), - "__ceil__" => context.new_rustfunc(PyInt::ceil), - "__floor__" => context.new_rustfunc(PyInt::floor), - "__index__" => context.new_rustfunc(PyInt::index), - "__trunc__" => context.new_rustfunc(PyInt::trunc), - "__int__" => context.new_rustfunc(PyInt::int), - "__floordiv__" => context.new_rustfunc(PyInt::floordiv), - "__hash__" => context.new_rustfunc(PyInt::hash), - "__lshift__" => context.new_rustfunc(PyInt::lshift), - "__rshift__" => context.new_rustfunc(PyInt::rshift), + PyInt::extend_class(context, &context.int_type); + extend_class!(context, &context.int_type, { "__new__" => context.new_rustfunc(int_new), - "__mod__" => context.new_rustfunc(PyInt::mod_), - "__mul__" => context.new_rustfunc(PyInt::mul), - "__rmul__" => context.new_rustfunc(PyInt::mul), - "__or__" => context.new_rustfunc(PyInt::or), - "__neg__" => context.new_rustfunc(PyInt::neg), - "__pos__" => context.new_rustfunc(PyInt::pos), - "__pow__" => context.new_rustfunc(PyInt::pow), - "__repr__" => context.new_rustfunc(PyInt::repr), - "__sub__" => context.new_rustfunc(PyInt::sub), - "__rsub__" => context.new_rustfunc(PyInt::rsub), - "__format__" => context.new_rustfunc(PyInt::format), - "__truediv__" => context.new_rustfunc(PyInt::truediv), - "__rtruediv__" => context.new_rustfunc(PyInt::rtruediv), - "__xor__" => context.new_rustfunc(PyInt::xor), - "__rxor__" => context.new_rustfunc(PyInt::rxor), - "__bool__" => context.new_rustfunc(PyInt::bool), - "__invert__" => context.new_rustfunc(PyInt::invert), - "bit_length" => context.new_rustfunc(PyInt::bit_length), - "conjugate" => context.new_rustfunc(PyInt::conjugate), - "real" => context.new_property(PyInt::real), - "imag" => context.new_property(PyInt::imag) }); } From 6a190e39f348db73c76fc875fcdedb9211659514 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 10 Apr 2019 20:41:06 -0500 Subject: [PATCH 169/223] Fix __radd__ and __rmul__ --- vm/src/obj/objint.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index a51908209..5be50f6c4 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -176,6 +176,11 @@ impl PyInt { } } + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.add(other, vm) + } + #[pymethod(name = "__sub__")] fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { @@ -203,6 +208,11 @@ impl PyInt { } } + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.mul(other, vm) + } + #[pymethod(name = "__truediv__")] fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { From bd5772d9142f8bbd93e65a6ca4ec4cb214bbc4ac Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Tue, 9 Apr 2019 12:03:02 +0100 Subject: [PATCH 170/223] Implement dict.__eq__ --- tests/snippets/dict.py | 26 ++++++++++++++------------ vm/src/obj/objdict.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 45c6c3b43..6507efb19 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,21 +1,23 @@ from testutils import assertRaises -def dict_eq(d1, d2): - return (all(k in d2 and d1[k] == d2[k] for k in d1) - and all(k in d1 and d1[k] == d2[k] for k in d2)) +assert dict(a=2, b=3) == {'a': 2, 'b': 3} +assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} +assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} - -assert dict_eq(dict(a=2, b=3), {'a': 2, 'b': 3}) -assert dict_eq(dict({'a': 2, 'b': 3}, b=4), {'a': 2, 'b': 4}) -assert dict_eq(dict([('a', 2), ('b', 3)]), {'a': 2, 'b': 3}) +assert {} == {} +assert not {'a': 2} == {} +assert not {} == {'a': 2} +assert not {'b': 2} == {'a': 2} +assert not {'a': 4} == {'a': 2} +assert {'a': 2} == {'a': 2} a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) c['d'] = 3 c['a']['g'] = 2 -assert dict_eq(a, {'g': 2}) -assert dict_eq(b, {'a': a, 'd': 9}) +assert a == {'g': 2} +assert b == {'a': a, 'd': 9} a.clear() assert len(a) == 0 @@ -142,10 +144,10 @@ assert list(x) == ['a', 'b'] y = x.copy() x['c'] = 12 -assert dict_eq(y, {'a': 2, 'b': 10}) +assert y == {'a': 2, 'b': 10} y.update({'c': 19, "d": -1, 'b': 12}) -assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} y.update(y) -assert dict_eq(y, {'a': 2, 'b': 12, 'c': 19, 'd': -1}) # hasn't changed +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 4a503fdcf..8e34ca738 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -7,6 +7,7 @@ use crate::pyobject::{ }; use crate::vm::{ReprGuard, VirtualMachine}; +use super::objbool; use super::objiter; use super::objstr; use crate::dictdatatype; @@ -96,6 +97,35 @@ impl PyDictRef { !self.entries.borrow().is_empty() } + fn inner_eq(self, other: &PyDict, vm: &VirtualMachine) -> PyResult { + if other.entries.borrow().len() != self.entries.borrow().len() { + return Ok(false); + } + for (k, v1) in self { + match other.entries.borrow().get(vm, &k)? { + Some(v2) => { + let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; + if !value { + return Ok(false); + } + } + None => { + return Ok(false); + } + } + } + return Ok(true); + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + let eq = self.inner_eq(other, vm)?; + Ok(vm.ctx.new_bool(eq)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + fn len(self, _vm: &VirtualMachine) -> usize { self.entries.borrow().len() } @@ -387,6 +417,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__eq__" => context.new_rustfunc(PyDictRef::eq), "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), "__new__" => context.new_rustfunc(PyDictRef::new), From b70f989952f99b535750e34b904f0ea88d32ab7d Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 08:39:03 +0100 Subject: [PATCH 171/223] dict equality - objects always equal themselves. --- tests/snippets/dict.py | 3 +++ vm/src/obj/objdict.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 6507efb19..4c35af2f4 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -11,6 +11,9 @@ assert not {'b': 2} == {'a': 2} assert not {'a': 4} == {'a': 2} assert {'a': 2} == {'a': 2} +nan = float('nan') +assert {'a': nan} == {'a': nan} + a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 8e34ca738..be85c7b2f 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -3,7 +3,8 @@ use std::fmt; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, + PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; @@ -104,6 +105,9 @@ impl PyDictRef { for (k, v1) in self { match other.entries.borrow().get(vm, &k)? { Some(v2) => { + if v1.is(&v2) { + continue; + } let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; if !value { return Ok(false); From 84eff4a7ac8a0010f6ee151637b1f5a90da4d566 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 10 Apr 2019 20:56:40 +1200 Subject: [PATCH 172/223] Add __name__ and __module__ to functions --- tests/snippets/class.py | 8 ++++++++ tests/snippets/function.py | 17 +++++++++++++++++ vm/src/frame.rs | 10 +++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 9dbce566d..0acd38fce 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -1,3 +1,6 @@ +__name__ = "class" + + class Foo: def __init__(self, x): assert x == 5 @@ -14,6 +17,11 @@ foo = Foo(5) assert foo.y == Foo.y assert foo.x == 5 assert foo.square() == 25 +assert Foo.__name__ == "Foo" +assert Foo.__qualname__ == "Foo" +assert Foo.__module__ == "class" +assert Foo.square.__name__ == "square" +assert Foo.square.__qualname__ == "Foo.square" class Bar: diff --git a/tests/snippets/function.py b/tests/snippets/function.py index e4d0ab544..b499c6776 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,9 +1,16 @@ + +__name__ = "function" + + def foo(): """test""" return 42 assert foo() == 42 assert foo.__doc__ == "test" +assert foo.__name__ == "foo" +assert foo.__qualname__ == "foo" +assert foo.__module__ == "function" def my_func(a,): return a+2 @@ -55,3 +62,13 @@ def f5(): assert f5.__doc__ == 'abcw00t', f5.__doc__ + +def f6(): + def nested(): + pass + + assert nested.__name__ == "nested" + assert nested.__qualname__ == "f6..nested" + + +f6() diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..d614691c9 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -566,7 +566,7 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let _qualified_name = self.pop_value(); + let qualified_name = self.pop_value(); let code_obj = self .pop_value() .downcast() @@ -606,6 +606,14 @@ impl Frame { .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); + vm.set_attr(&obj, "__name__", qualified_name.clone())?; + vm.set_attr(&obj, "__qualname__", qualified_name)?; + let module = self + .scope + .globals + .get_item_option("__name__", vm)? + .unwrap_or_else(|| vm.get_none()); + vm.set_attr(&obj, "__module__", module)?; vm.set_attr(&obj, "__annotations__", annotations)?; self.push_value(obj); From 2164cb5e65946c1eda326fdc6a5b8ba7817e917a Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 11 Apr 2019 20:11:26 +1200 Subject: [PATCH 173/223] Produce correct value for __qualname__ and add attributes to class. --- tests/snippets/class.py | 2 +- vm/src/builtins.rs | 28 +++++++++++++++++++--------- vm/src/compile.rs | 31 +++++++++++++++++++++++++++++-- vm/src/frame.rs | 14 ++++++++------ vm/src/function.rs | 14 ++++++++++++++ 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 0acd38fce..ded02dbfa 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -22,7 +22,7 @@ assert Foo.__qualname__ == "Foo" assert Foo.__module__ == "class" assert Foo.square.__name__ == "square" assert Foo.square.__qualname__ == "Foo.square" - +assert Foo.square.__module__ == "class" class Bar: """ W00t """ diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 27e024aaf..6a94b8362 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -18,7 +18,7 @@ use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; -use crate::function::{Args, OptionalArg, PyFuncArgs}; +use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -783,11 +783,17 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { }); } -pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { - let function = args.shift(); - let name_arg = args.shift(); - let bases = args.args.clone(); - let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { +pub fn builtin_build_class_( + function: PyObjectRef, + qualified_name: PyStringRef, + bases: Args, + mut kwargs: KwArgs, + vm: &VirtualMachine, +) -> PyResult { + let name = qualified_name.value.split('.').next_back().unwrap(); + let name_obj = vm.new_str(name.to_string()); + + let mut metaclass = if let Some(metaclass) = kwargs.pop_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? } else { vm.get_type() @@ -801,21 +807,25 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu } } - let bases = vm.context().new_tuple(bases); + let bases = bases.into_tuple(vm); // Prepare uses full __getattribute__ resolution chain. let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; - let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; + let namespace = vm.invoke(prepare, vec![name_obj.clone(), bases.clone()])?; let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?; let cells = vm.ctx.new_dict(); vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; + + namespace.set_item("__name__", name_obj.clone(), vm)?; + namespace.set_item("__qualname__", qualified_name.into_object(), vm)?; + let class = vm.call_method( metaclass.as_object(), "__call__", - vec![name_arg, bases, namespace.into_object()], + vec![name_obj, bases, namespace.into_object()], )?; cells.set_item("__class__", class.clone(), vm)?; Ok(class) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index dd045c3bf..d5a4b48ec 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -19,6 +19,7 @@ struct Compiler { nxt_label: usize, source_path: Option, current_source_location: ast::Location, + current_qualified_path: Option, in_loop: bool, in_function_def: bool, } @@ -75,6 +76,7 @@ impl Compiler { nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), + current_qualified_path: None, in_loop: false, in_function_def: false, } @@ -610,6 +612,10 @@ impl Compiler { self.in_loop = false; self.in_function_def = true; + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(self.create_qualified_name(name, ".")); + self.prepare_decorators(decorator_list)?; let mut flags = self.enter_function(name, args)?; @@ -668,7 +674,7 @@ impl Compiler { }); self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -681,6 +687,7 @@ impl Compiler { name: name.to_string(), }); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -696,6 +703,11 @@ impl Compiler { ) -> Result<(), CompileError> { let was_in_loop = self.in_loop; self.in_loop = false; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(qualified_name.clone()); + self.prepare_decorators(decorator_list)?; self.emit(Instruction::LoadBuildClass); let line_number = self.get_source_line_number(); @@ -711,6 +723,12 @@ impl Compiler { let (new_body, doc_str) = get_doc(body); + self.emit(Instruction::LoadName { + name: "__name__".to_string(), + }); + self.emit(Instruction::StoreName { + name: "__module__".to_string(), + }); self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, @@ -737,7 +755,7 @@ impl Compiler { self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -779,6 +797,7 @@ impl Compiler { self.emit(Instruction::StoreName { name: name.to_string(), }); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; Ok(()) } @@ -1604,6 +1623,14 @@ impl Compiler { self.current_source_location.get_row() } + fn create_qualified_name(&self, name: &str, suffix: &str) -> String { + if let Some(ref qualified_path) = self.current_qualified_path { + format!("{}.{}{}", qualified_path, name, suffix) + } else { + format!("{}{}", name, suffix) + } + } + fn mark_generator(&mut self) { self.current_code_object().is_generator = true; } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index d614691c9..6fde05a59 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -10,7 +10,6 @@ use crate::builtins; use crate::bytecode; use crate::function::PyFuncArgs; use crate::obj::objbool; -use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objint::PyInt; @@ -18,6 +17,7 @@ use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objstr::PyString; use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; @@ -566,7 +566,10 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let qualified_name = self.pop_value(); + let qualified_name = self + .pop_value() + .downcast::() + .expect("qualified name to be a string"); let code_obj = self .pop_value() .downcast() @@ -606,7 +609,8 @@ impl Frame { .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); - vm.set_attr(&obj, "__name__", qualified_name.clone())?; + let name = qualified_name.value.split('.').next_back().unwrap(); + vm.set_attr(&obj, "__name__", vm.new_str(name.to_string()))?; vm.set_attr(&obj, "__qualname__", qualified_name)?; let module = self .scope @@ -747,9 +751,7 @@ impl Frame { Ok(None) } bytecode::Instruction::LoadBuildClass => { - let rustfunc = - PyBuiltinFunction::new(Box::new(builtins::builtin_build_class_)).into_ref(vm); - self.push_value(rustfunc.into_object()); + self.push_value(vm.ctx.new_rustfunc(builtins::builtin_build_class_)); Ok(None) } bytecode::Instruction::UnpackSequence { size } => { diff --git a/vm/src/function.rs b/vm/src/function.rs index 305c4de5e..705a172cb 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -244,6 +244,12 @@ pub trait FromArgs: Sized { /// an appropriate FromArgs implementation must be created. pub struct KwArgs(HashMap); +impl KwArgs { + pub fn pop_kwarg(&mut self, name: &str) -> Option { + self.0.remove(name) + } +} + impl FromArgs for KwArgs where T: TryFromObject, @@ -274,8 +280,16 @@ impl IntoIterator for KwArgs { /// /// `Args` optionally accepts a generic type parameter to allow type checks /// or conversions of each argument. +#[derive(Clone)] pub struct Args(Vec); +impl Args> { + pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_tuple(self.0.into_iter().map(|obj| obj.into_object()).collect()) + } +} + impl FromArgs for Args where T: TryFromObject, From 664554bc5e0bf38287dd2bbe12ce1f509c03c359 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 15:21:56 +0100 Subject: [PATCH 174/223] Fix dir bug - not including object attributes correctly. --- tests/snippets/builtin_dir.py | 3 +++ vm/src/obj/objobject.rs | 38 ++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index a121fb718..b5fbf8d0b 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -8,6 +8,9 @@ a = A() assert "test" in dir(a), "test not in a" assert "test" in dir(A), "test not in A" +a.x = 3 +assert "x" in dir(a), "x not in a" + class B(A): def __dir__(self): return ('q', 'h') diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 2be4ff4f4..8f6c90a36 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,6 +1,6 @@ use super::objdict::PyDictRef; use super::objlist::PyList; -use super::objstr::PyStringRef; +use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; @@ -117,13 +117,29 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id()) } -pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { - let attributes = get_attributes(&obj); +pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut attributes: PyAttributes = objtype::get_attributes(obj.class()); + + // Get instance attributes: + if let Some(dict) = &obj.dict { + for (key, value) in dict { + if let Some(key_string) = key.payload::() { + attributes.insert(key_string.to_string(), value.clone()); + } else { + return Err(vm.new_type_error(format!( + "Attribute is not a string: {:?}", + vm.to_pystr(&key)? + ))); + } + } + } + let attributes: Vec = attributes .keys() .map(|k| vm.ctx.new_str(k.to_string())) .collect(); - PyList::from(attributes) + + Ok(PyList::from(attributes)) } fn object_format( @@ -230,17 +246,3 @@ fn object_getattr( Ok(None) } } - -pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { - // Get class attributes: - let mut attributes = objtype::get_attributes(obj.class()); - - // Get instance attributes: - if let Some(dict) = &obj.dict { - for (key, value) in dict { - attributes.insert(key.to_string(), value.clone()); - } - } - - attributes -} From aa38a1aa51913f74eeeea4089268650495db23f3 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Wed, 10 Apr 2019 10:59:04 +0100 Subject: [PATCH 175/223] Start refactor of ast. --- vm/src/stdlib/ast.rs | 594 +++++++++++++++---------------------------- 1 file changed, 203 insertions(+), 391 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index e7ac5308c..67c58a15d 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -10,14 +10,15 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; use crate::function::PyFuncArgs; +use crate::obj::objlist::PyListRef; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; #[derive(Debug)] struct AstNode; -// type AstNodeRef = PyRef; +type AstNodeRef = PyRef; impl PyValue for AstNode { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -41,32 +42,37 @@ impl AstToPyAst { } */ -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { - let mut body = vec![]; - for statement in &program.statements { - body.push(statement_to_ast(&vm, statement)); +macro_rules! node { + ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { + { + let node = create_node($vm, stringify!($node_name)); + $( + $vm.set_attr(&node, stringify!($attr_name), $attr_value).unwrap(); + )* + node + } + }; + ( $vm: expr, $node_name:ident) => { + create_node($vm, stringify!($node_name)) } - // TODO: create Module node: - // let ast_node = ctx.new_instance(this.Module); - let ast_node = create_node(vm, "program"); - let py_body = vm.ctx.new_list(body); - vm.set_attr(&ast_node, "body", py_body).unwrap(); - ast_node +} + +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { + let py_body = statements_to_ast(vm, &program.statements); + node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, _name: &str) -> PyObjectRef { - // TODO: instantiate a class of type given by name - // TODO: lookup in the current module? - PyObject::new(AstNode, AstNode::class(vm), Some(vm.ctx.new_dict())) +fn create_node(vm: &VirtualMachine, name: &str) -> PyObjectRef { + PyObject::new(AstNode, vm.class("ast", name), Some(vm.ctx.new_dict())) } -fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyObjectRef { - let mut py_statements = vec![]; - for statement in statements { - py_statements.push(statement_to_ast(vm, statement)); - } - vm.ctx.new_list(py_statements) +fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { + let body = statements + .iter() + .map(|statement| statement_to_ast(&vm, statement)) + .collect(); + vm.ctx.new_list(body).downcast().unwrap() } fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { @@ -76,22 +82,11 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P body, decorator_list, .. - } => { - let node = create_node(vm, "ClassDef"); - - // Set name: - vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) - .unwrap(); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.set_attr(&node, "decorator_list", py_decorator_list) - .unwrap(); - node - } + } => node!(vm, ClassDef, { + name => vm.ctx.new_str(name.to_string()), + body => statements_to_ast(vm, body), + decorator_list => expressions_to_ast(vm, decorator_list), + }), ast::Statement::FunctionDef { name, args, @@ -99,140 +94,86 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P decorator_list, returns, } => { - let node = create_node(vm, "FunctionDef"); - - // Set name: - vm.set_attr(&node, "name", vm.ctx.new_str(name.to_string())) - .unwrap(); - - vm.set_attr(&node, "args", parameters_to_ast(vm, args)) - .unwrap(); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.set_attr(&node, "decorator_list", py_decorator_list) - .unwrap(); - let py_returns = if let Some(hint) = returns { expression_to_ast(vm, hint) } else { vm.ctx.none() }; - vm.set_attr(&node, "returns", py_returns).unwrap(); - node + node!(vm, FunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args), + body => statements_to_ast(vm, body), + decorator_list => expressions_to_ast(vm, decorator_list), + returns => py_returns + }) } - ast::Statement::Continue => create_node(vm, "Continue"), - ast::Statement::Break => create_node(vm, "Break"), - ast::Statement::Pass => create_node(vm, "Pass"), + ast::Statement::Continue => node!(vm, Continue), + ast::Statement::Break => node!(vm, Break), + ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { - let node = create_node(vm, "Pass"); - - vm.set_attr(&node, "test", expression_to_ast(vm, test)) - .unwrap(); - let py_msg = match msg { Some(msg) => expression_to_ast(vm, msg), None => vm.ctx.none(), }; - vm.set_attr(&node, "msg", py_msg).unwrap(); - - node + node!(vm, Assert, { + test => expression_to_ast(vm, test), + msg => py_msg + }) } ast::Statement::Delete { targets } => { - let node = create_node(vm, "Delete"); - let py_targets = vm .ctx .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); - vm.set_attr(&node, "targets", py_targets).unwrap(); - node + node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { - let node = create_node(vm, "Return"); - let py_value = if let Some(value) = value { expression_to_ast(vm, value) } else { vm.ctx.none() }; - vm.set_attr(&node, "value", py_value).unwrap(); - node - } - ast::Statement::If { test, body, orelse } => { - let node = create_node(vm, "If"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) - } else { - vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node + node!(vm, Return, { + value => py_value + }) } + ast::Statement::If { test, body, orelse } => node!(vm, If, { + test => expression_to_ast(vm, test), + body => statements_to_ast(vm, body), + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() + } else { + vm.ctx.none() + } + }), ast::Statement::For { target, iter, body, orelse, - } => { - let node = create_node(vm, "For"); - - let py_target = expression_to_ast(vm, target); - vm.set_attr(&node, "target", py_target).unwrap(); - - let py_iter = expression_to_ast(vm, iter); - vm.set_attr(&node, "iter", py_iter).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } => node!(vm, For, { + target => expression_to_ast(vm, target), + iter => expression_to_ast(vm, iter), + body => statements_to_ast(vm, body), + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() } else { vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node - } - ast::Statement::While { test, body, orelse } => { - let node = create_node(vm, "While"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = statements_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } + }), + ast::Statement::While { test, body, orelse } => node!(vm, While, { + test => expression_to_ast(vm, test), + body => statements_to_ast(vm, body), + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse).into_object() } else { vm.ctx.none() - }; - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node - } - ast::Statement::Expression { expression } => { - let node = create_node(vm, "Expr"); - - let value = expression_to_ast(vm, expression); - vm.set_attr(&node, "value", value).unwrap(); - - node - } + } + }), + ast::Statement::Expression { expression } => node!(vm, Expr, { + value => expression_to_ast(vm, expression) + }), x => { unimplemented!("{:?}", x); } @@ -245,35 +186,23 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P node } -fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyObjectRef { - let mut py_expression_nodes = vec![]; - for expression in expressions { - py_expression_nodes.push(expression_to_ast(vm, expression)); - } - vm.ctx.new_list(py_expression_nodes) +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { + let py_expression_nodes = expressions + .iter() + .map(|expression| expression_to_ast(vm, expression)) + .collect(); + vm.ctx.new_list(py_expression_nodes).downcast().unwrap() } fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { let node = match &expression { - ast::Expression::Call { function, args, .. } => { - let node = create_node(vm, "Call"); - - let py_func_ast = expression_to_ast(vm, function); - vm.set_attr(&node, "func", py_func_ast).unwrap(); - - let py_args = expressions_to_ast(vm, args); - vm.set_attr(&node, "args", py_args).unwrap(); - - node - } + ast::Expression::Call { function, args, .. } => node!(vm, Call, { + func => expression_to_ast(vm, function), + args => expressions_to_ast(vm, args), + }), ast::Expression::Binop { a, op, b } => { - let node = create_node(vm, "BinOp"); - - let py_a = expression_to_ast(vm, a); - vm.set_attr(&node, "left", py_a).unwrap(); - // Operator: - let str_op = match op { + let op = match op { ast::Operator::Add => "Add", ast::Operator::Sub => "Sub", ast::Operator::Mult => "Mult", @@ -288,53 +217,43 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Operator::BitAnd => "BitAnd", ast::Operator::FloorDiv => "FloorDiv", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - - let py_b = expression_to_ast(vm, b); - vm.set_attr(&node, "right", py_b).unwrap(); - node + node!(vm, BinOp, { + left => expression_to_ast(vm, a), + op => vm.ctx.new_str(op.to_string()), + right => expression_to_ast(vm, b), + }) } ast::Expression::Unop { op, a } => { - let node = create_node(vm, "UnaryOp"); - - let str_op = match op { + let op = match op { ast::UnaryOperator::Not => "Not", ast::UnaryOperator::Inv => "Invert", ast::UnaryOperator::Neg => "USub", ast::UnaryOperator::Pos => "UAdd", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - - let py_a = expression_to_ast(vm, a); - vm.set_attr(&node, "operand", py_a).unwrap(); - - node + node!(vm, UnaryOp, { + op => vm.ctx.new_str(op.to_string()), + operand => expression_to_ast(vm, a), + }) } ast::Expression::BoolOp { a, op, b } => { - let node = create_node(vm, "BoolOp"); - // Attach values: let py_a = expression_to_ast(vm, a); let py_b = expression_to_ast(vm, b); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); - vm.set_attr(&node, "values", py_values).unwrap(); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.set_attr(&node, "op", py_op).unwrap(); - node + node!(vm, BoolOp, { + op => py_op, + values => py_values, + }) } ast::Expression::Compare { vals, ops } => { - let node = create_node(vm, "Compare"); - let py_a = expression_to_ast(vm, &vals[0]); - vm.set_attr(&node, "left", py_a).unwrap(); // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -355,53 +274,31 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj .collect(), ); - vm.set_attr(&node, "ops", py_ops).unwrap(); - let py_b = vm.ctx.new_list( vals.iter() .skip(1) .map(|x| expression_to_ast(vm, x)) .collect(), ); - vm.set_attr(&node, "comparators", py_b).unwrap(); - node - } - ast::Expression::Identifier { name } => { - let node = create_node(vm, "Identifier"); - - // Id: - let py_name = vm.ctx.new_str(name.clone()); - vm.set_attr(&node, "id", py_name).unwrap(); - node - } - ast::Expression::Lambda { args, body } => { - let node = create_node(vm, "Lambda"); - - vm.set_attr(&node, "args", parameters_to_ast(vm, args)) - .unwrap(); - - let py_body = expression_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - node - } - ast::Expression::IfExpression { test, body, orelse } => { - let node = create_node(vm, "IfExp"); - - let py_test = expression_to_ast(vm, test); - vm.set_attr(&node, "test", py_test).unwrap(); - - let py_body = expression_to_ast(vm, body); - vm.set_attr(&node, "body", py_body).unwrap(); - - let py_orelse = expression_to_ast(vm, orelse); - vm.set_attr(&node, "orelse", py_orelse).unwrap(); - - node + node!(vm, Compare, { + left => py_a, + ops => py_ops, + comparators => py_b, + }) } + ast::Expression::Identifier { name } => node!(vm, Identifier, { + id => vm.ctx.new_str(name.clone()) + }), + ast::Expression::Lambda { args, body } => node!(vm, Lambda, { + args => parameters_to_ast(vm, args), + body => expression_to_ast(vm, body), + }), + ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { + text => expression_to_ast(vm, test), + body => expression_to_ast(vm, body), + or_else => expression_to_ast(vm, orelse), + }), ast::Expression::Number { value } => { - let node = create_node(vm, "Num"); - let py_n = match value { ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), ast::Number::Float { value } => vm.ctx.new_float(*value), @@ -409,62 +306,30 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - vm.set_attr(&node, "n", py_n).unwrap(); - - node - } - ast::Expression::True => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.new_bool(true)).unwrap(); - - node - } - ast::Expression::False => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.new_bool(false)).unwrap(); - - node - } - ast::Expression::None => { - let node = create_node(vm, "NameConstant"); - - vm.set_attr(&node, "value", vm.ctx.none()).unwrap(); - - node - } - ast::Expression::Ellipsis => create_node(vm, "Ellipsis"), - ast::Expression::List { elements } => { - let node = create_node(vm, "List"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node - } - ast::Expression::Tuple { elements } => { - let node = create_node(vm, "Tuple"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node - } - ast::Expression::Set { elements } => { - let node = create_node(vm, "Set"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.set_attr(&node, "elts", py_elts).unwrap(); - - node + node!(vm, Num, { + n => py_n + }) } + ast::Expression::True => node!(vm, NameConstant, { + value => vm.ctx.new_bool(true) + }), + ast::Expression::False => node!(vm, NameConstant, { + value => vm.ctx.new_bool(false) + }), + ast::Expression::None => node!(vm, NameConstant, { + value => vm.ctx.none() + }), + ast::Expression::Ellipsis => node!(vm, Ellipsis), + ast::Expression::List { elements } => node!(vm, List, { + elts => expressions_to_ast(vm, &elements) + }), + ast::Expression::Tuple { elements } => node!(vm, Tuple, { + elts => expressions_to_ast(vm, &elements) + }), + ast::Expression::Set { elements } => node!(vm, Set, { + elts => expressions_to_ast(vm, &elements) + }), ast::Expression::Dict { elements } => { - let node = create_node(vm, "Dict"); - let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { @@ -472,96 +337,65 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj values.push(expression_to_ast(vm, v)); } - let py_keys = vm.ctx.new_list(keys); - vm.set_attr(&node, "keys", py_keys).unwrap(); - - let py_values = vm.ctx.new_list(values); - vm.set_attr(&node, "values", py_values).unwrap(); - - node + node!(vm, Dict, { + keys => vm.ctx.new_list(keys), + values => vm.ctx.new_list(values), + }) } ast::Expression::Comprehension { kind, generators } => { - let node = match kind.deref() { - ast::ComprehensionKind::GeneratorExpression { .. } => { - create_node(vm, "GeneratorExp") - } - ast::ComprehensionKind::List { .. } => create_node(vm, "ListComp"), - ast::ComprehensionKind::Set { .. } => create_node(vm, "SetComp"), - ast::ComprehensionKind::Dict { .. } => create_node(vm, "DictComp"), - }; - let g = generators .iter() .map(|g| comprehension_to_ast(vm, g)) .collect(); let py_generators = vm.ctx.new_list(g); - vm.set_attr(&node, "generators", py_generators).unwrap(); - node + match kind.deref() { + ast::ComprehensionKind::GeneratorExpression { .. } => { + node!(vm, GeneratorExp, {generators => py_generators}) + } + ast::ComprehensionKind::List { .. } => { + node!(vm, ListComp, {generators => py_generators}) + } + ast::ComprehensionKind::Set { .. } => { + node!(vm, SetComp, {generators => py_generators}) + } + ast::ComprehensionKind::Dict { .. } => { + node!(vm, DictComp, {generators => py_generators}) + } + } } ast::Expression::Yield { value } => { - let node = create_node(vm, "Yield"); - let py_value = match value { Some(value) => expression_to_ast(vm, value), None => vm.ctx.none(), }; - vm.set_attr(&node, "value", py_value).unwrap(); - - node + node!(vm, Yield, { + value => py_value + }) } ast::Expression::YieldFrom { value } => { - let node = create_node(vm, "YieldFrom"); - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - node - } - ast::Expression::Subscript { a, b } => { - let node = create_node(vm, "Subscript"); - - let py_value = expression_to_ast(vm, a); - vm.set_attr(&node, "value", py_value).unwrap(); - - let py_slice = expression_to_ast(vm, b); - vm.set_attr(&node, "slice", py_slice).unwrap(); - - node - } - ast::Expression::Attribute { value, name } => { - let node = create_node(vm, "Attribute"); - - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - let py_attr = vm.ctx.new_str(name.to_string()); - vm.set_attr(&node, "attr", py_attr).unwrap(); - - node - } - ast::Expression::Starred { value } => { - let node = create_node(vm, "Starred"); - - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - - node - } - ast::Expression::Slice { elements } => { - let node = create_node(vm, "Slice"); - - let py_value = expressions_to_ast(vm, elements); - vm.set_attr(&node, "bounds", py_value).unwrap(); - - node + node!(vm, YieldFrom, { + value => py_value + }) } + ast::Expression::Subscript { a, b } => node!(vm, Subscript, { + value => expression_to_ast(vm, a), + slice => expression_to_ast(vm, b), + }), + ast::Expression::Attribute { value, name } => node!(vm, Attribute, { + value => expression_to_ast(vm, value), + attr => vm.ctx.new_str(name.to_string()), + }), + ast::Expression::Starred { value } => node!(vm, Starred, { + value => expression_to_ast(vm, value) + }), + ast::Expression::Slice { elements } => node!(vm, Slice, { + bounds => expressions_to_ast(vm, elements) + }), ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { - let node = create_node(vm, "Bytes"); - vm.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())) - .unwrap(); - node + node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } }; @@ -573,74 +407,50 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { - let node = create_node(vm, "arguments"); + let args = vm + .ctx + .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()); - vm.set_attr( - &node, - "args", - vm.ctx - .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), - ) - .unwrap(); - - node + node!(vm, arguments, { args => args }) } fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { - let node = create_node(vm, "arg"); - - let py_arg = vm.ctx.new_str(parameter.arg.to_string()); - vm.set_attr(&node, "arg", py_arg).unwrap(); - let py_annotation = if let Some(annotation) = ¶meter.annotation { expression_to_ast(vm, annotation) } else { vm.ctx.none() }; - vm.set_attr(&node, "annotation", py_annotation).unwrap(); - node + node!(vm, arg, { + arg => vm.ctx.new_str(parameter.arg.to_string()), + annotation => py_annotation + }) } fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { - let node = create_node(vm, "comprehension"); - - let py_target = expression_to_ast(vm, &comprehension.target); - vm.set_attr(&node, "target", py_target).unwrap(); - - let py_iter = expression_to_ast(vm, &comprehension.iter); - vm.set_attr(&node, "iter", py_iter).unwrap(); - - let py_ifs = expressions_to_ast(vm, &comprehension.ifs); - vm.set_attr(&node, "ifs", py_ifs).unwrap(); - - node + node!(vm, comprehension, { + target => expression_to_ast(vm, &comprehension.target), + iter => expression_to_ast(vm, &comprehension.iter), + ifs => expressions_to_ast(vm, &comprehension.ifs), + }) } fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { match string { ast::StringGroup::Constant { value } => { - let node = create_node(vm, "Str"); - vm.set_attr(&node, "s", vm.ctx.new_str(value.clone())) - .unwrap(); - node + node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - let node = create_node(vm, "FormattedValue"); - let py_value = expression_to_ast(vm, value); - vm.set_attr(&node, "value", py_value).unwrap(); - node + node!(vm, FormattedValue, { value => expression_to_ast(vm, value) }) } ast::StringGroup::Joined { values } => { - let node = create_node(vm, "JoinedStr"); let py_values = vm.ctx.new_list( values .iter() .map(|value| string_to_ast(vm, value)) .collect(), ); - vm.set_attr(&node, "values", py_values).unwrap(); - node + node!(vm, JoinedStr, { values => py_values }) } } } @@ -659,11 +469,13 @@ fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), - "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}), - "AST" => py_class!(ctx, "_ast.AST", ctx.object(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), + "AST" => ast_base, }) } From d7e0ceec125f8101b9317f56781bd277878a3caa Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 09:34:12 +0100 Subject: [PATCH 176/223] New (PyRef) style types. --- vm/src/stdlib/ast.rs | 104 +++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 67c58a15d..518465fe6 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -9,11 +9,10 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; -use crate::function::PyFuncArgs; use crate::obj::objlist::PyListRef; -use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -26,28 +25,12 @@ impl PyValue for AstNode { } } -/* - * Idea: maybe we can create a sort of struct with some helper functions? -struct AstToPyAst { - ctx: &PyContext, -} - -impl AstToPyAst { - fn new(ctx: &PyContext) -> Self { - AstToPyAst { - ctx: ctx, - } - } - -} -*/ - macro_rules! node { ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { { let node = create_node($vm, stringify!($node_name)); $( - $vm.set_attr(&node, stringify!($attr_name), $attr_value).unwrap(); + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value).unwrap(); )* node } @@ -57,25 +40,27 @@ macro_rules! node { } } -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> AstNodeRef { let py_body = statements_to_ast(vm, &program.statements); node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, name: &str) -> PyObjectRef { - PyObject::new(AstNode, vm.class("ast", name), Some(vm.ctx.new_dict())) +fn create_node(vm: &VirtualMachine, name: &str) -> AstNodeRef { + AstNode + .into_ref_with_type(vm, vm.class("ast", name)) + .unwrap() } fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { let body = statements .iter() - .map(|statement| statement_to_ast(&vm, statement)) + .map(|statement| statement_to_ast(&vm, statement).into_object()) .collect(); vm.ctx.new_list(body).downcast().unwrap() } -fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { +fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> AstNodeRef { let node = match &statement.node { ast::Statement::ClassDef { name, @@ -95,7 +80,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P returns, } => { let py_returns = if let Some(hint) = returns { - expression_to_ast(vm, hint) + expression_to_ast(vm, hint).into_object() } else { vm.ctx.none() }; @@ -112,7 +97,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { let py_msg = match msg { - Some(msg) => expression_to_ast(vm, msg), + Some(msg) => expression_to_ast(vm, msg).into_object(), None => vm.ctx.none(), }; node!(vm, Assert, { @@ -121,15 +106,18 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P }) } ast::Statement::Delete { targets } => { - let py_targets = vm - .ctx - .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); + let py_targets = vm.ctx.new_tuple( + targets + .iter() + .map(|v| expression_to_ast(vm, v).into_object()) + .collect(), + ); node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { let py_value = if let Some(value) = value { - expression_to_ast(vm, value) + expression_to_ast(vm, value).into_object() } else { vm.ctx.none() }; @@ -181,7 +169,7 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); - vm.set_attr(&node, "lineno", lineno).unwrap(); + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); node } @@ -189,12 +177,12 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { let py_expression_nodes = expressions .iter() - .map(|expression| expression_to_ast(vm, expression)) + .map(|expression| expression_to_ast(vm, expression).into_object()) .collect(); vm.ctx.new_list(py_expression_nodes).downcast().unwrap() } -fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNodeRef { let node = match &expression { ast::Expression::Call { function, args, .. } => node!(vm, Call, { func => expression_to_ast(vm, function), @@ -237,8 +225,8 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } ast::Expression::BoolOp { a, op, b } => { // Attach values: - let py_a = expression_to_ast(vm, a); - let py_b = expression_to_ast(vm, b); + let py_a = expression_to_ast(vm, a).into_object(); + let py_b = expression_to_ast(vm, b).into_object(); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); let str_op = match op { @@ -277,7 +265,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let py_b = vm.ctx.new_list( vals.iter() .skip(1) - .map(|x| expression_to_ast(vm, x)) + .map(|x| expression_to_ast(vm, x).into_object()) .collect(), ); node!(vm, Compare, { @@ -333,8 +321,8 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k)); - values.push(expression_to_ast(vm, v)); + keys.push(expression_to_ast(vm, k).into_object()); + values.push(expression_to_ast(vm, v).into_object()); } node!(vm, Dict, { @@ -345,7 +333,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Expression::Comprehension { kind, generators } => { let g = generators .iter() - .map(|g| comprehension_to_ast(vm, g)) + .map(|g| comprehension_to_ast(vm, g).into_object()) .collect(); let py_generators = vm.ctx.new_list(g); @@ -366,7 +354,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj } ast::Expression::Yield { value } => { let py_value = match value { - Some(value) => expression_to_ast(vm, value), + Some(value) => expression_to_ast(vm, value).into_object(), None => vm.ctx.none(), }; node!(vm, Yield, { @@ -401,22 +389,24 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); - vm.set_attr(&node, "lineno", lineno).unwrap(); - + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); node } -fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { - let args = vm - .ctx - .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()); +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> AstNodeRef { + let args = vm.ctx.new_list( + args.args + .iter() + .map(|a| parameter_to_ast(vm, a).into_object()) + .collect(), + ); node!(vm, arguments, { args => args }) } -fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeRef { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(vm, annotation) + expression_to_ast(vm, annotation).into_object() } else { vm.ctx.none() }; @@ -427,7 +417,7 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObject }) } -fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { +fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> AstNodeRef { node!(vm, comprehension, { target => expression_to_ast(vm, &comprehension.target), iter => expression_to_ast(vm, &comprehension.iter), @@ -435,7 +425,7 @@ fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) }) } -fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> AstNodeRef { match string { ast::StringGroup::Constant { value } => { node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) @@ -447,7 +437,7 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef let py_values = vm.ctx.new_list( values .iter() - .map(|value| string_to_ast(vm, value)) + .map(|value| string_to_ast(vm, value).into_object()) .collect(), ); node!(vm, JoinedStr, { values => py_values }) @@ -455,15 +445,11 @@ fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef } } -fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); - - let source_string = objstr::get_value(source); - let internal_ast = parser::parse_program(&source_string) +fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { + let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - let ast_node = program_to_ast(&vm, &internal_ast); - Ok(ast_node) + Ok(program_to_ast(&vm, &internal_ast)) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From 8a1466557531880e5999b176cef746f19e1bd1cd Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 14:49:24 +0100 Subject: [PATCH 177/223] Ast refactor: Everything can fail. --- vm/src/stdlib/ast.rs | 228 +++++++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 114 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 518465fe6..0612c0d80 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -28,11 +28,11 @@ impl PyValue for AstNode { macro_rules! node { ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { { - let node = create_node($vm, stringify!($node_name)); + let node = create_node($vm, stringify!($node_name))?; $( - $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value).unwrap(); + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value)?; )* - node + Ok(node) } }; ( $vm: expr, $node_name:ident) => { @@ -40,27 +40,31 @@ macro_rules! node { } } -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> AstNodeRef { - let py_body = statements_to_ast(vm, &program.statements); +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyResult { + let py_body = statements_to_ast(vm, &program.statements)?; node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, name: &str) -> AstNodeRef { - AstNode - .into_ref_with_type(vm, vm.class("ast", name)) - .unwrap() +fn create_node(vm: &VirtualMachine, name: &str) -> PyResult { + AstNode.into_ref_with_type(vm, vm.class("ast", name)) } -fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyListRef { - let body = statements +fn statements_to_ast( + vm: &VirtualMachine, + statements: &[ast::LocatedStatement], +) -> PyResult { + let body: PyResult> = statements .iter() - .map(|statement| statement_to_ast(&vm, statement).into_object()) + .map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object())) .collect(); - vm.ctx.new_list(body).downcast().unwrap() + Ok(vm.ctx.new_list(body?).downcast().unwrap()) } -fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> AstNodeRef { +fn statement_to_ast( + vm: &VirtualMachine, + statement: &ast::LocatedStatement, +) -> PyResult { let node = match &statement.node { ast::Statement::ClassDef { name, @@ -69,8 +73,8 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A .. } => node!(vm, ClassDef, { name => vm.ctx.new_str(name.to_string()), - body => statements_to_ast(vm, body), - decorator_list => expressions_to_ast(vm, decorator_list), + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, }), ast::Statement::FunctionDef { name, @@ -80,15 +84,15 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A returns, } => { let py_returns = if let Some(hint) = returns { - expression_to_ast(vm, hint).into_object() + expression_to_ast(vm, hint)?.into_object() } else { vm.ctx.none() }; node!(vm, FunctionDef, { name => vm.ctx.new_str(name.to_string()), - args => parameters_to_ast(vm, args), - body => statements_to_ast(vm, body), - decorator_list => expressions_to_ast(vm, decorator_list), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, returns => py_returns }) } @@ -97,27 +101,25 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { let py_msg = match msg { - Some(msg) => expression_to_ast(vm, msg).into_object(), + Some(msg) => expression_to_ast(vm, msg)?.into_object(), None => vm.ctx.none(), }; node!(vm, Assert, { - test => expression_to_ast(vm, test), + test => expression_to_ast(vm, test)?, msg => py_msg }) } ast::Statement::Delete { targets } => { - let py_targets = vm.ctx.new_tuple( - targets - .iter() - .map(|v| expression_to_ast(vm, v).into_object()) - .collect(), - ); - + let targets: PyResult<_> = targets + .iter() + .map(|v| Ok(expression_to_ast(vm, v)?.into_object())) + .collect(); + let py_targets = vm.ctx.new_tuple(targets?); node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { let py_value = if let Some(value) = value { - expression_to_ast(vm, value).into_object() + expression_to_ast(vm, value)?.into_object() } else { vm.ctx.none() }; @@ -127,10 +129,10 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A }) } ast::Statement::If { test, body, orelse } => node!(vm, If, { - test => expression_to_ast(vm, test), - body => statements_to_ast(vm, body), + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, orelse => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } @@ -141,52 +143,52 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> A body, orelse, } => node!(vm, For, { - target => expression_to_ast(vm, target), - iter => expression_to_ast(vm, iter), - body => statements_to_ast(vm, body), + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, or_else => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } }), ast::Statement::While { test, body, orelse } => node!(vm, While, { - test => expression_to_ast(vm, test), - body => statements_to_ast(vm, body), + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, orelse => if let Some(orelse) = orelse { - statements_to_ast(vm, orelse).into_object() + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() } }), ast::Statement::Expression { expression } => node!(vm, Expr, { - value => expression_to_ast(vm, expression) + value => expression_to_ast(vm, expression)? }), x => { - unimplemented!("{:?}", x); + return Err(vm.new_type_error(format!("Ast not implemented: {:?}", x))); } - }; + }?; // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyListRef { - let py_expression_nodes = expressions +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyResult { + let py_expression_nodes: PyResult<_> = expressions .iter() - .map(|expression| expression_to_ast(vm, expression).into_object()) + .map(|expression| Ok(expression_to_ast(vm, expression)?.into_object())) .collect(); - vm.ctx.new_list(py_expression_nodes).downcast().unwrap() + Ok(vm.ctx.new_list(py_expression_nodes?).downcast().unwrap()) } -fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNodeRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult { let node = match &expression { ast::Expression::Call { function, args, .. } => node!(vm, Call, { - func => expression_to_ast(vm, function), - args => expressions_to_ast(vm, args), + func => expression_to_ast(vm, function)?, + args => expressions_to_ast(vm, args)?, }), ast::Expression::Binop { a, op, b } => { // Operator: @@ -206,9 +208,9 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo ast::Operator::FloorDiv => "FloorDiv", }; node!(vm, BinOp, { - left => expression_to_ast(vm, a), + left => expression_to_ast(vm, a)?, op => vm.ctx.new_str(op.to_string()), - right => expression_to_ast(vm, b), + right => expression_to_ast(vm, b)?, }) } ast::Expression::Unop { op, a } => { @@ -220,13 +222,13 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }; node!(vm, UnaryOp, { op => vm.ctx.new_str(op.to_string()), - operand => expression_to_ast(vm, a), + operand => expression_to_ast(vm, a)?, }) } ast::Expression::BoolOp { a, op, b } => { // Attach values: - let py_a = expression_to_ast(vm, a).into_object(); - let py_b = expression_to_ast(vm, b).into_object(); + let py_a = expression_to_ast(vm, a)?.into_object(); + let py_b = expression_to_ast(vm, b)?.into_object(); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); let str_op = match op { @@ -241,7 +243,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::Compare { vals, ops } => { - let py_a = expression_to_ast(vm, &vals[0]); + let left = expression_to_ast(vm, &vals[0])?; // Operator: let to_operator = |op: &ast::Comparison| match op { @@ -256,35 +258,35 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = vm.ctx.new_list( + let ops = vm.ctx.new_list( ops.iter() .map(|x| vm.ctx.new_str(to_operator(x).to_string())) .collect(), ); - let py_b = vm.ctx.new_list( - vals.iter() - .skip(1) - .map(|x| expression_to_ast(vm, x).into_object()) - .collect(), - ); + let comparators: PyResult<_> = vals + .iter() + .skip(1) + .map(|x| Ok(expression_to_ast(vm, x)?.into_object())) + .collect(); + let comparators = vm.ctx.new_list(comparators?); node!(vm, Compare, { - left => py_a, - ops => py_ops, - comparators => py_b, + left => left, + ops => ops, + comparators => comparators, }) } ast::Expression::Identifier { name } => node!(vm, Identifier, { id => vm.ctx.new_str(name.clone()) }), ast::Expression::Lambda { args, body } => node!(vm, Lambda, { - args => parameters_to_ast(vm, args), - body => expression_to_ast(vm, body), + args => parameters_to_ast(vm, args)?, + body => expression_to_ast(vm, body)?, }), ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { - text => expression_to_ast(vm, test), - body => expression_to_ast(vm, body), - or_else => expression_to_ast(vm, orelse), + text => expression_to_ast(vm, test)?, + body => expression_to_ast(vm, body)?, + or_else => expression_to_ast(vm, orelse)?, }), ast::Expression::Number { value } => { let py_n = match value { @@ -309,20 +311,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }), ast::Expression::Ellipsis => node!(vm, Ellipsis), ast::Expression::List { elements } => node!(vm, List, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Tuple { elements } => node!(vm, Tuple, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Set { elements } => node!(vm, Set, { - elts => expressions_to_ast(vm, &elements) + elts => expressions_to_ast(vm, &elements)? }), ast::Expression::Dict { elements } => { let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k).into_object()); - values.push(expression_to_ast(vm, v).into_object()); + keys.push(expression_to_ast(vm, k)?.into_object()); + values.push(expression_to_ast(vm, v)?.into_object()); } node!(vm, Dict, { @@ -331,11 +333,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::Comprehension { kind, generators } => { - let g = generators - .iter() - .map(|g| comprehension_to_ast(vm, g).into_object()) - .collect(); - let py_generators = vm.ctx.new_list(g); + let py_generators = map_ast(comprehension_to_ast, vm, generators)?; match kind.deref() { ast::ComprehensionKind::GeneratorExpression { .. } => { @@ -354,7 +352,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo } ast::Expression::Yield { value } => { let py_value = match value { - Some(value) => expression_to_ast(vm, value).into_object(), + Some(value) => expression_to_ast(vm, value)?.into_object(), None => vm.ctx.none(), }; node!(vm, Yield, { @@ -362,51 +360,45 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> AstNo }) } ast::Expression::YieldFrom { value } => { - let py_value = expression_to_ast(vm, value); + let py_value = expression_to_ast(vm, value)?; node!(vm, YieldFrom, { value => py_value }) } ast::Expression::Subscript { a, b } => node!(vm, Subscript, { - value => expression_to_ast(vm, a), - slice => expression_to_ast(vm, b), + value => expression_to_ast(vm, a)?, + slice => expression_to_ast(vm, b)?, }), ast::Expression::Attribute { value, name } => node!(vm, Attribute, { - value => expression_to_ast(vm, value), + value => expression_to_ast(vm, value)?, attr => vm.ctx.new_str(name.to_string()), }), ast::Expression::Starred { value } => node!(vm, Starred, { - value => expression_to_ast(vm, value) + value => expression_to_ast(vm, value)? }), ast::Expression::Slice { elements } => node!(vm, Slice, { - bounds => expressions_to_ast(vm, elements) + bounds => expressions_to_ast(vm, elements)? }), ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } - }; + }?; // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> AstNodeRef { - let args = vm.ctx.new_list( - args.args - .iter() - .map(|a| parameter_to_ast(vm, a).into_object()) - .collect(), - ); - +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyResult { + let args = map_ast(parameter_to_ast, vm, &args.args)?; node!(vm, arguments, { args => args }) } -fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeRef { +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(vm, annotation).into_object() + expression_to_ast(vm, annotation)?.into_object() } else { vm.ctx.none() }; @@ -417,29 +409,37 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> AstNodeR }) } -fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> AstNodeRef { +fn map_ast( + f: fn(vm: &VirtualMachine, &T) -> PyResult, + vm: &VirtualMachine, + items: &Vec, +) -> PyResult { + let list: PyResult> = + items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); + Ok(vm.ctx.new_list(list?)) +} + +fn comprehension_to_ast( + vm: &VirtualMachine, + comprehension: &ast::Comprehension, +) -> PyResult { node!(vm, comprehension, { - target => expression_to_ast(vm, &comprehension.target), - iter => expression_to_ast(vm, &comprehension.iter), - ifs => expressions_to_ast(vm, &comprehension.ifs), + target => expression_to_ast(vm, &comprehension.target)?, + iter => expression_to_ast(vm, &comprehension.iter)?, + ifs => expressions_to_ast(vm, &comprehension.ifs)?, }) } -fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> AstNodeRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyResult { match string { ast::StringGroup::Constant { value } => { node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - node!(vm, FormattedValue, { value => expression_to_ast(vm, value) }) + node!(vm, FormattedValue, { value => expression_to_ast(vm, value)? }) } ast::StringGroup::Joined { values } => { - let py_values = vm.ctx.new_list( - values - .iter() - .map(|value| string_to_ast(vm, value).into_object()) - .collect(), - ); + let py_values = map_ast(string_to_ast, vm, &values)?; node!(vm, JoinedStr, { values => py_values }) } } @@ -449,7 +449,7 @@ fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - Ok(program_to_ast(&vm, &internal_ast)) + program_to_ast(&vm, &internal_ast) } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { From 856914575148b0ec850be205a90ebcc82b36d0d2 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 15:03:39 +0100 Subject: [PATCH 178/223] Define classes for all the ast types. --- vm/src/stdlib/ast.rs | 52 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 0612c0d80..816882f31 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -458,10 +458,54 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), - "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "AST" => ast_base.clone(), + // TODO: There's got to be a better way! + "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), + "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), + "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), + "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), + "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), + "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), + "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), + "Bytes" => py_class!(ctx, "_ast.Bytes", ast_base.clone(), {}), "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), - "AST" => ast_base, + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Compare" => py_class!(ctx, "_ast.Compare", ast_base.clone(), {}), + "comprehension" => py_class!(ctx, "_ast.comprehension", ast_base.clone(), {}), + "Continue" => py_class!(ctx, "_ast.Continue", ast_base.clone(), {}), + "Delete" => py_class!(ctx, "_ast.Delete", ast_base.clone(), {}), + "Dict" => py_class!(ctx, "_ast.Dict", ast_base.clone(), {}), + "DictComp" => py_class!(ctx, "_ast.DictComp", ast_base.clone(), {}), + "Ellipsis" => py_class!(ctx, "_ast.Ellipsis", ast_base.clone(), {}), + "Expr" => py_class!(ctx, "_ast.Expr", ast_base.clone(), {}), + "For" => py_class!(ctx, "_ast.For", ast_base.clone(), {}), + "FormattedValue" => py_class!(ctx, "_ast.FormattedValue", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "GeneratorExp" => py_class!(ctx, "_ast.GeneratorExp", ast_base.clone(), {}), + "Identifier" => py_class!(ctx, "_ast.Identifier", ast_base.clone(), {}), + "If" => py_class!(ctx, "_ast.If", ast_base.clone(), {}), + "IfExp" => py_class!(ctx, "_ast.IfExp", ast_base.clone(), {}), + "JoinedStr" => py_class!(ctx, "_ast.JoinedStr", ast_base.clone(), {}), + "Lambda" => py_class!(ctx, "_ast.Lambda", ast_base.clone(), {}), + "List" => py_class!(ctx, "_ast.List", ast_base.clone(), {}), + "ListComp" => py_class!(ctx, "_ast.ListComp", ast_base.clone(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "Num" => py_class!(ctx, "_ast.Num", ast_base.clone(), {}), + "Pass" => py_class!(ctx, "_ast.Pass", ast_base.clone(), {}), + "Return" => py_class!(ctx, "_ast.Return", ast_base.clone(), {}), + "Set" => py_class!(ctx, "_ast.Set", ast_base.clone(), {}), + "SetComp" => py_class!(ctx, "_ast.SetComp", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Str" => py_class!(ctx, "_ast.Str", ast_base.clone(), {}), + "Subscript" => py_class!(ctx, "_ast.Subscript", ast_base.clone(), {}), + "Tuple" => py_class!(ctx, "_ast.Tuple", ast_base.clone(), {}), + "UnaryOp" => py_class!(ctx, "_ast.UnaryOp", ast_base.clone(), {}), + "While" => py_class!(ctx, "_ast.While", ast_base.clone(), {}), + "Yield" => py_class!(ctx, "_ast.Yield", ast_base.clone(), {}), + "YieldFrom" => py_class!(ctx, "_ast.YieldFrom", ast_base.clone(), {}), }) } From 72467069bea77b987ca4961e5cd19e25cca5ad4d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:10:31 +0300 Subject: [PATCH 179/223] Add os.getenv --- vm/src/stdlib/os.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 25937cd5c..217e4d381 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,11 +1,11 @@ -use std::fs; use std::fs::File; use std::fs::OpenOptions; use std::io::{ErrorKind, Read, Write}; +use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::PyFuncArgs; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytes::PyBytesRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; @@ -173,6 +173,13 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } +fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + match env::var(&key.value) { + Ok(val) => Ok(vm.new_str(val)), + Err(_) => Ok(default.into_option().unwrap_or(vm.get_none())), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -194,6 +201,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), + "getenv" => ctx.new_rustfunc(os_getenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 9b475a2c417902ea1bcc5f8f3bb87fc3241181cb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:15:02 +0300 Subject: [PATCH 180/223] Add os.putenv --- vm/src/stdlib/os.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 217e4d381..0b10ff2d9 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -180,6 +180,10 @@ fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMa } } +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { + env::set_var(&key.value, &value.value) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -202,6 +206,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), "getenv" => ctx.new_rustfunc(os_getenv), + "putenv" => ctx.new_rustfunc(os_putenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 951659a54d71efa99b939f919a278a664ff2864e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 8 Apr 2019 19:20:58 +0300 Subject: [PATCH 181/223] Add os.unsetenv --- vm/src/stdlib/os.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0b10ff2d9..6a13003b8 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -184,6 +184,10 @@ fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { env::set_var(&key.value, &value.value) } +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { + env::remove_var(&key.value) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -207,6 +211,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "listdir" => ctx.new_rustfunc(os_listdir), "getenv" => ctx.new_rustfunc(os_getenv), "putenv" => ctx.new_rustfunc(os_putenv), + "unsetenv" => ctx.new_rustfunc(os_unsetenv), "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From c33593efa9e97a4ca04cb40b9ea87e21656505f5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Tue, 9 Apr 2019 09:31:31 +0300 Subject: [PATCH 182/223] Remove getenv and add environ --- vm/src/stdlib/os.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6a13003b8..66698b3cb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -5,13 +5,14 @@ use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::PyFuncArgs; use crate::obj::objbytes::PyBytesRef; +use crate::obj::objdict::PyDictRef; use crate::obj::objint; use crate::obj::objint::PyIntRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -173,13 +174,6 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -fn os_getenv(key: PyStringRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { - match env::var(&key.value) { - Ok(val) => Ok(vm.new_str(val)), - Err(_) => Ok(default.into_option().unwrap_or(vm.get_none())), - } -} - fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { env::set_var(&key.value, &value.value) } @@ -188,6 +182,14 @@ fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { env::remove_var(&key.value) } +fn _os_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { + environ.set_item(&key, vm.new_str(value), vm).unwrap(); + } + environ +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -197,6 +199,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "posix".to_string() }; + let environ = _os_environ(vm); + py_module!(vm, "os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), @@ -209,9 +213,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mkdirs" => ctx.new_rustfunc(os_mkdirs), "rmdir" => ctx.new_rustfunc(os_rmdir), "listdir" => ctx.new_rustfunc(os_listdir), - "getenv" => ctx.new_rustfunc(os_getenv), "putenv" => ctx.new_rustfunc(os_putenv), "unsetenv" => ctx.new_rustfunc(os_unsetenv), + "environ" => environ, "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), From 4293b11429195905728cf0d5d1481837d1cea12a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 18:37:42 +0300 Subject: [PATCH 183/223] Add environ class --- Lib/os.py | 67 +++++++++++++++++++++++++++++++++++++ tests/snippets/stdlib_os.py | 14 ++++++++ vm/src/stdlib/mod.rs | 2 +- vm/src/stdlib/os.rs | 2 +- 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 Lib/os.py diff --git a/Lib/os.py b/Lib/os.py new file mode 100644 index 000000000..badb08ebb --- /dev/null +++ b/Lib/os.py @@ -0,0 +1,67 @@ +from _os import * + +class _Environ(): + def __init__(self, data, putenv, unsetenv): + self.putenv = putenv + self.unsetenv = unsetenv + self._data = data + + def __getitem__(self, key): + return self._data[key] + + def __setitem__(self, key, value): + self.putenv(key, value) + self._data[key] = value + + def __delitem__(self, key): + self.unsetenv(key) + del self._data[key] + + def __iter__(self): + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: + yield key + + def __len__(self): + return len(self._data) + + def __repr__(self): + return 'environ({{{}}})'.format(', '.join( + ('{}: {}'.format(key, value) + for key, value in self._data.items()))) + + def copy(self): + return dict(self) + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") + +def _createenviron(): + return _Environ(environ,_putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + + +def getenv(key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default. + key, default and the result are str.""" + try: + return environ[key] + except KeyError: + return default diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ba6033ff8..15fa176d8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -40,3 +40,17 @@ assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) assert os.O_RDONLY == 0 assert os.O_WRONLY == 1 assert os.O_RDWR == 2 + +ENV_KEY = "TEST_ENV_VAR" +ENV_VALUE = "value" + +assert os.getenv(ENV_KEY) == None +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY, 5) == 5 +os.environ[ENV_KEY] = ENV_VALUE +assert ENV_KEY in os.environ +assert os.getenv(ENV_KEY) == ENV_VALUE +del os.environ[ENV_KEY] +os.unsetenv(ENV_KEY) +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY) == None diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index a69a80d11..9b0d0b345 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -50,7 +50,7 @@ pub fn get_module_inits() -> HashMap { #[cfg(not(target_arch = "wasm32"))] { modules.insert("io".to_string(), Box::new(io::make_module)); - modules.insert("os".to_string(), Box::new(os::make_module)); + modules.insert("_os".to_string(), Box::new(os::make_module)); modules.insert("socket".to_string(), Box::new(socket::make_module)); } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 66698b3cb..26707d3ca 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -201,7 +201,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let environ = _os_environ(vm); - py_module!(vm, "os", { + py_module!(vm, "_os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), From 69e49e6cba58a43c6cb230ae8be2b733d46c9223 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 18:44:09 +0300 Subject: [PATCH 184/223] Simplify os.py --- Lib/os.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index badb08ebb..8b7876c7a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -39,23 +39,7 @@ class _Environ(): self[key] = value return self[key] -try: - _putenv = putenv -except NameError: - _putenv = lambda key, value: None - -try: - _unsetenv = unsetenv -except NameError: - _unsetenv = lambda key: _putenv(key, "") - -def _createenviron(): - return _Environ(environ,_putenv, _unsetenv) - -# unicode environ -environ = _createenviron() -del _createenviron - +environ = _Environ(environ, putenv, unsetenv) def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. From eb40e08d8a59c0e43b00f8c2206573bc7f8ad83f Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:02:04 +0300 Subject: [PATCH 185/223] Add _collections_abc.py from CPython --- Lib/_collections_abc.py | 1013 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1013 insertions(+) create mode 100644 Lib/_collections_abc.py diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py new file mode 100644 index 000000000..c36398797 --- /dev/null +++ b/Lib/_collections_abc.py @@ -0,0 +1,1013 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +from abc import ABCMeta, abstractmethod +import sys + +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", + ] + +# This module has been renamed from collections.abc to _collections_abc to +# speed up interpreter startup. Some of the types such as MutableMapping are +# required early but collections module imports a lot of other modules. +# See issue #19218 +__name__ = "collections.abc" + +# Private list of types that we want to register with the various ABCs +# so that they will pass tests like: +# it = iter(somebytearray) +# assert isinstance(it, Iterable) +# Note: in other implementations, these types might not be distinct +# and they may have their own implementation specific types that +# are not included on this list. +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +longrange_iterator = type(iter(range(1 << 1000))) +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +mappingproxy = type(type.__dict__) +generator = type((lambda: (yield))()) +## coroutine ## +async def _coro(): pass +_coro = _coro() +coroutine = type(_coro) +_coro.close() # Prevent ResourceWarning +del _coro +## asynchronous generator ## +async def _ag(): yield +_ag = _ag() +async_generator = type(_ag) +del _ag + + +### ONE-TRICK PONIES ### + +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + return _check_methods(C, "__hash__") + return NotImplemented + + +class Awaitable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __await__(self): + yield + + @classmethod + def __subclasshook__(cls, C): + if cls is Awaitable: + return _check_methods(C, "__await__") + return NotImplemented + + +class Coroutine(Awaitable): + + __slots__ = () + + @abstractmethod + def send(self, value): + """Send a value into the coroutine. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the coroutine. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside coroutine. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("coroutine ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Coroutine: + return _check_methods(C, '__await__', 'send', 'throw', 'close') + return NotImplemented + + +Coroutine.register(coroutine) + + +class AsyncIterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __aiter__(self): + return AsyncIterator() + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterable: + return _check_methods(C, "__aiter__") + return NotImplemented + + +class AsyncIterator(AsyncIterable): + + __slots__ = () + + @abstractmethod + async def __anext__(self): + """Return the next item or raise StopAsyncIteration when exhausted.""" + raise StopAsyncIteration + + def __aiter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterator: + return _check_methods(C, "__anext__", "__aiter__") + return NotImplemented + + +class AsyncGenerator(AsyncIterator): + + __slots__ = () + + async def __anext__(self): + """Return the next item from the asynchronous generator. + When exhausted, raise StopAsyncIteration. + """ + return await self.asend(None) + + @abstractmethod + async def asend(self, value): + """Send a value into the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + raise StopAsyncIteration + + @abstractmethod + async def athrow(self, typ, val=None, tb=None): + """Raise an exception in the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + async def aclose(self): + """Raise GeneratorExit inside coroutine. + """ + try: + await self.athrow(GeneratorExit) + except (GeneratorExit, StopAsyncIteration): + pass + else: + raise RuntimeError("asynchronous generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncGenerator: + return _check_methods(C, '__aiter__', '__anext__', + 'asend', 'athrow', 'aclose') + return NotImplemented + + +AsyncGenerator.register(async_generator) + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + return _check_methods(C, "__iter__") + return NotImplemented + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + 'Return the next item from the iterator. When exhausted, raise StopIteration' + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + return _check_methods(C, '__iter__', '__next__') + return NotImplemented + +Iterator.register(bytes_iterator) +Iterator.register(bytearray_iterator) +#Iterator.register(callable_iterator) +Iterator.register(dict_keyiterator) +Iterator.register(dict_valueiterator) +Iterator.register(dict_itemiterator) +Iterator.register(list_iterator) +Iterator.register(list_reverseiterator) +Iterator.register(range_iterator) +Iterator.register(longrange_iterator) +Iterator.register(set_iterator) +Iterator.register(str_iterator) +Iterator.register(tuple_iterator) +Iterator.register(zip_iterator) + + +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + +class Generator(Iterator): + + __slots__ = () + + def __next__(self): + """Return the next item from the generator. + When exhausted, raise StopIteration. + """ + return self.send(None) + + @abstractmethod + def send(self, value): + """Send a value into the generator. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the generator. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside generator. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Generator: + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') + return NotImplemented + +Generator.register(generator) + + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + return _check_methods(C, "__len__") + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + return _check_methods(C, "__contains__") + return NotImplemented + +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + return _check_methods(C, "__call__") + return NotImplemented + + +### SETS ### + + +class Set(Collection): + + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), redefine __le__ and __ge__, + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) < len(other): + return False + for elem in other: + if elem not in self: + return False + return True + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + __rand__ = __and__ + + def isdisjoint(self, other): + 'Return True if two sets have a null intersection.' + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + __ror__ = __or__ + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __rsub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in other + if value not in self) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + __rxor__ = __xor__ + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + """A mutable set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__, __len__, + add(), and discard(). + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Collection): + + __slots__ = () + + """A Mapping is a generic container for associating key/value + pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __iter__, and __len__. + + """ + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return KeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return ItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + +Mapping.register(mappingproxy) + + +class MappingView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + +ItemsView.register(dict_items) + + +class ValuesView(MappingView, Collection): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + + __slots__ = () + + """A MutableMapping is a generic container for associating + key/value pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __setitem__, __delitem__, + __iter__, and __len__. + + """ + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + '''D.popitem() -> (k, v), remove and return some (key, value) pair + as a 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + 'D.clear() -> None. Remove all items from D.' + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. + If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v + In either case, this is followed by: for k, v in F.items(): D[k] = v + ''' + if not args: + raise TypeError("descriptor 'update' of 'MutableMapping' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Reversible, Collection): + + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + '''S.index(value, [start, [stop]]) -> integer -- return first index of value. + Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. + ''' + if start is not None and start < 0: + start = max(len(self) + start, 0) + if stop is not None and stop < 0: + stop += len(self) + + i = start + while stop is None or i < stop: + try: + v = self[i] + if v is value or v == value: + return i + except IndexError: + break + i += 1 + raise ValueError + + def count(self, value): + 'S.count(value) -> integer -- return number of occurrences of value' + return sum(1 for v in self if v is value or v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +Sequence.register(memoryview) + + +class ByteString(Sequence): + + """This unifies bytes and bytearray. + + XXX Should add all their methods. + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + + __slots__ = () + + """All the operations on a read-write sequence. + + Concrete subclasses must provide __new__ or __init__, + __getitem__, __setitem__, __delitem__, __len__, and insert(). + + """ + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + 'S.insert(index, value) -- insert value before index' + raise IndexError + + def append(self, value): + 'S.append(value) -- append value to the end of the sequence' + self.insert(len(self), value) + + def clear(self): + 'S.clear() -> None -- remove all items from S' + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + 'S.reverse() -- reverse *IN PLACE*' + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + '''S.pop([index]) -> item -- remove and return item at index (default last). + Raise IndexError if list is empty or index is out of range. + ''' + v = self[index] + del self[index] + return v + + def remove(self, value): + '''S.remove(value) -- remove first occurrence of value. + Raise ValueError if the value is not present. + ''' + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString From 127282979364ec780e164fa114a9544ce14a3386 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:03:08 +0300 Subject: [PATCH 186/223] Comment unsupported code from _collections_abc.py --- Lib/_collections_abc.py | 274 ++++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c36398797..3158a4f16 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -40,7 +40,7 @@ dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) list_iterator = type(iter([])) -list_reverseiterator = type(iter(reversed([]))) +# list_reverseiterator = type(iter(reversed([]))) range_iterator = type(iter(range(0))) longrange_iterator = type(iter(range(1 << 1000))) set_iterator = type(iter(set())) @@ -53,21 +53,21 @@ dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) -generator = type((lambda: (yield))()) +# generator = type((lambda: (yield))()) ## coroutine ## -async def _coro(): pass -_coro = _coro() -coroutine = type(_coro) -_coro.close() # Prevent ResourceWarning -del _coro -## asynchronous generator ## -async def _ag(): yield -_ag = _ag() -async_generator = type(_ag) -del _ag +# async def _coro(): pass +# _coro = _coro() +# coroutine = type(_coro) +# _coro.close() # Prevent ResourceWarning +# del _coro +# ## asynchronous generator ## +# async def _ag(): yield +# _ag = _ag() +# async_generator = type(_ag) +# del _ag -### ONE-TRICK PONIES ### +# ## ONE-TRICK PONIES ### def _check_methods(C, *methods): mro = C.__mro__ @@ -96,148 +96,148 @@ class Hashable(metaclass=ABCMeta): return NotImplemented -class Awaitable(metaclass=ABCMeta): +# class Awaitable(metaclass=ABCMeta): - __slots__ = () +# __slots__ = () - @abstractmethod - def __await__(self): - yield +# @abstractmethod +# def __await__(self): +# yield - @classmethod - def __subclasshook__(cls, C): - if cls is Awaitable: - return _check_methods(C, "__await__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Awaitable: +# return _check_methods(C, "__await__") +# return NotImplemented -class Coroutine(Awaitable): +# class Coroutine(Awaitable): - __slots__ = () +# __slots__ = () - @abstractmethod - def send(self, value): - """Send a value into the coroutine. - Return next yielded value or raise StopIteration. - """ - raise StopIteration +# @abstractmethod +# def send(self, value): +# """Send a value into the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# raise StopIteration - @abstractmethod - def throw(self, typ, val=None, tb=None): - """Raise an exception in the coroutine. - Return next yielded value or raise StopIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val +# @abstractmethod +# def throw(self, typ, val=None, tb=None): +# """Raise an exception in the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val - def close(self): - """Raise GeneratorExit inside coroutine. - """ - try: - self.throw(GeneratorExit) - except (GeneratorExit, StopIteration): - pass - else: - raise RuntimeError("coroutine ignored GeneratorExit") +# def close(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# self.throw(GeneratorExit) +# except (GeneratorExit, StopIteration): +# pass +# else: +# raise RuntimeError("coroutine ignored GeneratorExit") - @classmethod - def __subclasshook__(cls, C): - if cls is Coroutine: - return _check_methods(C, '__await__', 'send', 'throw', 'close') - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Coroutine: +# return _check_methods(C, '__await__', 'send', 'throw', 'close') +# return NotImplemented -Coroutine.register(coroutine) +# Coroutine.register(coroutine) -class AsyncIterable(metaclass=ABCMeta): +# class AsyncIterable(metaclass=ABCMeta): - __slots__ = () +# __slots__ = () - @abstractmethod - def __aiter__(self): - return AsyncIterator() +# @abstractmethod +# def __aiter__(self): +# return AsyncIterator() - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterable: - return _check_methods(C, "__aiter__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterable: +# return _check_methods(C, "__aiter__") +# return NotImplemented -class AsyncIterator(AsyncIterable): +# class AsyncIterator(AsyncIterable): - __slots__ = () +# __slots__ = () - @abstractmethod - async def __anext__(self): - """Return the next item or raise StopAsyncIteration when exhausted.""" - raise StopAsyncIteration +# @abstractmethod +# async def __anext__(self): +# """Return the next item or raise StopAsyncIteration when exhausted.""" +# raise StopAsyncIteration - def __aiter__(self): - return self +# def __aiter__(self): +# return self - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterator: - return _check_methods(C, "__anext__", "__aiter__") - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterator: +# return _check_methods(C, "__anext__", "__aiter__") +# return NotImplemented -class AsyncGenerator(AsyncIterator): +# class AsyncGenerator(AsyncIterator): - __slots__ = () +# __slots__ = () - async def __anext__(self): - """Return the next item from the asynchronous generator. - When exhausted, raise StopAsyncIteration. - """ - return await self.asend(None) +# async def __anext__(self): +# """Return the next item from the asynchronous generator. +# When exhausted, raise StopAsyncIteration. +# """ +# return await self.asend(None) - @abstractmethod - async def asend(self, value): - """Send a value into the asynchronous generator. - Return next yielded value or raise StopAsyncIteration. - """ - raise StopAsyncIteration +# @abstractmethod +# async def asend(self, value): +# """Send a value into the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# raise StopAsyncIteration - @abstractmethod - async def athrow(self, typ, val=None, tb=None): - """Raise an exception in the asynchronous generator. - Return next yielded value or raise StopAsyncIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val +# @abstractmethod +# async def athrow(self, typ, val=None, tb=None): +# """Raise an exception in the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val - async def aclose(self): - """Raise GeneratorExit inside coroutine. - """ - try: - await self.athrow(GeneratorExit) - except (GeneratorExit, StopAsyncIteration): - pass - else: - raise RuntimeError("asynchronous generator ignored GeneratorExit") +# async def aclose(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# await self.athrow(GeneratorExit) +# except (GeneratorExit, StopAsyncIteration): +# pass +# else: +# raise RuntimeError("asynchronous generator ignored GeneratorExit") - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncGenerator: - return _check_methods(C, '__aiter__', '__anext__', - 'asend', 'athrow', 'aclose') - return NotImplemented +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncGenerator: +# return _check_methods(C, '__aiter__', '__anext__', +# 'asend', 'athrow', 'aclose') +# return NotImplemented -AsyncGenerator.register(async_generator) +# AsyncGenerator.register(async_generator) class Iterable(metaclass=ABCMeta): @@ -274,20 +274,20 @@ class Iterator(Iterable): return _check_methods(C, '__iter__', '__next__') return NotImplemented -Iterator.register(bytes_iterator) -Iterator.register(bytearray_iterator) -#Iterator.register(callable_iterator) -Iterator.register(dict_keyiterator) -Iterator.register(dict_valueiterator) -Iterator.register(dict_itemiterator) -Iterator.register(list_iterator) -Iterator.register(list_reverseiterator) -Iterator.register(range_iterator) -Iterator.register(longrange_iterator) -Iterator.register(set_iterator) -Iterator.register(str_iterator) -Iterator.register(tuple_iterator) -Iterator.register(zip_iterator) +# Iterator.register(bytes_iterator) +# Iterator.register(bytearray_iterator) +# Iterator.register(callable_iterator) +# Iterator.register(dict_keyiterator) +# Iterator.register(dict_valueiterator) +# Iterator.register(dict_itemiterator) +# Iterator.register(list_iterator) +# Iterator.register(list_reverseiterator) +# Iterator.register(range_iterator) +# Iterator.register(longrange_iterator) +# Iterator.register(set_iterator) +# Iterator.register(str_iterator) +# Iterator.register(tuple_iterator) +# Iterator.register(zip_iterator) class Reversible(Iterable): @@ -353,7 +353,7 @@ class Generator(Iterator): 'send', 'throw', 'close') return NotImplemented -Generator.register(generator) +# Generator.register(generator) class Sized(metaclass=ABCMeta): From b293956258f45bdade3b97ae73df423d82678e9d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Thu, 11 Apr 2019 19:15:37 +0300 Subject: [PATCH 187/223] Use original _Environ as much as possible --- Lib/os.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8b7876c7a..8c19a89c6 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,34 +1,56 @@ from _os import * -class _Environ(): - def __init__(self, data, putenv, unsetenv): +# Change environ to automatically call putenv(), unsetenv if they exist. +from _collections_abc import MutableMapping + +class _Environ(MutableMapping): + def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): + self.encodekey = encodekey + self.decodekey = decodekey + self.encodevalue = encodevalue + self.decodevalue = decodevalue self.putenv = putenv self.unsetenv = unsetenv self._data = data def __getitem__(self, key): - return self._data[key] + try: + value = self._data[self.encodekey(key)] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) + + return self.decodevalue(value) def __setitem__(self, key, value): + key = self.encodekey(key) + value = self.encodevalue(value) self.putenv(key, value) self._data[key] = value def __delitem__(self, key): - self.unsetenv(key) - del self._data[key] + encodedkey = self.encodekey(key) + self.unsetenv(encodedkey) + try: + del self._data[encodedkey] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) def __iter__(self): # list() from dict object is an atomic operation keys = list(self._data) for key in keys: - yield key + yield self.decodekey(key) def __len__(self): return len(self._data) def __repr__(self): return 'environ({{{}}})'.format(', '.join( - ('{}: {}'.format(key, value) + ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) for key, value in self._data.items()))) def copy(self): @@ -39,13 +61,62 @@ class _Environ(): self[key] = value return self[key] -environ = _Environ(environ, putenv, unsetenv) +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None +# else: +# if "putenv" not in __all__: +# __all__.append("putenv") + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") +# else: +# if "unsetenv" not in __all__: +# __all__.append("unsetenv") + +def _createenviron(): + # if name == 'nt': + # # Where Env Var Names Must Be UPPERCASE + # def check_str(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value + # encode = check_str + # decode = str + # def encodekey(key): + # return encode(key).upper() + # data = {} + # for key, value in environ.items(): + # data[encodekey(key)] = value + # else: + # # Where Env Var Names Can Be Mixed Case + # encoding = sys.getfilesystemencoding() + # def encode(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value.encode(encoding, 'surrogateescape') + # def decode(value): + # return value.decode(encoding, 'surrogateescape') + # encodekey = encode + decode = str + encode = str + encodekey = encode + data = environ + return _Environ(data, + encodekey, decode, + encode, decode, + _putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. The optional second argument can specify an alternate default. key, default and the result are str.""" - try: - return environ[key] - except KeyError: - return default + return environ.get(key, default) From 28cf4c001043490a14cc668efd2d5d8538b27430 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Thu, 11 Apr 2019 22:18:29 +0200 Subject: [PATCH 188/223] Symbol role scan fully covered. --- vm/src/compile.rs | 68 +++++++-- vm/src/symboltable.rs | 333 ++++++++++++++++++++++++++++++------------ 2 files changed, 298 insertions(+), 103 deletions(-) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index cc52661a4..2221fde7d 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -10,13 +10,14 @@ use crate::error::CompileError; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; -use crate::symboltable::SymbolTable; +use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope}; use crate::VirtualMachine; use num_complex::Complex64; use rustpython_parser::{ast, parser}; struct Compiler { code_object_stack: Vec, + scope_stack: Vec, nxt_label: usize, source_path: Option, current_source_location: ast::Location, @@ -38,17 +39,18 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - let mut symbol_table = SymbolTable::new(); - symbol_table.scan_program(&ast); - compiler.compile_program(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_table) } Mode::Eval => { let statement = parser::parse_statement(source).map_err(CompileError::Parse)?; - compiler.compile_statement_eval(&statement) + let symbol_table = statements_to_symbol_table(&statement); + compiler.compile_statement_eval(&statement, symbol_table) } Mode::Single => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program_single(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program_single(&ast, symbol_table) } }?; @@ -75,6 +77,7 @@ impl Compiler { fn new() -> Self { Compiler { code_object_stack: Vec::new(), + scope_stack: Vec::new(), nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), @@ -97,11 +100,17 @@ impl Compiler { } fn pop_code_object(&mut self) -> CodeObject { + // self.scope_stack.pop().unwrap(); self.code_object_stack.pop().unwrap() } - fn compile_program(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { let size_before = self.code_object_stack.len(); + self.scope_stack.push(symbol_scope); self.compile_statements(&program.statements)?; assert!(self.code_object_stack.len() == size_before); @@ -113,7 +122,12 @@ impl Compiler { Ok(()) } - fn compile_program_single(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program_single( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_scope); for statement in &program.statements { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; @@ -133,7 +147,9 @@ impl Compiler { fn compile_statement_eval( &mut self, statements: &[ast::LocatedStatement], + symbol_table: SymbolScope, ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_table); for statement in statements { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; @@ -464,6 +480,7 @@ impl Compiler { line_number, name.to_string(), )); + self.enter_scope(); let mut flags = bytecode::FunctionOpArg::empty(); if have_defaults { @@ -624,6 +641,7 @@ impl Compiler { }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); // Prepare type annotations: let mut num_annotations = 0; @@ -708,6 +726,7 @@ impl Compiler { line_number, name.to_string(), )); + self.enter_scope(); let (new_body, doc_str) = get_doc(body); @@ -718,6 +737,7 @@ impl Compiler { self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { @@ -1212,6 +1232,8 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { + self.lookup_name(name); + // TODO: if global, do something else! self.emit(Instruction::LoadName { name: name.to_string(), }); @@ -1223,6 +1245,7 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1569,6 +1592,30 @@ impl Compiler { Ok(()) } + // Scope helpers: + fn enter_scope(&mut self) { + // println!("Enter scope {:?}", self.scope_stack); + // Enter first subscope! + let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0); + self.scope_stack.push(scope); + } + + fn leave_scope(&mut self) { + // println!("Leave scope {:?}", self.scope_stack); + let scope = self.scope_stack.pop().unwrap(); + assert!(scope.sub_scopes.is_empty()); + } + + fn lookup_name(&self, name: &str) -> &SymbolRole { + // println!("Looking up {:?}", name); + for scope in self.scope_stack.iter().rev() { + if let Some(role) = scope.lookup(name) { + return role; + } + } + unreachable!(); + } + // Low level helper functions: fn emit(&mut self, instruction: Instruction) { let location = self.current_source_location.clone(); @@ -1624,13 +1671,13 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option< (body, None) } - #[cfg(test)] mod tests { use super::Compiler; use crate::bytecode::CodeObject; use crate::bytecode::Constant::*; use crate::bytecode::Instruction::*; + use crate::symboltable::make_symbol_table; use rustpython_parser::parser; fn compile_exec(source: &str) -> CodeObject { @@ -1638,7 +1685,8 @@ mod tests { compiler.source_path = Some("source_path".to_string()); compiler.push_new_code_object("".to_string()); let ast = parser::parse_program(&source.to_string()).unwrap(); - compiler.compile_program(&ast).unwrap(); + let symbol_scope = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() } diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 6525e71e1..78655da53 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -8,6 +8,23 @@ load and store instructions for names. use rustpython_parser::ast; use std::collections::HashMap; +pub fn make_symbol_table(program: &ast::Program) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_program(program).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +pub fn statements_to_symbol_table(statements: &[ast::LocatedStatement]) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_statements(statements).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +#[derive(Debug)] pub enum SymbolRole { Global, Nonlocal, @@ -15,42 +32,96 @@ pub enum SymbolRole { Assigned, } -pub struct SymbolTable { - // TODO: split-up into nested scopes. - symbols: HashMap, +/// Symbolscope captures all symbols in the current scope, and +/// has a list of subscopes in this scope. +pub struct SymbolScope { + /// A set of symbols present on this scope level. + pub symbols: HashMap, + + /// A list of subscopes in the order as found in the + /// AST nodes. + pub sub_scopes: Vec, } -impl SymbolTable { +type SymbolTableResult = Result<(), String>; + +impl SymbolScope { pub fn new() -> Self { - SymbolTable { + SymbolScope { symbols: HashMap::new(), + sub_scopes: vec![], } } pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { - return self.symbols.get(name); + self.symbols.get(name) + } +} + +impl std::fmt::Debug for SymbolScope { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "SymbolScope({:?} symbols, {:?} sub scopes)", + self.symbols.len(), + self.sub_scopes.len() + ) + } +} + +pub struct SymbolTableBuilder { + // Scope stack. + pub scopes: Vec, +} + +impl SymbolTableBuilder { + pub fn new() -> Self { + SymbolTableBuilder { scopes: vec![] } } - pub fn scan_program(&mut self, program: &ast::Program) { - self.scan_statements(&program.statements); + pub fn enter_scope(&mut self) { + // Create new scope and push into scope stack. + let scope = SymbolScope::new(); + self.scopes.push(scope); } - pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) { + pub fn leave_scope(&mut self) { + // Pop scope and add to subscopes of parent scope. + let scope = self.scopes.pop().unwrap(); + self.scopes.last_mut().unwrap().sub_scopes.push(scope); + } + + pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { + self.scan_statements(&program.statements)?; + Ok(()) + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult { for statement in statements { - self.scan_statement(statement) + self.scan_statement(statement)?; } + Ok(()) } - fn scan_statement(&mut self, statement: &ast::LocatedStatement) { + fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; + } + } + Ok(()) + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult { match &statement.node { ast::Statement::Global { names } => { for name in names { - self.register_name(name, SymbolRole::Global); + self.register_name(name, SymbolRole::Global)?; } } ast::Statement::Nonlocal { names } => { for name in names { - self.register_name(name, SymbolRole::Nonlocal); + self.register_name(name, SymbolRole::Nonlocal)?; } } ast::Statement::FunctionDef { @@ -60,22 +131,25 @@ impl SymbolTable { decorator_list, returns, } => { - self.scan_expressions(decorator_list); - self.register_name(name, SymbolRole::Assigned); - for parameter in &args.args {} - for parameter in &args.kwonlyargs {} + self.scan_expressions(decorator_list)?; + self.register_name(name, SymbolRole::Assigned)?; - self.scan_expressions(&args.defaults); + self.enter_scope(); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + + self.scan_expressions(&args.defaults)?; for kw_default in &args.kw_defaults { if let Some(expression) = kw_default { - self.scan_expression(&expression); + self.scan_expression(&expression)?; } } - self.scan_statements(body); + self.scan_statements(body)?; if let Some(expression) = returns { - self.scan_expression(expression); + self.scan_expression(expression)?; } + self.leave_scope(); } ast::Statement::ClassDef { name, @@ -84,20 +158,22 @@ impl SymbolTable { keywords, decorator_list, } => { - self.register_name(name, SymbolRole::Assigned); - self.scan_statements(body); - self.scan_expressions(bases); + self.register_name(name, SymbolRole::Assigned)?; + self.enter_scope(); + self.scan_statements(body)?; + self.leave_scope(); + self.scan_expressions(bases)?; for keyword in keywords { - self.scan_expression(&keyword.value); + self.scan_expression(&keyword.value)?; } - self.scan_expressions(decorator_list); + self.scan_expressions(decorator_list)?; } - ast::Statement::Expression { expression } => self.scan_expression(expression), + ast::Statement::Expression { expression } => self.scan_expression(expression)?, ast::Statement::If { test, body, orelse } => { - self.scan_expression(test); - self.scan_statements(body); + self.scan_expression(test)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::For { @@ -106,58 +182,66 @@ impl SymbolTable { body, orelse, } => { - self.scan_expression(target); - self.scan_expression(iter); - self.scan_statements(body); + self.scan_expression(target)?; + self.scan_expression(iter)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::While { test, body, orelse } => { - self.scan_expression(test); - self.scan_statements(body); + self.scan_expression(test)?; + self.scan_statements(body)?; if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { // No symbols here. } - ast::Statement::Import { import_parts } => for part in import_parts {}, + ast::Statement::Import { import_parts } => { + for part in import_parts { + if let Some(alias) = &part.alias { + self.register_name(alias, SymbolRole::Assigned)?; + } else { + if let Some(symbol) = &part.symbol { + self.register_name(symbol, SymbolRole::Assigned)?; + } else { + self.register_name(&part.module, SymbolRole::Assigned)?; + } + } + } + } ast::Statement::Return { value } => { if let Some(expression) = value { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Statement::Assert { test, msg } => { - self.scan_expression(test); + self.scan_expression(test)?; if let Some(expression) = msg { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Statement::Delete { targets } => { - self.scan_expressions(targets); + self.scan_expressions(targets)?; } ast::Statement::Assign { targets, value } => { - self.scan_expressions(targets); - self.scan_expression(value); + self.scan_expressions(targets)?; + self.scan_expression(value)?; } - ast::Statement::AugAssign { - target, - op: _, - value, - } => { - self.scan_expression(target); - self.scan_expression(value); + ast::Statement::AugAssign { target, value, .. } => { + self.scan_expression(target)?; + self.scan_expression(value)?; } ast::Statement::With { items, body } => { for item in items { - self.scan_expression(&item.context_expr); + self.scan_expression(&item.context_expr)?; if let Some(expression) = &item.optional_vars { - self.scan_expression(expression); + self.scan_expression(expression)?; } } - self.scan_statements(body); + self.scan_statements(body)?; } ast::Statement::Try { body, @@ -165,67 +249,78 @@ impl SymbolTable { orelse, finalbody, } => { - self.scan_statements(body); + self.scan_statements(body)?; + for handler in handlers { + if let Some(expression) = &handler.typ { + self.scan_expression(expression)?; + } + if let Some(name) = &handler.name { + self.register_name(name, SymbolRole::Assigned)?; + } + self.scan_statements(&handler.body)?; + } if let Some(code) = orelse { - self.scan_statements(code); + self.scan_statements(code)?; } if let Some(code) = finalbody { - self.scan_statements(code); + self.scan_statements(code)?; } } ast::Statement::Raise { exception, cause } => { if let Some(expression) = exception { - self.scan_expression(expression); + self.scan_expression(expression)?; } if let Some(expression) = cause { - self.scan_expression(expression); + self.scan_expression(expression)?; } } } + Ok(()) } - fn scan_expressions(&mut self, expressions: &[ast::Expression]) { + fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult { for expression in expressions { - self.scan_expression(expression); + self.scan_expression(expression)?; } + Ok(()) } - fn scan_expression(&mut self, expression: &ast::Expression) { + fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { match expression { - ast::Expression::Binop { a, op: _, b } => { - self.scan_expression(a); - self.scan_expression(b); + ast::Expression::Binop { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::BoolOp { a, op: _, b } => { - self.scan_expression(a); - self.scan_expression(b); + ast::Expression::BoolOp { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::Compare { vals, ops: _ } => { - self.scan_expressions(vals); + ast::Expression::Compare { vals, .. } => { + self.scan_expressions(vals)?; } ast::Expression::Subscript { a, b } => { - self.scan_expression(a); - self.scan_expression(b); + self.scan_expression(a)?; + self.scan_expression(b)?; } - ast::Expression::Attribute { value, name: _ } => { - self.scan_expression(value); + ast::Expression::Attribute { value, .. } => { + self.scan_expression(value)?; } ast::Expression::Dict { elements } => { for (key, value) in elements { - self.scan_expression(key); - self.scan_expression(value); + self.scan_expression(key)?; + self.scan_expression(value)?; } } ast::Expression::Yield { value } => { if let Some(expression) = value { - self.scan_expression(expression); + self.scan_expression(expression)?; } } ast::Expression::YieldFrom { value } => { - self.scan_expression(value); + self.scan_expression(value)?; } - ast::Expression::Unop { op: _, a } => { - self.scan_expression(a); + ast::Expression::Unop { a, .. } => { + self.scan_expression(a)?; } ast::Expression::True | ast::Expression::False @@ -233,40 +328,92 @@ impl SymbolTable { | ast::Expression::Ellipsis => {} ast::Expression::Number { .. } => {} ast::Expression::Starred { value } => { - self.scan_expression(value); + self.scan_expression(value)?; } ast::Expression::Bytes { .. } => {} ast::Expression::Tuple { elements } | ast::Expression::Set { elements } | ast::Expression::List { elements } | ast::Expression::Slice { elements } => { - self.scan_expressions(elements); + self.scan_expressions(elements)?; + } + ast::Expression::Comprehension { kind, generators } => { + match **kind { + ast::ComprehensionKind::GeneratorExpression { ref element } + | ast::ComprehensionKind::List { ref element } + | ast::ComprehensionKind::Set { ref element } => { + self.scan_expression(element)?; + } + ast::ComprehensionKind::Dict { ref key, ref value } => { + self.scan_expression(&key)?; + self.scan_expression(&value)?; + } + } + + for generator in generators { + self.scan_expression(&generator.target)?; + self.scan_expression(&generator.iter)?; + for if_expr in &generator.ifs { + self.scan_expression(if_expr)?; + } + } } - ast::Expression::Comprehension { kind, generators } => {} ast::Expression::Call { function, args, keywords, } => { - self.scan_expression(function); - self.scan_expressions(args); + self.scan_expression(function)?; + self.scan_expressions(args)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + } + ast::Expression::String { value } => { + self.scan_string_group(value)?; } - ast::Expression::String { value } => {} ast::Expression::Identifier { name } => { - self.register_name(name, SymbolRole::Used); + self.register_name(name, SymbolRole::Used)?; } ast::Expression::Lambda { args, body } => { - self.scan_expression(body); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + self.enter_scope(); + self.scan_expression(body)?; + self.leave_scope(); } ast::Expression::IfExpression { test, body, orelse } => { - self.scan_expression(test); - self.scan_expression(body); - self.scan_expression(orelse); + self.scan_expression(test)?; + self.scan_expression(body)?; + self.scan_expression(orelse)?; } } + Ok(()) } - fn register_name(&mut self, name: &String, role: SymbolRole) { - self.symbols.insert(name.clone(), role); + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { + match group { + ast::StringGroup::Constant { .. } => {} + ast::StringGroup::FormattedValue { value, .. } => { + self.scan_expression(value)?; + } + ast::StringGroup::Joined { values } => { + for subgroup in values { + self.scan_string_group(subgroup)?; + } + } + } + Ok(()) + } + + fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { + let current_scope = self.scopes.last_mut().unwrap(); + if let Some(old_role) = current_scope.symbols.get(name) { + debug!("TODO: {:?}", old_role); + // Role already set.. + } else { + current_scope.symbols.insert(name.to_string(), role); + } + Ok(()) } } From 6c4b09264175e93f9334ab3048fe3d8ea8ce2da6 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 10:50:45 +0300 Subject: [PATCH 189/223] Support exception __cause__ --- tests/snippets/try_exceptions.py | 32 +++++++++++++++++++ vm/src/frame.rs | 54 +++++++++++++++++++------------- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 4e3b7269a..5d7509ba5 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -95,3 +95,35 @@ except TypeError as ex: l.append(3) print('boom', type(ex)) assert l == [1, 3] + +cause = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__cause__ == None + cause = ex + raise NameError from ex +except NameError as ex2: + assert ex2.__cause__ == cause + +try: + raise ZeroDivisionError from None +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError from 5 +except TypeError: + pass + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..a3dfd6032 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -677,31 +677,17 @@ impl Frame { } bytecode::Instruction::Raise { argc } => { + let cause = match argc { + 2 => self.get_exception(vm, true)?, + _ => vm.get_none(), + }; let exception = match argc { - 1 => self.pop_value(), - 0 | 2 | 3 => panic!("Not implemented!"), + 1 | 2 => self.get_exception(vm, false)?, + 0 | 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { - info!("Exception raised: {:?}", exception); - Err(exception) - } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { - if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { - let exception = vm.new_empty_exception(exception)?; - info!("Exception raised: {:?}", exception); - Err(exception) - } else { - let msg = format!( - "Can only raise BaseException derived types, not {}", - exception - ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) - } - } else { - Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) - } + vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + Err(exception) } bytecode::Instruction::Break => { @@ -1204,6 +1190,30 @@ impl Frame { let stack = self.stack.borrow_mut(); stack[stack.len() - depth - 1].clone() } + + fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { + let exception = self.pop_value(); + if none_allowed && vm.get_none().is(&exception) { + Ok(exception) + } else if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { + Ok(exception) + } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exception)?; + Ok(exception) + } else { + let msg = format!( + "Can only raise BaseException derived types, not {}", + exception + ); + let type_error_type = vm.ctx.exceptions.type_error.clone(); + let type_error = vm.new_exception(type_error_type, msg); + Err(type_error) + } + } else { + Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) + } + } } impl fmt::Debug for Frame { From 63643e1fdba7a621102553597aae8f20bbda5b6a Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 12 Apr 2019 20:30:37 +1200 Subject: [PATCH 190/223] Implement generator.throw --- tests/snippets/generators.py | 22 +++++++++++++ vm/src/frame.rs | 11 +++++++ vm/src/obj/objgenerator.rs | 64 ++++++++++++++++++++++++------------ vm/src/vm.rs | 15 +++++++-- 4 files changed, 89 insertions(+), 23 deletions(-) diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 65044c0da..3b7dc77c3 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + r = [] @@ -42,3 +44,23 @@ def g4(): r = list(g4()) assert r == [None, (2,)] + +def catch_exception(): + try: + yield 1 + except ValueError: + yield 2 + yield 3 + + +g = catch_exception() +assert next(g) == 1 + +assert g.throw(ValueError, ValueError(), None) == 2 +assert next(g) == 3 + +g = catch_exception() +assert next(g) == 1 + +with assertRaises(KeyError): + assert g.throw(KeyError, KeyError(), None) == 2 diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ff9baba35..17322f21f 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -283,6 +283,17 @@ impl Frame { } } + pub fn throw( + &self, + vm: &VirtualMachine, + exception: PyObjectRef, + ) -> Result { + match self.unwind_exception(vm, exception) { + None => self.run(vm), + Some(exception) => Err(exception), + } + } + pub fn fetch_instruction(&self) -> &bytecode::Instruction { let ins2 = &self.code.instructions[*self.lasti.borrow()]; *self.lasti.borrow_mut() += 1; diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 9718d29a4..2541263f1 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -3,12 +3,13 @@ */ use crate::frame::{ExecutionResult, FrameRef}; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objtype::{isinstance, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; +#[pyclass(name = "generator", __inside_vm)] #[derive(Debug)] pub struct PyGenerator { frame: FrameRef, @@ -20,38 +21,59 @@ impl PyValue for PyGenerator { } } -impl PyGeneratorRef { +#[pyimpl(__inside_vm)] +impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { frame }.into_ref(vm) } - fn iter(self, _vm: &VirtualMachine) -> PyGeneratorRef { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyGeneratorRef, _vm: &VirtualMachine) -> PyGeneratorRef { + zelf } - fn next(self, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { self.send(vm.get_none(), vm) } - fn send(self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.frame.push_value(value.clone()); - match vm.run_frame(self.frame.clone())? { - ExecutionResult::Yield(value) => Ok(value), - ExecutionResult::Return(_value) => { - // Stop iteration! - let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); - Err(vm.new_exception(stop_iteration, "End of generator".to_string())) - } + let result = vm.run_frame(self.frame.clone())?; + handle_execution_result(result, vm) + } + + #[pymethod] + fn throw( + &self, + _exc_type: PyObjectRef, + exc_val: PyObjectRef, + _exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // TODO what should we do with the other parameters? CPython normalises them with + // PyErr_NormalizeException, do we want to do the same. + if !isinstance(&exc_val, &vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error("Can't throw non exception".to_string())); + } + let result = vm.frame_throw(self.frame.clone(), exc_val)?; + handle_execution_result(result, vm) + } +} + +fn handle_execution_result(result: ExecutionResult, vm: &VirtualMachine) -> PyResult { + match result { + ExecutionResult::Yield(value) => Ok(value), + ExecutionResult::Return(_value) => { + // Stop iteration! + let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); + Err(vm.new_exception(stop_iteration, "End of generator".to_string())) } } } -pub fn init(context: &PyContext) { - let generator_type = &context.generator_type; - extend_class!(context, generator_type, { - "__iter__" => context.new_rustfunc(PyGeneratorRef::iter), - "__next__" => context.new_rustfunc(PyGeneratorRef::next), - "send" => context.new_rustfunc(PyGeneratorRef::send) - }); +pub fn init(ctx: &PyContext) { + PyGenerator::extend_class(ctx, &ctx.generator_type); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7be05023b..5ae5b0b50 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -21,7 +21,7 @@ use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; -use crate::obj::objgenerator::PyGeneratorRef; +use crate::obj::objgenerator::PyGenerator; use crate::obj::objiter; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; @@ -94,6 +94,17 @@ impl VirtualMachine { result } + pub fn frame_throw( + &self, + frame: FrameRef, + exception: PyObjectRef, + ) -> PyResult { + self.frames.borrow_mut().push(frame.clone()); + let result = frame.throw(self, exception); + self.frames.borrow_mut().pop(); + result + } + pub fn current_frame(&self) -> Option> { let frames = self.frames.borrow(); if frames.is_empty() { @@ -374,7 +385,7 @@ impl VirtualMachine { // If we have a generator, create a new generator if code.code.is_generator { - Ok(PyGeneratorRef::new(frame, self).into_object()) + Ok(PyGenerator::new(frame, self).into_object()) } else { self.run_frame_full(frame) } From 80ec464d84f6c5ad931fad41c7c84e64f9a82c45 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Fri, 12 Apr 2019 12:25:19 +0100 Subject: [PATCH 191/223] Use dict.update to combine class and object attrs in object_dir. --- vm/src/obj/objdict.rs | 13 +++++++++++++ vm/src/obj/objobject.rs | 27 ++++++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index be85c7b2f..efd38a7fd 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -239,6 +239,19 @@ impl PyDictRef { attrs } + pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult { + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + + for (key, value) in attrs { + entries + .borrow_mut() + .insert(vm, &vm.ctx.new_str(key), value)?; + } + + Ok(PyDict { entries }.into_ref(vm)) + } + fn hash(self, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 8f6c90a36..f02bbabd1 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,6 +1,6 @@ use super::objdict::PyDictRef; use super::objlist::PyList; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; @@ -118,26 +118,19 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { } pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut attributes: PyAttributes = objtype::get_attributes(obj.class()); + let attributes: PyAttributes = objtype::get_attributes(obj.class()); + + let dict = PyDictRef::from_attributes(attributes, vm)?; // Get instance attributes: - if let Some(dict) = &obj.dict { - for (key, value) in dict { - if let Some(key_string) = key.payload::() { - attributes.insert(key_string.to_string(), value.clone()); - } else { - return Err(vm.new_type_error(format!( - "Attribute is not a string: {:?}", - vm.to_pystr(&key)? - ))); - } - } + if let Some(object_dict) = &obj.dict { + vm.invoke( + vm.get_attribute(dict.clone().into_object(), "update")?, + object_dict.clone().into_object(), + )?; } - let attributes: Vec = attributes - .keys() - .map(|k| vm.ctx.new_str(k.to_string())) - .collect(); + let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k.clone()).collect(); Ok(PyList::from(attributes)) } From 908f758cbedcb0bddb3f5b32c99aa4855da668eb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 15:50:17 +0300 Subject: [PATCH 192/223] Print cause on exception --- vm/src/exceptions.rs | 15 +++++++++++++-- vm/src/frame.rs | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index d81764e25..abd670a14 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -2,7 +2,7 @@ use crate::function::PyFuncArgs; use crate::obj::objsequence; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{create_type, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,8 +18,19 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -// Print exception including traceback: +// print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { + if !vm.get_none().is(&cause) { + print_exception(vm, &cause); + println!("\nThe above exception was the direct cause of the following exception:\n"); + } + } + print_exception_inner(vm, exc) +} + +// Print exception including traceback: +pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a3dfd6032..a1da85d3a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -686,6 +686,7 @@ impl Frame { 0 | 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; + info!("Exception raised: {:?} with cause: {:?}", exception, cause); vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; Err(exception) } From 76c597f2afc21edc1c6a5158120b619cb88e7f49 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Fri, 12 Apr 2019 19:24:23 +0200 Subject: [PATCH 193/223] Add global bytecode and testcase. --- tests/snippets/global_nonlocal.py | 13 ++++++ vm/src/bytecode.rs | 12 ++++- vm/src/compile.rs | 73 ++++++++++++++++++------------- vm/src/frame.rs | 60 ++++++++++++++++++------- vm/src/symboltable.rs | 59 +++++++++++++++++-------- 5 files changed, 148 insertions(+), 69 deletions(-) create mode 100644 tests/snippets/global_nonlocal.py diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py new file mode 100644 index 000000000..48d124210 --- /dev/null +++ b/tests/snippets/global_nonlocal.py @@ -0,0 +1,13 @@ + +# Test global and nonlocal funkyness + +a = 2 + +def b(): + global a + a = 4 + +assert a == 2 +b() +assert a == 4 + diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index b24801861..2725aa586 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -38,6 +38,12 @@ bitflags! { pub type Label = usize; +#[derive(Debug, Clone, PartialEq)] +pub enum NameScope { + Local, + Global, +} + /// A Single bytecode instruction. #[derive(Debug, Clone, PartialEq)] pub enum Instruction { @@ -50,9 +56,11 @@ pub enum Instruction { }, LoadName { name: String, + scope: NameScope, }, StoreName { name: String, + scope: NameScope, }, DeleteName { name: String, @@ -322,8 +330,8 @@ impl Instruction { match self { Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)), ImportStar { name } => w!(ImportStar, name), - LoadName { name } => w!(LoadName, name), - StoreName { name } => w!(StoreName, name), + LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)), + StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)), DeleteName { name } => w!(DeleteName, name), StoreSubscript => w!(StoreSubscript), DeleteSubscript => w!(DeleteSubscript), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 2221fde7d..31e43c04d 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -171,6 +171,31 @@ impl Compiler { Ok(()) } + fn scope_for_name(&self, name: &str) -> bytecode::NameScope { + let role = self.lookup_name(name); + match role { + SymbolRole::Global => bytecode::NameScope::Global, + _ => bytecode::NameScope::Local, + } + } + + fn load_name(&mut self, name: &str) { + // TODO: if global, do something else! + let scope = self.scope_for_name(name); + self.emit(Instruction::LoadName { + name: name.to_string(), + scope, + }); + } + + fn store_name(&mut self, name: &str) { + let scope = self.scope_for_name(name); + self.emit(Instruction::StoreName { + name: name.to_string(), + scope, + }); + } + fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> { trace!("Compiling {:?}", statement); self.set_source_location(&statement.location); @@ -194,15 +219,14 @@ impl Compiler { name: module.clone(), symbol: symbol.clone(), }); - self.emit(Instruction::StoreName { - name: match alias { - Some(alias) => alias.clone(), - None => match symbol { - Some(symbol) => symbol.clone(), - None => module.clone(), - }, + let name = match alias { + Some(alias) => alias.clone(), + None => match symbol { + Some(symbol) => symbol.clone(), + None => module.clone(), }, - }); + }; + self.store_name(&name); } } } @@ -337,6 +361,7 @@ impl Compiler { self.compile_test(test, Some(end_label), None, EvalContext::Statement)?; self.emit(Instruction::LoadName { name: String::from("AssertionError"), + scope: bytecode::NameScope::Local, }); match msg { Some(e) => { @@ -544,6 +569,7 @@ impl Compiler { // Check exception type: self.emit(Instruction::LoadName { name: String::from("isinstance"), + scope: bytecode::NameScope::Local, }); self.emit(Instruction::Rotate { amount: 2 }); self.compile_expression(exc_type)?; @@ -558,9 +584,7 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { - self.emit(Instruction::StoreName { - name: alias.clone(), - }); + self.store_name(alias); } else { // Drop exception from top of stack: self.emit(Instruction::Pop); @@ -695,9 +719,7 @@ impl Compiler { self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; @@ -796,9 +818,7 @@ impl Compiler { self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); self.in_loop = was_in_loop; Ok(()) } @@ -943,9 +963,7 @@ impl Compiler { fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); } ast::Expression::Subscript { a, b } => { self.compile_expression(a)?; @@ -1232,11 +1250,7 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { - self.lookup_name(name); - // TODO: if global, do something else! - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + self.load_name(name); } ast::Expression::Lambda { args, body } => { let name = "".to_string(); @@ -1453,6 +1467,7 @@ impl Compiler { // Load iterator onto stack (passed as first argument): self.emit(Instruction::LoadName { name: String::from(".0"), + scope: bytecode::NameScope::Local, }); } else { // Evaluate iterated item: @@ -1608,12 +1623,8 @@ impl Compiler { fn lookup_name(&self, name: &str) -> &SymbolRole { // println!("Looking up {:?}", name); - for scope in self.scope_stack.iter().rev() { - if let Some(role) = scope.lookup(name) { - return role; - } - } - unreachable!(); + let scope = self.scope_stack.last().unwrap(); + scope.lookup(name).unwrap() } // Low level helper functions: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2570c0934..a72ed6a76 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -315,8 +315,14 @@ impl Frame { ref symbol, } => self.import(vm, name, symbol), bytecode::Instruction::ImportStar { ref name } => self.import_star(vm, name), - bytecode::Instruction::LoadName { ref name } => self.load_name(vm, name), - bytecode::Instruction::StoreName { ref name } => self.store_name(vm, name), + bytecode::Instruction::LoadName { + ref name, + ref scope, + } => self.load_name(vm, name, scope), + bytecode::Instruction::StoreName { + ref name, + ref scope, + } => self.store_name(vm, name, scope), bytecode::Instruction::DeleteName { ref name } => self.delete_name(vm, name), bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), @@ -984,9 +990,21 @@ impl Frame { vm.call_method(context_manager, "__exit__", args) } - fn store_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { + fn store_name( + &self, + vm: &VirtualMachine, + name: &str, + scope: &bytecode::NameScope, + ) -> FrameResult { let obj = self.pop_value(); - self.scope.store_name(&vm, name, obj); + match scope { + bytecode::NameScope::Global => { + self.scope.globals.set_item(name, obj, vm)?; + } + bytecode::NameScope::Local => { + self.scope.store_name(&vm, name, obj); + } + } Ok(None) } @@ -995,19 +1013,27 @@ impl Frame { Ok(None) } - fn load_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - match self.scope.load_name(&vm, name) { - Some(value) => { - self.push_value(value); - Ok(None) - } - None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - Err(name_error) - } - } + fn load_name( + &self, + vm: &VirtualMachine, + name: &str, + scope: &bytecode::NameScope, + ) -> FrameResult { + let value = match scope { + bytecode::NameScope::Global => self.scope.globals.get_item(name, vm)?, + bytecode::NameScope::Local => match self.scope.load_name(&vm, name) { + Some(value) => value, + None => { + let name_error_type = vm.ctx.exceptions.name_error.clone(); + let msg = format!("name '{}' is not defined", name); + let name_error = vm.new_exception(name_error_type, msg); + return Err(name_error); + } + }, + }; + + self.push_value(value); + Ok(None) } fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 78655da53..2d6b49884 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -105,9 +105,14 @@ impl SymbolTableBuilder { fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { for parameter in parameters { - if let Some(annotation) = ¶meter.annotation { - self.scan_expression(&annotation)?; - } + self.scan_parameter(parameter)?; + } + Ok(()) + } + + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; } Ok(()) } @@ -134,16 +139,7 @@ impl SymbolTableBuilder { self.scan_expressions(decorator_list)?; self.register_name(name, SymbolRole::Assigned)?; - self.enter_scope(); - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - - self.scan_expressions(&args.defaults)?; - for kw_default in &args.kw_defaults { - if let Some(expression) = kw_default { - self.scan_expression(&expression)?; - } - } + self.enter_function(args)?; self.scan_statements(body)?; if let Some(expression) = returns { @@ -376,9 +372,7 @@ impl SymbolTableBuilder { self.register_name(name, SymbolRole::Used)?; } ast::Expression::Lambda { args, body } => { - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - self.enter_scope(); + self.enter_function(args)?; self.scan_expression(body)?; self.leave_scope(); } @@ -391,6 +385,27 @@ impl SymbolTableBuilder { Ok(()) } + fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { + // Evaulate eventual default parameters: + self.scan_expressions(&args.defaults)?; + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression)?; + } + } + + self.enter_scope(); + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter(name)?; + } + Ok(()) + } + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { match group { ast::StringGroup::Constant { .. } => {} @@ -408,9 +423,15 @@ impl SymbolTableBuilder { fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { let current_scope = self.scopes.last_mut().unwrap(); - if let Some(old_role) = current_scope.symbols.get(name) { - debug!("TODO: {:?}", old_role); - // Role already set.. + if let Some(_old_role) = current_scope.symbols.get(name) { + // Role already set.. + // debug!("TODO: {:?}", old_role); + match role { + SymbolRole::Global => return Err("global must appear first".to_string()), + _ => { + // Ok? + } + } } else { current_scope.symbols.insert(name.to_string(), role); } From 585e6e5428c42e0615eb6990315ee0e39a6e5d40 Mon Sep 17 00:00:00 2001 From: Adrian Wielgosik Date: Fri, 12 Apr 2019 19:28:09 +0200 Subject: [PATCH 194/223] Drop IntoPyNativeFunc impl to fix Nightly/Beta breakages. --- vm/src/function.rs | 6 ------ wasm/lib/src/vm_class.rs | 37 ++++++++++++++++++------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index 305c4de5e..514fe4cb6 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -437,12 +437,6 @@ where } } -impl IntoPyNativeFunc for PyNativeFunc { - fn into_func(self) -> PyNativeFunc { - self - } -} - pub struct OwnedParam(std::marker::PhantomData); pub struct RefParam(std::marker::PhantomData); diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 5efa7c849..ff72dddbb 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -199,15 +199,15 @@ impl WASMVirtualMachine { fn error() -> JsValue { TypeError::new("Unknown stdout option, please pass a function or 'console'").into() } - let print_fn: Box PyResult> = - if let Some(s) = stdout.as_string() { - match s.as_str() { - "console" => Box::new(wasm_builtins::builtin_print_console), - _ => return Err(error()), - } - } else if stdout.is_function() { - let func = js_sys::Function::from(stdout); - Box::new(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { + let print_fn: PyObjectRef = if let Some(s) = stdout.as_string() { + match s.as_str() { + "console" => vm.ctx.new_rustfunc(wasm_builtins::builtin_print_console), + _ => return Err(error()), + } + } else if stdout.is_function() { + let func = js_sys::Function::from(stdout); + vm.ctx + .new_rustfunc(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { func.call1( &JsValue::UNDEFINED, &wasm_builtins::format_print_args(vm, args)?.into(), @@ -215,16 +215,15 @@ impl WASMVirtualMachine { .map_err(|err| convert::js_to_py(vm, err))?; Ok(vm.get_none()) }) - } else if stdout.is_undefined() || stdout.is_null() { - fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { - Ok(vm.get_none()) - } - Box::new(noop) - } else { - return Err(error()); - }; - vm.set_attr(&vm.builtins, "print", vm.ctx.new_rustfunc(print_fn)) - .unwrap(); + } else if stdout.is_undefined() || stdout.is_null() { + fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) + } + vm.ctx.new_rustfunc(noop) + } else { + return Err(error()); + }; + vm.set_attr(&vm.builtins, "print", print_fn).unwrap(); Ok(()) })? } From a91cfd16bfb9b58057b2507aed14ee1cd3ebfb61 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Fri, 12 Apr 2019 14:28:26 -0400 Subject: [PATCH 195/223] Fix WASM import error --- wasm/demo/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index 9b58afe39..602e25241 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -1,6 +1,6 @@ import './style.css'; import 'codemirror/lib/codemirror.css'; -import 'xterm/dist/xterm.css'; +import 'xterm/lib/xterm.css'; // A dependency graph that contains any wasm must all be imported // asynchronously. This `index.js` file does the single async import, so From 20e0cc8bd771ccfea1fc0269db031070e03abf76 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 23:06:53 +0300 Subject: [PATCH 196/223] Use assertRaises --- tests/snippets/try_exceptions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 5d7509ba5..821d4e8c8 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -1,4 +1,4 @@ - +from testutils import assertRaises try: raise BaseException() @@ -117,10 +117,8 @@ try: except ZeroDivisionError as ex: assert ex.__cause__ == None -try: +with assertRaises(TypeError): raise ZeroDivisionError from 5 -except TypeError: - pass try: raise ZeroDivisionError from NameError From 256abe63b6ccdb5df6f86fe91f8e3ca2c7f3d211 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 23:08:26 +0300 Subject: [PATCH 197/223] Use vm.new_type_error --- vm/src/frame.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a1da85d3a..d67fd6b8d 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1207,9 +1207,7 @@ impl Frame { "Can only raise BaseException derived types, not {}", exception ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) + Err(vm.new_type_error(msg)) } } else { Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) From bd78f7e12bfe89a8a56185b167e599e8bb13b73c Mon Sep 17 00:00:00 2001 From: jgirardet Date: Sat, 13 Apr 2019 08:21:37 +0200 Subject: [PATCH 198/223] add capitalize --- tests/snippets/bytes.py | 6 +++++- vm/src/obj/objbyteinner.rs | 9 +++++++++ vm/src/obj/objbytes.rs | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 0ffbc29f8..1ad23ea18 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -109,11 +109,14 @@ assert not bytes(b"tuPpEr").isupper() assert bytes(b"Is Title Case").istitle() assert not bytes(b"is Not title casE").istitle() -# upper lower +# upper lower, capitalize l = bytes(b"lower") b = bytes(b"UPPER") assert l.lower().islower() assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytes().capitalize() == bytes() # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" @@ -127,3 +130,4 @@ try: bytes.fromhex("6Z2") except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 532504b02..9bedb0bf0 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -305,6 +305,15 @@ impl PyByteInner { self.elements.to_ascii_uppercase() } + pub fn capitalize(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::new(); + if let Some((first, second)) = self.elements.split_first() { + new.push(first.to_ascii_uppercase()); + second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); + } + new + } + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { let bla = self .elements diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2c8edca6a..953e5b2bb 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -209,6 +209,11 @@ impl PyBytesRef { Ok(vm.ctx.new_bytes(self.inner.upper(vm))) } + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.capitalize(vm))) + } + #[pymethod(name = "hex")] fn hex(self, vm: &VirtualMachine) -> PyResult { self.inner.hex(vm) From 73dbcfbe0108f9a7cffb83ef9ab0d7540e11b8a3 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Fri, 12 Apr 2019 16:30:57 +0200 Subject: [PATCH 199/223] Add bytes.center fix str.center add some tests introduce is_byte --- tests/snippets/bytes.py | 27 ++++++++++++++++++++++++ tests/snippets/strings.py | 23 +++++++++++++++++++++ vm/src/obj/objbyteinner.rs | 40 ++++++++++++++++++++++++++++++++++++ vm/src/obj/objbytes.rs | 42 +++++++++++++++++++++++++++++++++++++- vm/src/obj/objstr.rs | 18 ++++++++++++++-- 5 files changed, 147 insertions(+), 3 deletions(-) diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py index 1ad23ea18..d2a60943c 100644 --- a/tests/snippets/bytes.py +++ b/tests/snippets/bytes.py @@ -131,3 +131,30 @@ try: except ValueError as e: str(e) == "non-hexadecimal number found in fromhex() arg at position 1" +# center +assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [b"kok".center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +b"kok".center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".center(2, "a") +with assertRaises(TypeError): + b"b".center(2, b"ba") +b"kok".center(5, bytearray(b"x")) diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 31511db11..adcc106bb 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -55,6 +55,29 @@ assert b.rstrip() == ' hallo' c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' +assert ["koki".center(i, "|") for i in range(3, 10)] == [ + "koki", + "koki", + "|koki", + "|koki|", + "||koki|", + "||koki||", + "|||koki||", +] + + +assert ["kok".center(i, "|") for i in range(2, 10)] == [ + "kok", + "kok", + "kok|", + "|kok|", + "|kok||", + "||kok||", + "||kok|||", + "|||kok|||", +] + + # assert c.isascii() assert c.index('a') == 1 assert c.rindex('l') == 3 diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 9bedb0bf0..64ac24a3d 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -1,4 +1,5 @@ use crate::pyobject::PyObjectRef; +use num_bigint::BigInt; use crate::function::OptionalArg; @@ -13,8 +14,12 @@ use std::hash::{Hash, Hasher}; use super::objint; use super::objsequence::PySliceableSequence; use crate::obj::objint::PyInt; +use num_integer::Integer; use num_traits::ToPrimitive; +use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; +use super::objbytes::PyBytes; + #[derive(Debug, Default, Clone)] pub struct PyByteInner { pub elements: Vec, @@ -355,4 +360,39 @@ impl PyByteInner { .map(|x| x.unwrap()) .collect::>()) } + + pub fn center(&self, width: &BigInt, fillbyte: u8, _vm: &VirtualMachine) -> Vec { + let width = width.to_usize().unwrap(); + + // adjust right et left side + if width <= self.len() { + return self.elements.clone(); + } + let diff: usize = width - self.len(); + let mut ln: usize = diff / 2; + let mut rn: usize = ln; + + if diff.is_odd() && self.len() % 2 == 0 { + ln += 1 + } + + if diff.is_odd() && self.len() % 2 != 0 { + rn += 1 + } + + // merge all + let mut res = vec![fillbyte; ln]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; rn][..]); + + res + } +} + +pub fn is_byte(obj: &PyObjectRef) -> Option> { + match_class!(obj.clone(), + + i @ PyBytes => Some(i.get_value().to_vec()), + j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), + _ => None) } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 953e5b2bb..f183738c3 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -7,7 +7,7 @@ use std::ops::Deref; use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; -use super::objbyteinner::PyByteInner; +use super::objbyteinner::{is_byte, PyByteInner}; use super::objiter; use super::objslice::PySlice; use super::objtype::PyClassRef; @@ -34,6 +34,10 @@ impl PyBytes { inner: PyByteInner { elements }, } } + + pub fn get_value(&self) -> &[u8] { + &self.inner.elements + } } impl Deref for PyBytes { @@ -229,6 +233,42 @@ impl PyBytesRef { obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) ) } + + #[pymethod(name = "center")] + fn center( + self, + width: PyObjectRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let sym = if let OptionalArg::Present(v) = fillbyte { + match is_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + } else { + 32 // default is space + }; + + match_class!(width, + i @PyInt => Ok(vm.ctx.new_bytes(self.inner.center(i.as_bigint(), sym, vm))), + obj => {Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)))} + ) + } } #[derive(Debug)] diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 62cc923b1..2d5f1f8e1 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -691,8 +691,22 @@ impl PyStringRef { ) -> PyResult { let value = &self.value; let rep_char = Self::get_fill_char(&rep, vm)?; - let left_buff: usize = (len - value.len()) / 2; - let right_buff = len - value.len() - left_buff; + let value_len = self.value.chars().count(); + + if len <= value_len { + return Ok(value.to_string()); + } + let diff: usize = len - value_len; + let mut left_buff: usize = diff / 2; + let mut right_buff: usize = left_buff; + + if diff % 2 != 0 && value_len % 2 == 0 { + left_buff += 1 + } + + if diff % 2 != 0 && value_len % 2 != 0 { + right_buff += 1 + } Ok(format!( "{}{}{}", rep_char.repeat(left_buff), From 83ea419c63515f1a05910bfcc99209dd4bbb0e66 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sat, 13 Apr 2019 11:55:10 +0200 Subject: [PATCH 200/223] Add load and store of global to name protocol. --- vm/src/frame.rs | 42 +++++++++++++++++++++++++++--------------- vm/src/symboltable.rs | 23 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index a72ed6a76..e8276d1c8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -129,6 +129,8 @@ pub trait NameProtocol { fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn delete_name(&self, vm: &VirtualMachine, name: &str); fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); } impl NameProtocol for Scope { @@ -162,6 +164,14 @@ impl NameProtocol for Scope { fn delete_name(&self, vm: &VirtualMachine, key: &str) { self.get_locals().del_item(key, vm).unwrap(); } + + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { + self.globals.get_item_option(name, vm).unwrap() + } + + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.globals.set_item(name, value, vm).unwrap(); + } } #[derive(Clone, Debug)] @@ -994,12 +1004,12 @@ impl Frame { &self, vm: &VirtualMachine, name: &str, - scope: &bytecode::NameScope, + name_scope: &bytecode::NameScope, ) -> FrameResult { let obj = self.pop_value(); - match scope { + match name_scope { bytecode::NameScope::Global => { - self.scope.globals.set_item(name, obj, vm)?; + self.scope.store_global(vm, name, obj); } bytecode::NameScope::Local => { self.scope.store_name(&vm, name, obj); @@ -1017,19 +1027,21 @@ impl Frame { &self, vm: &VirtualMachine, name: &str, - scope: &bytecode::NameScope, + name_scope: &bytecode::NameScope, ) -> FrameResult { - let value = match scope { - bytecode::NameScope::Global => self.scope.globals.get_item(name, vm)?, - bytecode::NameScope::Local => match self.scope.load_name(&vm, name) { - Some(value) => value, - None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - return Err(name_error); - } - }, + let optional_value = match name_scope { + bytecode::NameScope::Global => self.scope.load_global(vm, name), + bytecode::NameScope::Local => self.scope.load_name(&vm, name), + }; + + let value = match optional_value { + Some(value) => value, + None => { + let name_error_type = vm.ctx.exceptions.name_error.clone(); + let msg = format!("name '{}' is not defined", name); + let name_error = vm.new_exception(name_error_type, msg); + return Err(name_error); + } }; self.push_value(value); diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index 2d6b49884..d7f30c870 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -111,6 +111,17 @@ impl SymbolTableBuilder { } fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.register_name(¶meter.arg, SymbolRole::Assigned) + } + + fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter_annotation(parameter)?; + } + Ok(()) + } + + fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { if let Some(annotation) = ¶meter.annotation { self.scan_expression(&annotation)?; } @@ -394,7 +405,19 @@ impl SymbolTableBuilder { } } + // Annotations are scanned in outer scope: + self.scan_parameters_annotations(&args.args)?; + self.scan_parameters_annotations(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter_annotation(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter_annotation(name)?; + } + self.enter_scope(); + + // Fill scope with parameter names: self.scan_parameters(&args.args)?; self.scan_parameters(&args.kwonlyargs)?; if let ast::Varargs::Named(name) = &args.vararg { From 16644daffa2fc7e140e02f1737e8be32963e6fb0 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 12:04:33 -0700 Subject: [PATCH 201/223] Fix sub-crate compilation --- derive/src/pyclass.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 433d5911e..e072db32d 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -16,11 +16,11 @@ enum ClassItem { }, } -fn meta_to_vec(meta: Meta) -> Result, Meta> { +fn meta_to_vec(meta: Meta) -> Option> { match meta { - Meta::Word(_) => Ok(Vec::new()), - Meta::List(list) => Ok(list.nested.into_iter().collect()), - Meta::NameValue(_) => Err(meta), + Meta::Word(_) => Some(Vec::new()), + Meta::List(list) => Some(list.nested.into_iter().collect()), + Meta::NameValue(_) => None, } } From 883946102849401a5953c4758e0ccffda6e38b61 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 12:45:09 -0700 Subject: [PATCH 202/223] str: improve {is,}title impl and add tests --- vm/src/obj/objstr.rs | 126 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 20 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 0e16a8b3b..3e0bcccb6 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -43,6 +43,14 @@ impl PyString { } } +impl From<&str> for PyString { + fn from(s: &str) -> PyString { + PyString { + value: s.to_string(), + } + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { @@ -396,9 +404,33 @@ impl PyString { } } + /// Return a titlecased version of the string where words start with an + /// uppercase character and the remaining characters are lowercase. #[pymethod] fn title(&self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + let mut title = String::new(); + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_lowercase() { + if !previous_is_cased { + title.extend(c.to_uppercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else if c.is_uppercase() { + if previous_is_cased { + title.extend(c.to_lowercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else { + previous_is_cased = false; + title.push(c); + } + } + title } #[pymethod] @@ -609,13 +641,34 @@ impl PyString { vm.ctx.new_tuple(new_tup) } + /// Return `true` if the sequence is ASCII titlecase and the sequence is not + /// empty, `false` otherwise. #[pymethod] fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { - false - } else { - self.value.split(' ').all(|word| word == make_title(word)) + return false; } + + let mut cased = false; + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_uppercase() { + if previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else if c.is_lowercase() { + if !previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else { + previous_is_cased = false; + } + } + cased } #[pymethod] @@ -967,22 +1020,55 @@ fn adjust_indices( } } -// helper function to title strings -fn make_title(s: &str) -> String { - let mut titled_str = String::new(); - let mut capitalize_char: bool = true; - for c in s.chars() { - if c.is_alphabetic() { - if !capitalize_char { - titled_str.push(c); - } else if capitalize_char { - titled_str.push(c.to_ascii_uppercase()); - capitalize_char = false; - } - } else { - titled_str.push(c); - capitalize_char = true; +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn str_title() { + let vm = VirtualMachine::new(); + + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyString::from(input).title(&vm), String::from(title)); + } + } + + #[test] + fn str_istitle() { + let vm = VirtualMachine::new(); + + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + ]; + + for s in pos { + assert!(PyString::from(s).istitle(&vm)); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyString::from(s).istitle(&vm)); } } - titled_str } From 1f8181b56160b4591f17da22467553750ae00f7e Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 09:40:59 +1200 Subject: [PATCH 203/223] Update formatting to be consistent with rustfmt included in 1.34 --- parser/src/lexer.rs | 6 +++++- vm/src/obj/objbytes.rs | 10 +++++----- vm/src/obj/objsuper.rs | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index e42e3b248..dfc9ecd51 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -826,7 +826,11 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + tok_end, + ))); } } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 279dbefe0..fd9bfd5bf 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -62,13 +62,13 @@ pub fn init(context: &PyContext) { PyBytesRef::extend_class(context, &context.bytes_type); let bytes_type = &context.bytes_type; extend_class!(context, bytes_type, { -"fromhex" => context.new_rustfunc(PyBytesRef::fromhex), -}); + "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), + }); let bytesiterator_type = &context.bytesiterator_type; extend_class!(context, bytesiterator_type, { -"__next__" => context.new_rustfunc(PyBytesIteratorRef::next), -"__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), -}); + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), + }); } #[pyimpl(__inside_vm)] diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 94fe0fc7b..194212a49 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -127,8 +127,10 @@ fn super_new( match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { From 6650ad895a3ca925556f802efc6c1d3b91ef7b52 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 10:07:50 +1200 Subject: [PATCH 204/223] Fix a bunch of clippy warnings --- vm/src/compile.rs | 4 ++-- vm/src/frame.rs | 6 +++--- vm/src/function.rs | 2 +- vm/src/obj/objbyteinner.rs | 3 +-- vm/src/obj/objdict.rs | 2 +- vm/src/obj/objint.rs | 2 +- vm/src/obj/objrange.rs | 4 ++-- vm/src/obj/objset.rs | 12 ++++++------ vm/src/obj/objslice.rs | 2 +- vm/src/obj/objtype.rs | 2 +- vm/src/pyobject.rs | 12 +++--------- vm/src/stdlib/os.rs | 4 ++-- 12 files changed, 24 insertions(+), 31 deletions(-) diff --git a/vm/src/compile.rs b/vm/src/compile.rs index d5a4b48ec..163e2fcf1 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -871,8 +871,8 @@ impl Compiler { vals: &[ast::Expression], ops: &[ast::Comparison], ) -> Result<(), CompileError> { - assert!(ops.len() > 0); - assert!(vals.len() == ops.len() + 1); + assert!(!ops.is_empty()); + assert_eq!(vals.len(), ops.len() + 1); let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 14154016d..064951770 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1215,9 +1215,9 @@ impl Frame { fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { let exception = self.pop_value(); - if none_allowed && vm.get_none().is(&exception) { - Ok(exception) - } else if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { + if none_allowed && vm.get_none().is(&exception) + || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) + { Ok(exception) } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { diff --git a/vm/src/function.rs b/vm/src/function.rs index 6e421ac98..9ed237a50 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -286,7 +286,7 @@ pub struct Args(Vec); impl Args> { pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx - .new_tuple(self.0.into_iter().map(|obj| obj.into_object()).collect()) + .new_tuple(self.0.into_iter().map(PyRef::into_object).collect()) } } diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index 64ac24a3d..d0a63633a 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -356,8 +356,7 @@ impl PyByteInner { .collect::>() .chunks(2) .map(|x| x.to_vec().iter().collect::()) - .map(|x| u8::from_str_radix(&x, 16)) - .map(|x| x.unwrap()) + .map(|x| u8::from_str_radix(&x, 16).unwrap()) .collect::>()) } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index efd38a7fd..fd77107ee 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -118,7 +118,7 @@ impl PyDictRef { } } } - return Ok(true); + Ok(true) } fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 5be50f6c4..52f321adf 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -518,7 +518,7 @@ pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult Ok(f.to_f64().to_bigint().unwrap()), s @ PyString => { i32::from_str_radix(s.as_str(), base) - .map(|i| BigInt::from(i)) + .map(BigInt::from) .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 2820dd9b4..00b4e0e2e 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -144,8 +144,8 @@ impl PyRange { vm: &VirtualMachine, ) -> PyResult { PyRange { - start: start, - stop: stop, + start, + stop, step: step .into_option() .unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)), diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 5ec4fca46..b8f9f6d3b 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -262,7 +262,7 @@ impl PySetInner { perform_action_with_hash(vm, &mut self.elements, &item, &discard) } - fn clear(&mut self) -> () { + fn clear(&mut self) { self.elements.clear(); } @@ -449,11 +449,11 @@ impl PySetRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let inner = self.inner.borrow(); let s = if inner.len() == 0 { - format!("set()") + "set()".to_string() } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { inner.repr(vm)? } else { - format!("set(...)") + "set(...)".to_string() }; Ok(vm.new_str(s)) } @@ -470,7 +470,7 @@ impl PySetRef { self.inner.borrow_mut().discard(&item, vm) } - fn clear(self, _vm: &VirtualMachine) -> () { + fn clear(self, _vm: &VirtualMachine) { self.inner.borrow_mut().clear() } @@ -656,11 +656,11 @@ impl PyFrozenSetRef { fn repr(self, vm: &VirtualMachine) -> PyResult { let inner = &self.inner; let s = if inner.len() == 0 { - format!("frozenset()") + "frozenset()".to_string() } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { format!("frozenset({})", inner.repr(vm)?) } else { - format!("frozenset(...)") + "frozenset(...)".to_string() }; Ok(vm.new_str(s)) } diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index c82a3ca6e..31fcfa52f 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -63,7 +63,7 @@ fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { step: step.map(|x| objint::get_value(x).clone()), } .into_ref_with_type(vm, cls.clone().downcast().unwrap()) - .map(|x| x.into_object()) + .map(PyRef::into_object) } fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index b1b50dfbd..819f85088 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -235,7 +235,7 @@ pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { let (typ, name, bases, dict) = args.bind(vm)?; - type_new_class(vm, typ, name, bases, dict).map(|x| x.into_object()) + type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { Err(vm.new_type_error(format!(": type_new: {:?}", args))) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index f91886996..3b21482ba 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -87,7 +87,6 @@ pub type PyAttributes = HashMap; impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { let type_name = self.class().name.clone(); // We don't have access to a vm, so just assume that if its parent's name @@ -698,7 +697,7 @@ impl PyContext { pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ: class, - dict: dict, + dict, payload: objobject::PyInstance, } .into_ref() @@ -1065,7 +1064,7 @@ where fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { if let Ok(method) = vm.get_method(obj.clone(), "__iter__") { Ok(PyIterable { - method: method, + method, _item: std::marker::PhantomData, }) } else if vm.get_method(obj.clone(), "__getitem__").is_ok() { @@ -1170,12 +1169,7 @@ where T: Sized + PyObjectPayload, { pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { - PyObject { - typ, - dict: dict, - payload, - } - .into_ref() + PyObject { typ, dict, payload }.into_ref() } // Move this object into a reference object, transferring ownership. diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 26707d3ca..f397b2d8a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -174,11 +174,11 @@ fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } } -fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) -> () { +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) { env::set_var(&key.value, &value.value) } -fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) -> () { +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) { env::remove_var(&key.value) } From d24dd170e9b299839a3c98f03ab21ef45c1c8584 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 12:09:54 +1200 Subject: [PATCH 205/223] Changes to RustPython to support importing and running unittest --- vm/src/builtins.rs | 18 ++++++------------ vm/src/eval.rs | 7 +------ vm/src/import.rs | 11 ++++------- vm/src/obj/objclassmethod.rs | 2 ++ vm/src/obj/objfunction.rs | 6 ++++++ vm/src/obj/objobject.rs | 16 +++++++++++++++- vm/src/pyobject.rs | 4 +++- vm/src/stdlib/io.rs | 6 +++++- vm/src/stdlib/mod.rs | 2 ++ vm/src/stdlib/thread.rs | 30 ++++++++++++++++++++++++++++++ vm/src/stdlib/types.rs | 1 + vm/src/sysmodule.rs | 8 ++++++++ vm/src/vm.rs | 5 +++++ 13 files changed, 88 insertions(+), 28 deletions(-) create mode 100644 vm/src/stdlib/thread.rs diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 6a94b8362..8f7721609 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -102,10 +102,8 @@ fn builtin_compile( } }; - compile::compile(vm, &source, &mode, filename.value.to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }) + compile::compile(vm, &source, &mode, filename.value.to_string()) + .map_err(|err| vm.new_syntax_error(&err)) } fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { @@ -151,10 +149,8 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else { return Err(vm.new_type_error("code argument must be str or code object".to_string())); }; @@ -181,10 +177,8 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) { code_obj } else { diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 6c7b3ca54..21bc9dadf 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,7 +1,5 @@ extern crate rustpython_parser; -use std::error::Error; - use crate::compile; use crate::frame::Scope; use crate::pyobject::PyResult; @@ -13,10 +11,7 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) debug!("Code object: {:?}", bytecode); vm.run_code_obj(bytecode, scope) } - Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - Err(vm.new_exception(syntax_error, err.description().to_string())) - } + Err(err) => Err(vm.new_syntax_error(&err)), } } diff --git a/vm/src/import.rs b/vm/src/import.rs index 14c0a7566..041fab650 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -2,7 +2,6 @@ * Import mechanics */ -use std::error::Error; use std::path::PathBuf; use crate::compile; @@ -25,17 +24,14 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let file_path = find_source(vm, current_path, module) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), e.description().to_string()))?; + .map_err(|e| vm.new_exception(import_error.clone(), format!("{}", e)))?; let code_obj = compile::compile( vm, &source, &compile::Mode::Exec, file_path.to_str().unwrap().to_string(), ) - .map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.description().to_string()) - })?; + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); @@ -64,12 +60,13 @@ fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result paths.insert(0, current_path); + let rel_name = name.replace('.', "/"); let suffixes = [".py", "/__init__.py"]; let mut file_paths = vec![]; for path in paths { for suffix in suffixes.iter() { let mut file_path = path.clone(); - file_path.push(format!("{}{}", name, suffix)); + file_path.push(format!("{}{}", rel_name, suffix)); file_paths.push(file_path); } } diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index ffacfdc07..7a8d90d1c 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -9,6 +9,8 @@ pub struct PyClassMethod { pub type PyClassMethodRef = PyRef; impl PyValue for PyClassMethod { + const HAVE_DICT: bool = true; + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.classmethod_type() } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 01e4b586f..05b66c346 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,4 +1,5 @@ use crate::frame::Scope; +use crate::function::{Args, KwArgs}; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objtuple::PyTupleRef; @@ -40,6 +41,10 @@ impl PyValue for PyFunction { } impl PyFunctionRef { + fn call(self, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.into_object(), (&args, &kwargs)) + } + fn code(self, _vm: &VirtualMachine) -> PyCodeRef { self.code.clone() } @@ -76,6 +81,7 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), + "__call__" => context.new_rustfunc(PyFunctionRef::call), "__code__" => context.new_property(PyFunctionRef::code), "__defaults__" => context.new_property(PyFunctionRef::defaults), "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index f02bbabd1..47cfa008d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -167,7 +167,11 @@ pub fn init(context: &PyContext) { "__ge__" => context.new_rustfunc(object_ge), "__setattr__" => context.new_rustfunc(object_setattr), "__delattr__" => context.new_rustfunc(object_delattr), - "__dict__" => context.new_property(object_dict), + "__dict__" => + PropertyBuilder::new(context) + .add_getter(object_dict) + .add_setter(object_dict_setter) + .create(), "__dir__" => context.new_rustfunc(object_dir), "__hash__" => context.new_rustfunc(object_hash), "__str__" => context.new_rustfunc(object_str), @@ -203,6 +207,16 @@ fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult } } +fn object_dict_setter( + _instance: PyObjectRef, + _value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on am object isn't yet implemented".to_string(), + )) +} + fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_str.value; trace!("object.__getattribute__({:?}, {:?})", obj, name); diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 3b21482ba..f369bab4a 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1191,6 +1191,8 @@ impl PyObject { } pub trait PyValue: fmt::Debug + Sized + 'static { + const HAVE_DICT: bool = false; + fn class(vm: &VirtualMachine) -> PyClassRef; fn into_ref(self, vm: &VirtualMachine) -> PyRef { @@ -1203,7 +1205,7 @@ pub trait PyValue: fmt::Debug + Sized + 'static { fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyClassRef) -> PyResult> { let class = Self::class(vm); if objtype::issubclass(&cls, &class) { - let dict = if cls.is(&class) { + let dict = if !Self::HAVE_DICT && cls.is(&class) { None } else { Some(vm.ctx.new_dict()) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index bd33eac75..d864519f5 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -95,6 +95,9 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +// TODO Check if closed, then if so raise ValueError +fn io_base_flush(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); vm.set_attr(buffered, "raw", raw.clone())?; @@ -369,7 +372,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), - "__exit__" => ctx.new_rustfunc(io_base_cm_exit) + "__exit__" => ctx.new_rustfunc(io_base_cm_exit), + "flush" => ctx.new_rustfunc(io_base_flush) }); // IOBase Subclasses diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 9b0d0b345..0933b7192 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -9,6 +9,7 @@ mod random; mod re; pub mod socket; mod string; +mod thread; mod time_module; mod tokenize; mod types; @@ -41,6 +42,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("random".to_string(), Box::new(random::make_module)); modules.insert("string".to_string(), Box::new(string::make_module)); modules.insert("struct".to_string(), Box::new(pystruct::make_module)); + modules.insert("_thread".to_string(), Box::new(thread::make_module)); modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); modules.insert("types".to_string(), Box::new(types::make_module)); diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs new file mode 100644 index 000000000..8de881893 --- /dev/null +++ b/vm/src/stdlib/thread.rs @@ -0,0 +1,30 @@ +/// Implementation of the _thread module, currently noop implementation as RustPython doesn't yet +/// support threading +use super::super::pyobject::PyObjectRef; +use crate::function::PyFuncArgs; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +fn rlock_acquire(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) +} + +fn rlock_release(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + +fn get_ident(_vm: &VirtualMachine) -> u32 { + 1 +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let rlock_type = py_class!(ctx, "_thread.RLock", ctx.object(), { + "acquire" => ctx.new_rustfunc(rlock_acquire), + "release" => ctx.new_rustfunc(rlock_release), + }); + + py_module!(vm, "_thread", { + "RLock" => rlock_type, + "get_ident" => ctx.new_rustfunc(get_ident) + }) +} diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs index ad76a80a4..e224a93c6 100644 --- a/vm/src/stdlib/types.rs +++ b/vm/src/stdlib/types.rs @@ -31,6 +31,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { py_module!(vm, "types", { "new_class" => ctx.new_rustfunc(types_new_class), "FunctionType" => ctx.function_type(), + "MethodType" => ctx.bound_method_type(), "LambdaType" => ctx.function_type(), "CodeType" => ctx.code_type(), "FrameType" => ctx.frame_type() diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 2f8690448..97de7a425 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,6 +3,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objstr::PyStringRef; use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; @@ -39,6 +40,11 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } +// TODO implement string interning, this will be key for performance +fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + value +} + pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; @@ -130,6 +136,7 @@ settrace() -- set the global debug tracing function "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, "ps1" => ctx.new_str(">>>>> ".to_string()), @@ -137,6 +144,7 @@ settrace() -- set the global debug tracing function "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), "modules" => modules.clone(), + "warnoptions" => ctx.new_list(vec![]), }); modules.set_item("sys", module.clone(), vm).unwrap(); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 5ae5b0b50..e360b8e5e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -234,6 +234,11 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } + pub fn new_syntax_error(&self, msg: &T) -> PyObjectRef { + let syntax_error = self.ctx.exceptions.syntax_error.clone(); + self.new_exception(syntax_error, msg.to_string()) + } + pub fn get_none(&self) -> PyObjectRef { self.ctx.none() } From a708ce2eb2e9179f957feb87b917110fa33a0539 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 12:15:55 +1200 Subject: [PATCH 206/223] Remove needless format! --- vm/src/import.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/import.rs b/vm/src/import.rs index 041fab650..b59ce6426 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -24,7 +24,7 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let file_path = find_source(vm, current_path, module) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), format!("{}", e)))?; + .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; let code_obj = compile::compile( vm, &source, From 762c281dd48c5c762e95dc25f491ccca5367c4f9 Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 13 Apr 2019 19:07:15 -0700 Subject: [PATCH 207/223] address feedback --- vm/src/obj/objstr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 3e0bcccb6..5a133615c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1038,7 +1038,7 @@ mod tests { ("Greek Ωppercases ...", "greek ωppercases ..."), ]; for (title, input) in tests { - assert_eq!(PyString::from(input).title(&vm), String::from(title)); + assert_eq!(PyString::from(input).title(&vm).as_str(), title); } } From 307689a15ee8ee6699ec7f9ebe3e2e4659f01247 Mon Sep 17 00:00:00 2001 From: ben Date: Sun, 14 Apr 2019 17:47:51 +1200 Subject: [PATCH 208/223] comment out lines that reference mappingproxy as type doesn't yet have a __dict__ --- Lib/_collections_abc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3158a4f16..c9f7daa16 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -52,7 +52,7 @@ dict_keys = type({}.keys()) dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## -mappingproxy = type(type.__dict__) +# mappingproxy = type(type.__dict__) # generator = type((lambda: (yield))()) ## coroutine ## # async def _coro(): pass @@ -688,7 +688,7 @@ class Mapping(Collection): __reversed__ = None -Mapping.register(mappingproxy) +# Mapping.register(mappingproxy) class MappingView(Sized): From 2c2925a68435e837af8aa9c8b6dc01bdf0e5666e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Fri, 12 Apr 2019 17:22:52 +0300 Subject: [PATCH 209/223] Support reraise --- tests/snippets/try_exceptions.py | 12 ++++++++++ vm/src/bytecode.rs | 3 ++- vm/src/compile.rs | 40 ++++++++++++++++++++++---------- vm/src/frame.rs | 12 +++++++--- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 821d4e8c8..2a6ed7b34 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -125,3 +125,15 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError +try: + try: + raise NameError + except: + raise +except NameError: + pass + +try: + raise +except RuntimeError: + pass diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 2725aa586..10c667890 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -135,6 +135,7 @@ pub enum Instruction { PopBlock, Raise { argc: usize, + in_exc: bool, }, BuildString { size: usize, @@ -363,7 +364,7 @@ impl Instruction { SetupWith { end } => w!(SetupWith, end), CleanupWith { end } => w!(CleanupWith, end), PopBlock => w!(PopBlock), - Raise { argc } => w!(Raise, argc), + Raise { argc, in_exc } => w!(Raise, argc, in_exc), BuildString { size } => w!(BuildString, size), BuildTuple { size, unpack } => w!(BuildTuple, size, unpack), BuildList { size, unpack } => w!(BuildList, size, unpack), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index fe40962fe..9e8dab905 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -24,6 +24,7 @@ struct Compiler { current_qualified_path: Option, in_loop: bool, in_function_def: bool, + in_exc_handler: bool, } /// Compile a given sourcecode into a bytecode object. @@ -85,6 +86,7 @@ impl Compiler { current_qualified_path: None, in_loop: false, in_function_def: false, + in_exc_handler: false, } } @@ -325,15 +327,24 @@ impl Compiler { match cause { Some(cause) => { self.compile_expression(cause)?; - self.emit(Instruction::Raise { argc: 2 }); + self.emit(Instruction::Raise { + argc: 2, + in_exc: self.in_exc_handler, + }); } None => { - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); } } } None => { - self.emit(Instruction::Raise { argc: 0 }); + self.emit(Instruction::Raise { + argc: 0, + in_exc: self.in_exc_handler, + }); } }, ast::Statement::Try { @@ -378,7 +389,10 @@ impl Compiler { }); } } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); self.set_label(end_label); } ast::Statement::Break => { @@ -558,6 +572,7 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: + self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now handler_label = self.new_label(); @@ -586,15 +601,10 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { + // Duplicate exception for context: + self.emit(Instruction::Duplicate); self.store_name(alias); - } else { - // Drop exception from top of stack: - self.emit(Instruction::Pop); } - } else { - // Catch all! - // Drop exception from top of stack: - self.emit(Instruction::Pop); } // Handler code: @@ -611,6 +621,9 @@ impl Compiler { target: handler_label, }); self.set_label(handler_label); + // Drop exception from top of stack: + self.emit(Instruction::Pop); + self.in_exc_handler = false; // If code flows here, we have an unhandled exception, // emit finally code and raise again! // Duplicate finally code here: @@ -619,7 +632,10 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: false, + }); // We successfully ran the try block: // else: diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 321ee2046..464578d6b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -715,14 +715,20 @@ impl Frame { Ok(None) } - bytecode::Instruction::Raise { argc } => { + bytecode::Instruction::Raise { argc, in_exc } => { + if argc.clone() == 0 && in_exc.clone() == false { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } let cause = match argc { 2 => self.get_exception(vm, true)?, _ => vm.get_none(), }; let exception = match argc { - 1 | 2 => self.get_exception(vm, false)?, - 0 | 3 => panic!("Not implemented!"), + 0 | 1 | 2 => self.get_exception(vm, false)?, + 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; info!("Exception raised: {:?} with cause: {:?}", exception, cause); From 3ec9b01928ddbd212f54e74d4afdcbb8b98a856b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:22:15 +0300 Subject: [PATCH 210/223] Add exception __context__ --- tests/snippets/try_exceptions.py | 5 +++++ vm/src/compile.rs | 12 ++++++------ vm/src/frame.rs | 30 ++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 2a6ed7b34..e75e73039 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -137,3 +137,8 @@ try: raise except RuntimeError: pass + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__context__ == None diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 9e8dab905..ba1f50e7b 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -572,6 +572,7 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: + let was_in_exc_handler = self.in_exc_handler; self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now @@ -609,6 +610,8 @@ impl Compiler { // Handler code: self.compile_statements(&handler.body)?; + // Drop exception from top of stack: + self.emit(Instruction::Pop); self.emit(Instruction::Jump { target: finally_label, }); @@ -621,9 +624,6 @@ impl Compiler { target: handler_label, }); self.set_label(handler_label); - // Drop exception from top of stack: - self.emit(Instruction::Pop); - self.in_exc_handler = false; // If code flows here, we have an unhandled exception, // emit finally code and raise again! // Duplicate finally code here: @@ -633,8 +633,8 @@ impl Compiler { self.compile_statements(statements)?; } self.emit(Instruction::Raise { - argc: 1, - in_exc: false, + argc: 0, + in_exc: true, }); // We successfully ran the try block: @@ -649,7 +649,7 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - + self.in_exc_handler = was_in_exc_handler; // unimplemented!(); Ok(()) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 464578d6b..2bf8215c1 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -716,23 +716,37 @@ impl Frame { } bytecode::Instruction::Raise { argc, in_exc } => { - if argc.clone() == 0 && in_exc.clone() == false { - return Err(vm.new_exception( - vm.ctx.exceptions.runtime_error.clone(), - "No active exception to reraise".to_string(), - )); - } let cause = match argc { 2 => self.get_exception(vm, true)?, _ => vm.get_none(), }; let exception = match argc { - 0 | 1 | 2 => self.get_exception(vm, false)?, + 0 => match in_exc { + true => self.get_exception(vm, false)?, + false => { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } + }, + 1 | 2 => self.get_exception(vm, false)?, 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - info!("Exception raised: {:?} with cause: {:?}", exception, cause); + let context = match argc { + 0 => vm.get_none(), // We have already got the exception, + _ => match in_exc { + true => self.get_exception(vm, false)?, + false => vm.get_none(), + }, + }; + info!( + "Exception raised: {:?} with cause: {:?} and context: {:?}", + exception, cause, context + ); vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + vm.set_attr(&exception, vm.new_str("__context__".to_string()), context)?; Err(exception) } From 188758929df8b82e5484e87f800ecff5994ae692 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:42:23 +0300 Subject: [PATCH 211/223] Add more tests to exception context --- tests/snippets/try_exceptions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index e75e73039..1af2d9762 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -106,6 +106,7 @@ try: raise NameError from ex except NameError as ex2: assert ex2.__cause__ == cause + assert ex2.__context__ == cause try: raise ZeroDivisionError from None @@ -138,7 +139,25 @@ try: except RuntimeError: pass +context = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__context__ == None + context = ex + raise NameError +except NameError as ex2: + assert ex2.__context__ == context + assert type(ex2.__context__) == ZeroDivisionError + try: raise ZeroDivisionError except ZeroDivisionError as ex: assert ex.__context__ == None + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + assert ex.__context__ == None From aec7f84960add779f0e81d5cb5e4169cc4aeab2b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 18:52:01 +0300 Subject: [PATCH 212/223] Print exception context --- vm/src/exceptions.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index abd670a14..7642908ee 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -20,12 +20,22 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + let mut had_casue = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { if !vm.get_none().is(&cause) { + had_casue = true; print_exception(vm, &cause); println!("\nThe above exception was the direct cause of the following exception:\n"); } } + if !had_casue { + if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { + if !vm.get_none().is(&context) { + print_exception(vm, &context); + println!("\nDuring handling of the above exception, another exception occurred:\n"); + } + } + } print_exception_inner(vm, exc) } From b6ce4073917bcf1d4e581896773aa0abce9a2a0c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 19:05:28 +0300 Subject: [PATCH 213/223] Remove else and finally from in_exc_handler --- tests/snippets/try_exceptions.py | 10 ++++++++++ vm/src/compile.rs | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 1af2d9762..d7785424a 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -161,3 +161,13 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError assert ex.__context__ == None + +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + pass + finally: + raise NameError +except NameError as ex2: + assert ex2.__context__ == None diff --git a/vm/src/compile.rs b/vm/src/compile.rs index ba1f50e7b..d28c10feb 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -637,6 +637,8 @@ impl Compiler { in_exc: true, }); + self.in_exc_handler = was_in_exc_handler; + // We successfully ran the try block: // else: self.set_label(else_label); @@ -649,7 +651,6 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.in_exc_handler = was_in_exc_handler; // unimplemented!(); Ok(()) } From 4068a36ce8a576b3b5c7b19716a8cb0ee0a43421 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 20:38:34 +0300 Subject: [PATCH 214/223] Use assertRaises --- tests/snippets/try_exceptions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index d7785424a..83e00c0fe 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -126,18 +126,14 @@ try: except ZeroDivisionError as ex: assert type(ex.__cause__) == NameError -try: +with assertRaises(NameError): try: raise NameError except: raise -except NameError: - pass -try: +with assertRaises(RuntimeError): raise -except RuntimeError: - pass context = None try: From 2ffd1c948447df991e927e6d7a6d88d42d2527a7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 14 Apr 2019 20:39:28 +0300 Subject: [PATCH 215/223] Fix typo --- vm/src/exceptions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 7642908ee..5f577d04f 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -20,15 +20,15 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { - let mut had_casue = false; + let mut had_cause = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { if !vm.get_none().is(&cause) { - had_casue = true; + had_cause = true; print_exception(vm, &cause); println!("\nThe above exception was the direct cause of the following exception:\n"); } } - if !had_casue { + if !had_cause { if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { if !vm.get_none().is(&context) { print_exception(vm, &context); From daaeb7883146a0cd3611f3c661f951a4e64bbd58 Mon Sep 17 00:00:00 2001 From: Darren Kaste Date: Sun, 14 Apr 2019 17:08:35 -0400 Subject: [PATCH 216/223] Prefix helper macro calls with `$crate` --- vm/src/macros.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 27d164718..b84643b5f 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -9,7 +9,7 @@ macro_rules! replace_expr { #[macro_export] macro_rules! count_tts { - ($($tts:tt)*) => {0usize $(+ replace_expr!($tts 1usize))*}; + ($($tts:tt)*) => {0usize $(+ $crate::replace_expr!($tts 1usize))*}; } #[macro_export] @@ -45,14 +45,14 @@ macro_rules! arg_check { } }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*] ) => { - arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); + $crate::arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*], optional=[$( ($optional_arg_name:ident, $optional_arg_type:expr) ),*] ) => { let mut arg_count = 0; // use macro magic to compile-time count number of required and optional arguments - let minimum_arg_count = count_tts!($($arg_name)*); - let maximum_arg_count = minimum_arg_count + count_tts!($($optional_arg_name)*); + let minimum_arg_count = $crate::count_tts!($($arg_name)*); + let maximum_arg_count = minimum_arg_count + $crate::count_tts!($($optional_arg_name)*); // verify that the number of given arguments is right if $args.args.len() < minimum_arg_count || $args.args.len() > maximum_arg_count { @@ -72,7 +72,7 @@ macro_rules! arg_check { // check if the type matches. If not, return with error // assign the arg to a variable $( - type_check!($vm, $args, arg_count, $arg_name, $arg_type); + $crate::type_check!($vm, $args, arg_count, $arg_name, $arg_type); let $arg_name = &$args.args[arg_count]; #[allow(unused_assignments)] { @@ -85,7 +85,7 @@ macro_rules! arg_check { // assign the arg to a variable $( let $optional_arg_name = if arg_count < $args.args.len() { - type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); + $crate::type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); let ret = Some(&$args.args[arg_count]); #[allow(unused_assignments)] { @@ -226,7 +226,7 @@ macro_rules! match_class { ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { match $obj.downcast::<$class>() { Ok($binding) => $expr, - Err(_obj) => match_class!(_obj, $($rest)*), + Err(_obj) => $crate::match_class!(_obj, $($rest)*), } }; @@ -236,7 +236,7 @@ macro_rules! match_class { if $obj.payload_is::<$class>() { $expr } else { - match_class!($obj, $($rest)*) + $crate::match_class!($obj, $($rest)*) } }; } From a8e064b1c0f32077998a913d00b08a29ebeaf9dc Mon Sep 17 00:00:00 2001 From: Darren Kaste Date: Sun, 14 Apr 2019 17:19:44 -0400 Subject: [PATCH 217/223] Import `TypeProtocol` before use in `type_check!` --- vm/src/macros.rs | 2 ++ vm/src/obj/objbool.rs | 4 +--- vm/src/obj/objellipsis.rs | 2 +- vm/src/obj/objlist.rs | 1 - vm/src/obj/objslice.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/keyword.rs | 2 +- vm/src/stdlib/math.rs | 2 +- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/pystruct.rs | 2 +- vm/src/stdlib/random.rs | 2 +- vm/src/stdlib/socket.rs | 2 +- vm/src/stdlib/time_module.rs | 2 +- vm/src/stdlib/tokenize.rs | 2 +- vm/src/sysmodule.rs | 2 +- 15 files changed, 15 insertions(+), 16 deletions(-) diff --git a/vm/src/macros.rs b/vm/src/macros.rs index b84643b5f..333673985 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -20,6 +20,8 @@ macro_rules! type_check { let arg = &$args.args[$arg_count]; if !$crate::obj::objtype::isinstance(arg, &expected_type) { + use $crate::pyobject::TypeProtocol; + let arg_typ = arg.class(); let expected_type_name = $vm.to_pystr(&expected_type)?; let actual_type = $vm.to_pystr(&arg_typ)?; diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index d4ef4b0fb..724c8bc5a 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -1,9 +1,7 @@ use num_traits::Zero; use crate::function::PyFuncArgs; -use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; use super::objint::PyInt; diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 6591d8173..cb188e1f1 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -1,5 +1,5 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 276a8ea89..a21bd9190 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -9,7 +9,6 @@ use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, - TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 31fcfa52f..60d911a92 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,7 +1,7 @@ use num_bigint::BigInt; use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index d864519f5..b8e72cc15 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,7 +20,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; fn compute_c_flag(mode: &str) -> u16 { diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index 3143af21a..062f41df7 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index dc7981ab6..2ee584a79 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; // Helper macro: diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f397b2d8a..fae9d6adb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -12,7 +12,7 @@ use crate::obj::objint; use crate::obj::objint::PyIntRef; use crate::obj::objstr; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; #[cfg(unix)] diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 2ff61d630..4a454b0c9 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -15,7 +15,7 @@ use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; #[derive(Debug)] diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index c6cc44a36..cf3c0ea18 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,7 +4,7 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index ab5072f7c..e893e0dab 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,7 +10,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objsequence::get_elements; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; use crate::obj::objtype::PyClassRef; diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index 93beabf8a..6c818689e 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 88a26881c..000dd5e1a 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 97de7a425..a5bf2a6f4 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -4,7 +4,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* From cc76bf7a1f4c7ecf9d72313107538bf4b20a600f Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 14 Apr 2019 20:09:15 -0500 Subject: [PATCH 218/223] Add weakproxy class --- vm/src/exceptions.rs | 3 +++ vm/src/obj/mod.rs | 1 + vm/src/obj/objweakproxy.rs | 50 ++++++++++++++++++++++++++++++++++++++ vm/src/obj/objweakref.rs | 2 +- vm/src/pyobject.rs | 9 +++++++ vm/src/stdlib/weakref.rs | 3 ++- 6 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 vm/src/obj/objweakproxy.rs diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 5f577d04f..58a0f929d 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -118,6 +118,7 @@ pub struct ExceptionZoo { pub os_error: PyClassRef, pub overflow_error: PyClassRef, pub permission_error: PyClassRef, + pub reference_error: PyClassRef, pub runtime_error: PyClassRef, pub stop_iteration: PyClassRef, pub syntax_error: PyClassRef, @@ -152,6 +153,7 @@ impl ExceptionZoo { let name_error = create_type("NameError", &type_type, &exception_type); let os_error = create_type("OSError", &type_type, &exception_type); let runtime_error = create_type("RuntimeError", &type_type, &exception_type); + let reference_error = create_type("ReferenceError", &type_type, &exception_type); let stop_iteration = create_type("StopIteration", &type_type, &exception_type); let syntax_error = create_type("SyntaxError", &type_type, &exception_type); let type_error = create_type("TypeError", &type_type, &exception_type); @@ -208,6 +210,7 @@ impl ExceptionZoo { syntax_warning, resource_warning, runtime_warning, + reference_error, user_warning, } } diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index c824e726d..bfe89fd15 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -34,5 +34,6 @@ pub mod objstr; pub mod objsuper; pub mod objtuple; pub mod objtype; +pub mod objweakproxy; pub mod objweakref; pub mod objzip; diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs new file mode 100644 index 000000000..735670f4e --- /dev/null +++ b/vm/src/obj/objweakproxy.rs @@ -0,0 +1,50 @@ +use super::objweakref::PyWeak; +use crate::function::OptionalArg; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[derive(Debug)] +pub struct PyWeakProxy { + weak: PyWeak, +} + +impl PyValue for PyWeakProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.weakproxy_type() + } +} + +pub type PyWeakProxyRef = PyRef; + +impl PyWeakProxyRef { + // TODO callbacks + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyWeakProxy { + weak: PyWeak::downgrade(&referent), + } + .into_ref_with_type(vm, cls) + } + + fn getattr(self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.get_attribute(obj, attr_name), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } +} + +pub fn init(context: &PyContext) { + extend_class!(context, &context.weakproxy_type, { + "__new__" => context.new_rustfunc(PyWeakProxyRef::create), + "__getattr__" => context.new_rustfunc(PyWeakProxyRef::getattr), + }); +} diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 05d6147ce..b3b7e03c9 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -1,9 +1,9 @@ +use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; use crate::pyobject::PyValue; use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; -use crate::function::OptionalArg; use std::rc::{Rc, Weak}; #[derive(Debug)] diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index f369bab4a..fdac31b83 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -51,6 +51,7 @@ use crate::obj::objstr; use crate::obj::objsuper; use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; +use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; @@ -158,6 +159,7 @@ pub struct PyContext { pub module_type: PyClassRef, pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, + pub weakproxy_type: PyClassRef, pub object: PyClassRef, pub exceptions: exceptions::ExceptionZoo, } @@ -255,6 +257,7 @@ impl PyContext { let readonly_property_type = create_type("readonly_property", &type_type, &object_type); let super_type = create_type("super", &type_type, &object_type); let weakref_type = create_type("ref", &type_type, &object_type); + let weakproxy_type = create_type("weakproxy", &type_type, &object_type); let generator_type = create_type("generator", &type_type, &object_type); let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); @@ -361,6 +364,7 @@ impl PyContext { module_type, bound_method_type, weakref_type, + weakproxy_type, type_type, exceptions, }; @@ -396,6 +400,7 @@ impl PyContext { objcode::init(&context); objframe::init(&context); objweakref::init(&context); + objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); exceptions::init(&context); @@ -554,6 +559,10 @@ impl PyContext { self.weakref_type.clone() } + pub fn weakproxy_type(&self) -> PyClassRef { + self.weakproxy_type.clone() + } + pub fn type_type(&self) -> PyClassRef { self.type_type.clone() } diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index b5f7b756a..d4529599e 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -12,6 +12,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "_weakref", { - "ref" => ctx.weakref_type() + "ref" => ctx.weakref_type(), + "proxy" => ctx.weakproxy_type(), }) } From 1612c954a17e38ed3c4f1fdee40096a9eabf468f Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 14 Apr 2019 20:14:54 -0500 Subject: [PATCH 219/223] Add weakproxy test --- tests/snippets/weakrefs.py | 17 ++++++++++++++++- vm/src/builtins.rs | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/snippets/weakrefs.py b/tests/snippets/weakrefs.py index ffc61f427..059ddd000 100644 --- a/tests/snippets/weakrefs.py +++ b/tests/snippets/weakrefs.py @@ -1,4 +1,5 @@ -from _weakref import ref +from _weakref import ref, proxy +from testutils import assert_raises class X: @@ -11,3 +12,17 @@ b = ref(a) assert callable(b) assert b() is a + +class G: + def __init__(self, h): + self.h = h + + +g = G(5) +p = proxy(g) + +assert p.h == 5 + +del g + +assert_raises(ReferenceError, lambda: p.h) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 8f7721609..a96df53e9 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -751,6 +751,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "NameError" => ctx.exceptions.name_error.clone(), "OverflowError" => ctx.exceptions.overflow_error.clone(), "RuntimeError" => ctx.exceptions.runtime_error.clone(), + "ReferenceError" => ctx.exceptions.reference_error.clone(), "NotImplementedError" => ctx.exceptions.not_implemented_error.clone(), "TypeError" => ctx.exceptions.type_error.clone(), "ValueError" => ctx.exceptions.value_error.clone(), From c9b479c43b62ef65bf632bce3d0e78ae9d57395f Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Thu, 11 Apr 2019 09:19:23 +0100 Subject: [PATCH 220/223] Guard for changes in dictionary size during iteration. --- tests/snippets/dict.py | 12 ++++++++++++ vm/src/dictdatatype.rs | 29 +++++++++++++++++++++++------ vm/src/obj/objdict.rs | 27 +++++++++++++++++++-------- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 4c35af2f4..05cdd9928 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -69,6 +69,18 @@ assert (1, 2) == next(items) data[3] = "changed" assert (3, "changed") == next(items) +# But we can't add or delete items during iteration. +d = {} +a = iter(d.items()) +d['a'] = 2 +b = iter(d.items()) +assert ('a', 2) == next(b) +with assertRaises(RuntimeError): + next(a) +del d['a'] +with assertRaises(RuntimeError): + next(b) + # View isn't itself an iterator. with assertRaises(TypeError): next(data.keys()) diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 6dc49125b..1f4b4eecd 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -33,6 +33,12 @@ struct DictEntry { value: T, } +#[derive(Debug)] +pub struct DictSize { + size: usize, + entries_size: usize, +} + impl Dict { pub fn new() -> Self { Dict { @@ -120,17 +126,28 @@ impl Dict { self.len() == 0 } - pub fn next_entry(&self, mut position: usize) -> Option<(usize, &PyObjectRef, &T)> { - while position < self.entries.len() { - if let Some(DictEntry { key, value, .. }) = &self.entries[position] { - return Some((position + 1, key, value)); - } else { - position += 1; + pub fn size(&self) -> DictSize { + DictSize { + size: self.size, + entries_size: self.entries.len(), + } + } + + pub fn next_entry(&self, position: &mut usize) -> Option<(&PyObjectRef, &T)> { + while *position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { + *position += 1; + return Some((key, value)); } + *position += 1; } None } + pub fn has_changed_size(&self, position: &DictSize) -> bool { + position.size != self.size || self.entries.len() != position.entries_size + } + /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { let hash_value = calc_hash(vm, key)?; diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index fd77107ee..7b11dd6c1 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -260,6 +260,10 @@ impl PyDictRef { let key = key.into_pyobject(vm).unwrap(); self.entries.borrow().contains(vm, &key).unwrap() } + + pub fn size(&self) -> dictdatatype::DictSize { + self.entries.borrow().size() + } } impl ItemProtocol for PyDictRef { @@ -315,11 +319,8 @@ impl Iterator for DictIter { type Item = (PyObjectRef, PyObjectRef); fn next(&mut self) -> Option { - match self.dict.entries.borrow().next_entry(self.position) { - Some((new_position, key, value)) => { - self.position = new_position; - Some((key.clone(), value.clone())) - } + match self.dict.entries.borrow().next_entry(&mut self.position) { + Some((key, value)) => Some((key.clone(), value.clone())), None => None, } } @@ -360,6 +361,7 @@ macro_rules! dict_iterator { #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, + pub size: dictdatatype::DictSize, pub position: Cell, } @@ -368,15 +370,24 @@ macro_rules! dict_iterator { fn new(dict: PyDictRef) -> Self { $iter_name { position: Cell::new(0), + size: dict.size(), dict, } } #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { - match self.dict.entries.borrow().next_entry(self.position.get()) { - Some((new_position, key, value)) => { - self.position.set(new_position); + let mut position = self.position.get(); + let dict = self.dict.entries.borrow(); + if dict.has_changed_size(&self.size) { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "dictionary changed size during iteration".to_string(), + )); + } + match dict.next_entry(&mut position) { + Some((key, value)) => { + self.position.set(position); Ok($result_fn(vm, key, value)) } None => Err(objiter::new_stop_iteration(vm)), From dfbaf3cdf01936c5769467e250ced6365863935e Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 15 Apr 2019 13:23:15 +0100 Subject: [PATCH 221/223] Remove pub from PyDict.entries (easy TODO) --- vm/src/obj/objdict.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7b11dd6c1..25df2ed14 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -19,8 +19,7 @@ pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { - // TODO: should be private - pub entries: RefCell, + entries: RefCell, } pub type PyDictRef = PyRef; From 08babef619763a3a69347c890cd448f3fe6cf2ea Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Mon, 15 Apr 2019 20:24:16 -0500 Subject: [PATCH 222/223] Remove the __inside_vm hack for procedural macros --- derive/src/from_args.rs | 27 +++++++++++---------------- derive/src/lib.rs | 37 +------------------------------------ derive/src/pyclass.rs | 17 ++++++----------- vm/src/builtins.rs | 1 - vm/src/lib.rs | 2 ++ vm/src/obj/objbytes.rs | 4 ++-- vm/src/obj/objdict.rs | 8 ++++---- vm/src/obj/objgenerator.rs | 4 ++-- vm/src/obj/objint.rs | 5 ++--- vm/src/obj/objstr.rs | 4 ++-- 10 files changed, 32 insertions(+), 77 deletions(-) diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs index 040ebf36c..d017a32e5 100644 --- a/derive/src/from_args.rs +++ b/derive/src/from_args.rs @@ -1,4 +1,3 @@ -use super::rustpython_path_derive; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; @@ -109,7 +108,7 @@ impl ArgAttribute { } } -fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { +fn generate_field(field: &Field) -> TokenStream2 { let mut pyarg_attrs = field .attrs .iter() @@ -132,7 +131,7 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { let name = &field.ident; let middle = quote! { - .map(|x| #rp_path::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? }; let ending = if let Some(default) = attr.default { quote! { @@ -140,16 +139,16 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { } } else if attr.optional { quote! { - .map(#rp_path::function::OptionalArg::Present) - .unwrap_or(#rp_path::function::OptionalArg::Missing) + .map(::rustpython_vm::function::OptionalArg::Present) + .unwrap_or(::rustpython_vm::function::OptionalArg::Missing) } } else { let err = match attr.kind { ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - #rp_path::function::ArgumentError::TooFewArgs + ::rustpython_vm::function::ArgumentError::TooFewArgs }, ParameterKind::KeywordOnly => quote! { - #rp_path::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) }, }; quote! { @@ -177,14 +176,10 @@ fn generate_field(field: &Field, rp_path: &syn::Path) -> TokenStream2 { } pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { - let rp_path = rustpython_path_derive(&input); let fields = match input.data { Data::Struct(ref data) => { match data.fields { - Fields::Named(ref fields) => fields - .named - .iter() - .map(|field| generate_field(field, &rp_path)), + Fields::Named(ref fields) => fields.named.iter().map(generate_field), Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message } } @@ -193,11 +188,11 @@ pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { let name = &input.ident; quote! { - impl #rp_path::function::FromArgs for #name { + impl ::rustpython_vm::function::FromArgs for #name { fn from_args( - vm: &#rp_path::VirtualMachine, - args: &mut #rp_path::function::PyFuncArgs - ) -> Result { + vm: &::rustpython_vm::VirtualMachine, + args: &mut ::rustpython_vm::function::PyFuncArgs + ) -> Result { Ok(#name { #(#fields)* }) } } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index adde078f0..26dce935a 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,47 +1,12 @@ extern crate proc_macro; use proc_macro::TokenStream; -use quote::quote; use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; mod from_args; mod pyclass; -fn rustpython_path(inside_vm: bool) -> syn::Path { - let path = if inside_vm { - quote!(crate) - } else { - quote!(::rustpython_vm) - }; - syn::parse2(path).unwrap() -} - -/// Does the item have the #[__inside_vm] attribute on it, signifying that the derive target is -/// being derived from inside the `rustpython_vm` crate. -fn rustpython_path_derive(input: &DeriveInput) -> syn::Path { - rustpython_path( - input - .attrs - .iter() - .any(|attr| attr.path.is_ident("__inside_vm")), - ) -} - -fn rustpython_path_attr(attr: &AttributeArgs) -> syn::Path { - rustpython_path(attr.iter().any(|meta| { - if let syn::NestedMeta::Meta(meta) = meta { - if let syn::Meta::Word(ident) = meta { - ident == "__inside_vm" - } else { - false - } - } else { - false - } - })) -} - -#[proc_macro_derive(FromArgs, attributes(__inside_vm, pyarg))] +#[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index e072db32d..36ce9df1a 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,4 +1,3 @@ -use super::rustpython_path_attr; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use std::collections::HashMap; @@ -138,15 +137,13 @@ impl ClassItem { } } -pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { +pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { let mut imp = if let Item::Impl(imp) = item { imp } else { return quote!(#item); }; - let rp_path = rustpython_path_attr(&attr); - let items = imp .items .iter_mut() @@ -199,7 +196,7 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { class.set_str_attr( #name, - #rp_path::obj::objproperty::PropertyBuilder::new(ctx) + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) .add_getter(Self::#getter) #add_setter .create(), @@ -209,10 +206,10 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #imp - impl #rp_path::pyobject::PyClassImpl for #ty { + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { fn impl_extend_class( - ctx: &#rp_path::pyobject::PyContext, - class: &#rp_path::obj::objtype::PyClassRef, + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, ) { #(#methods)* #(#properties)* @@ -228,8 +225,6 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { _ => panic!("#[pyclass] can only be on a struct or enum declaration"), }; - let rp_path = rustpython_path_attr(&attr); - let mut class_name = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { @@ -273,7 +268,7 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { quote! { #item - impl #rp_path::pyobject::PyClassDef for #ident { + impl ::rustpython_vm::pyobject::PyClassDef for #ident { const NAME: &'static str = #class_name; const DOC: Option<&'static str> = #doc; } diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index a96df53e9..f2ca1e4c1 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -530,7 +530,6 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } #[derive(Debug, FromArgs)] -#[__inside_vm] pub struct PrintOptions { #[pyarg(keyword_only, default = "None")] sep: Option, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 578f1d99a..8b14454fe 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -32,6 +32,8 @@ extern crate rustpython_parser; #[macro_use] extern crate rustpython_derive; +extern crate self as rustpython_vm; + pub use rustpython_derive::*; //extern crate eval; use eval::eval::*; diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index fd9bfd5bf..0556de053 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -21,7 +21,7 @@ use super::objtype::PyClassRef; /// - a text string encoded using the specified encoding\n \ /// - any object implementing the buffer API.\n \ /// - an integer"; -#[pyclass(name = "bytes", __inside_vm)] +#[pyclass(name = "bytes")] #[derive(Clone, Debug)] pub struct PyBytes { inner: PyByteInner, @@ -71,7 +71,7 @@ pub fn init(context: &PyContext) { }); } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyBytesRef { #[pymethod(name = "__new__")] fn bytes_new( diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 25df2ed14..e5419e932 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -327,13 +327,13 @@ impl Iterator for DictIter { macro_rules! dict_iterator { ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { - #[pyclass(name = $class_name, __inside_vm)] + #[pyclass(name = $class_name)] #[derive(Debug)] struct $name { pub dict: PyDictRef, } - #[pyimpl(__inside_vm)] + #[pyimpl] impl $name { fn new(dict: PyDictRef) -> Self { $name { dict: dict } @@ -356,7 +356,7 @@ macro_rules! dict_iterator { } } - #[pyclass(name = $iter_class_name, __inside_vm)] + #[pyclass(name = $iter_class_name)] #[derive(Debug)] struct $iter_name { pub dict: PyDictRef, @@ -364,7 +364,7 @@ macro_rules! dict_iterator { pub position: Cell, } - #[pyimpl(__inside_vm)] + #[pyimpl] impl $iter_name { fn new(dict: PyDictRef) -> Self { $iter_name { diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 2541263f1..2c51778fd 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -9,7 +9,7 @@ use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; -#[pyclass(name = "generator", __inside_vm)] +#[pyclass(name = "generator")] #[derive(Debug)] pub struct PyGenerator { frame: FrameRef, @@ -21,7 +21,7 @@ impl PyValue for PyGenerator { } } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { frame }.into_ref(vm) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 52f321adf..20f5de4ca 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -32,7 +32,7 @@ use crate::obj::objtype::PyClassRef; /// Base 0 means to interpret the base from the string as an integer literal. /// >>> int('0b100', base=0) /// 4 -#[pyclass(__inside_vm)] +#[pyclass] #[derive(Debug)] pub struct PyInt { value: BigInt, @@ -111,7 +111,7 @@ impl_try_from_object_int!( (u64, to_u64), ); -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyInt { #[pymethod(name = "__eq__")] fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { @@ -476,7 +476,6 @@ impl PyInt { } #[derive(FromArgs)] -#[__inside_vm] struct IntOptions { #[pyarg(positional_only, optional = true)] val_options: OptionalArg, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 390e92d27..1638a712c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -30,7 +30,7 @@ use super::objtype::{self, PyClassRef}; /// or repr(object). /// encoding defaults to sys.getdefaultencoding(). /// errors defaults to 'strict'." -#[pyclass(name = "str", __inside_vm)] +#[pyclass(name = "str")] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public @@ -74,7 +74,7 @@ impl TryIntoRef for &str { } } -#[pyimpl(__inside_vm)] +#[pyimpl] impl PyString { // TODO: should with following format // class str(object='') From 224863583b5cbba1e17ea9223829fa73a503ecb7 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 16 Apr 2019 19:13:39 +1200 Subject: [PATCH 223/223] Use new_syntax error in main.rs --- src/main.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 750faf98a..2a673736e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,11 +65,8 @@ fn main() { } fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult { - let code_obj = - compile::compile(vm, source, &compile::Mode::Exec, source_path).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })?; + let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path) + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj.borrow()); let vars = vm.ctx.new_scope(); // Keep track of local variables vm.run_code_obj(code_obj, vars) @@ -120,8 +117,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com // Don't inject syntax errors for line continuation Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err), Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - let exc = vm.new_exception(syntax_error, format!("{}", err)); + let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); Err(err) }