From acefe3841e2aaf3fca322325eabc9b6de107913e Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Tue, 23 Oct 2018 21:27:40 +0200 Subject: [PATCH] Add initial generator builtin type --- parser/src/ast.rs | 3 +++ parser/src/python.lalrpop | 18 ++++++++++--- vm/src/bytecode.rs | 2 ++ vm/src/compile.rs | 11 ++++++++ vm/src/frame.rs | 7 +++-- vm/src/obj/mod.rs | 1 + vm/src/obj/objgenerator.rs | 53 ++++++++++++++++++++++++++++++++++++++ vm/src/pyobject.rs | 16 ++++++++++++ vm/src/vm.rs | 12 +++++++-- 9 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 vm/src/obj/objgenerator.rs diff --git a/parser/src/ast.rs b/parser/src/ast.rs index c1946125c..a8232c677 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -145,6 +145,9 @@ pub enum Expression { Yield { expression: Option>, }, + YieldFrom { + expression: Box, + }, Compare { a: Box, op: Comparison, diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index d75724741..46f01f945 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -567,11 +567,22 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => { + "yield" => { ast::Expression::Yield { - expression: None, + expression: ex.map(|expr| Box::new( + if expr.len() > 1 { + ast::Expression::Tuple { elements: expr } + } else { + expr.into_iter().next().unwrap() + }) + ) } - } + }, + "yield" "from" => { + ast::Expression::YieldFrom { + expression: Box::new(e), + } + }, }; Test: ast::Expression = { @@ -1000,6 +1011,7 @@ extern { "except" => lexer::Tok::Except, "finally" => lexer::Tok::Finally, "for" => lexer::Tok::For, + "from" => lexer::Tok::From, "if" => lexer::Tok::If, "in" => lexer::Tok::In, "is" => lexer::Tok::Is, diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 92d186f27..36afcd621 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -27,6 +27,7 @@ pub struct CodeObject { pub varkeywords: Option, // **kwargs pub source_path: Option, pub obj_name: String, // Name of the object that created this code object + pub is_generator: bool, } impl CodeObject { @@ -48,6 +49,7 @@ impl CodeObject { varkeywords: varkeywords, source_path: source_path, obj_name: obj_name, + is_generator: false, } } } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 308bda549..771d86a2e 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -896,6 +896,7 @@ impl Compiler { self.emit(Instruction::BuildSlice { size: size }); } ast::Expression::Yield { expression } => { + self.mark_generator(); match expression { Some(expression) => self.compile_expression(expression)?, None => self.emit(Instruction::LoadConst { @@ -904,6 +905,12 @@ impl Compiler { }; self.emit(Instruction::YieldValue); } + ast::Expression::YieldFrom { expression } => { + self.mark_generator(); + self.compile_expression(expression)?; + self.emit(Instruction::YieldValue); + unimplemented!("yield from todo"); + } ast::Expression::True => { self.emit(Instruction::LoadConst { value: bytecode::Constant::Boolean { value: true }, @@ -1138,6 +1145,10 @@ impl Compiler { fn set_source_location(&mut self, location: &ast::Location) { self.current_source_location = location.clone(); } + + fn mark_generator(&mut self) { + self.current_code_object().is_generator = true; + } } #[cfg(test)] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 9a6675a9c..f6eb2265a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -12,10 +12,9 @@ pub enum Block { start: bytecode::Label, end: bytecode::Label, }, - // TODO: Remove this allow once TryExcept is in use - #[allow(dead_code)] - // TODO: Implement try/except blocks - TryExcept { handler: bytecode::Label }, + TryExcept { + handler: bytecode::Label, + }, With { end: bytecode::Label, context_manager: PyObjectRef, diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index d053a0071..67fdccf09 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -3,6 +3,7 @@ pub mod objbytes; pub mod objdict; pub mod objfloat; pub mod objfunction; +pub mod objgenerator; pub mod objint; pub mod objiter; pub mod objlist; diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs new file mode 100644 index 000000000..db506387c --- /dev/null +++ b/vm/src/obj/objgenerator.rs @@ -0,0 +1,53 @@ +/* + * The mythical generator. + */ + +use super::super::frame::Frame; +use super::super::pyobject::{ + AttributeProtocol, PyContext, PyFuncArgs, PyObject, PyObjectKind, PyObjectRef, PyResult, + TypeProtocol, +}; +use super::super::vm::VirtualMachine; +use super::objtype; + +pub fn init(context: &PyContext) { + let ref generator_type = context.generator_type; + generator_type.set_attr("__iter__", context.new_rustfunc(generator_iter)); + generator_type.set_attr("__next__", context.new_rustfunc(generator_next)); + generator_type.set_attr("__send__", context.new_rustfunc(generator_send)); +} + +pub fn new_generator(vm: &mut VirtualMachine, frame: Frame) -> PyResult { + let g = PyObject::new( + PyObjectKind::Generator { frame: frame }, + vm.ctx.generator_type.clone(), + ); + Ok(g) +} + +fn generator_iter(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(o, Some(vm.ctx.generator_type()))]); + Ok(o.clone()) +} + +fn generator_next(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(o, Some(vm.ctx.generator_type()))]); + send(vm, o) +} + +fn generator_send(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(o, Some(vm.ctx.generator_type()))]); + send(vm, o) +} + +fn send(vm: &mut VirtualMachine, _gen: &PyObjectRef) -> PyResult { + /* + TODO + if let PyObjectKind::Generator { frame } = &gen.borrow_mut().kind { + vm.run_frame(frame) + } else { + panic!("Cannot extract frame from non-generator"); + } + */ + Ok(vm.get_none()) +} diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index fd63329f1..2786d009b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,10 +1,12 @@ use super::bytecode; use super::exceptions; +use super::frame::Frame; use super::obj::objbool; use super::obj::objbytes; use super::obj::objdict; use super::obj::objfloat; use super::obj::objfunction; +use super::obj::objgenerator; use super::obj::objint; use super::obj::objiter; use super::obj::objlist; @@ -66,6 +68,7 @@ pub struct PyContext { pub iter_type: PyObjectRef, pub str_type: PyObjectRef, pub function_type: PyObjectRef, + pub generator_type: PyObjectRef, pub module_type: PyObjectRef, pub bound_method_type: PyObjectRef, pub member_descriptor_type: PyObjectRef, @@ -119,6 +122,7 @@ impl PyContext { let module_type = create_type("module", &type_type, &object_type, &dict_type); let function_type = create_type("function", &type_type, &object_type, &dict_type); + let generator_type = create_type("generator", &type_type, &object_type, &dict_type); let bound_method_type = create_type("method", &type_type, &object_type, &dict_type); let member_descriptor_type = create_type("member_descriptor", &type_type, &object_type, &dict_type); @@ -156,6 +160,7 @@ impl PyContext { str_type: str_type, object: object_type, function_type: function_type, + generator_type: generator_type, module_type: module_type, bound_method_type: bound_method_type, member_descriptor_type: member_descriptor_type, @@ -169,6 +174,7 @@ impl PyContext { objobject::init(&context); objdict::init(&context); objfunction::init(&context); + objgenerator::init(&context); objint::init(&context); objfloat::init(&context); objbytes::init(&context); @@ -218,6 +224,11 @@ impl PyContext { pub fn function_type(&self) -> PyObjectRef { self.function_type.clone() } + + pub fn generator_type(&self) -> PyObjectRef { + self.generator_type.clone() + } + pub fn bound_method_type(&self) -> PyObjectRef { self.bound_method_type.clone() } @@ -646,6 +657,9 @@ pub enum PyObjectKind { scope: PyObjectRef, defaults: PyObjectRef, }, + Generator { + frame: Frame, + }, BoundMethod { function: PyObjectRef, object: PyObjectRef, @@ -693,6 +707,7 @@ impl fmt::Debug for PyObjectKind { } => write!(f, "slice"), &PyObjectKind::Code { ref code } => write!(f, "code: {:?}", code), &PyObjectKind::Function { .. } => write!(f, "function"), + &PyObjectKind::Generator { .. } => write!(f, "generator"), &PyObjectKind::BoundMethod { ref function, ref object, @@ -774,6 +789,7 @@ impl PyObject { PyObjectKind::Instance { dict: _ } => format!(""), PyObjectKind::Code { code: _ } => format!(""), PyObjectKind::Function { .. } => format!(""), + PyObjectKind::Generator { .. } => format!(""), PyObjectKind::BoundMethod { .. } => format!(""), PyObjectKind::RustFunction { function: _ } => format!(""), PyObjectKind::Module { ref name, dict: _ } => format!("", name), diff --git a/vm/src/vm.rs b/vm/src/vm.rs index a619c832c..9bd222003 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -14,6 +14,7 @@ use super::bytecode; use super::frame::{copy_code, Block, Frame}; use super::import::import; use super::obj::objbool; +use super::obj::objgenerator; use super::obj::objiter; use super::obj::objlist; use super::obj::objobject; @@ -349,7 +350,7 @@ impl VirtualMachine { } } - fn run_frame(&mut self, frame: Frame) -> PyResult { + pub fn run_frame(&mut self, frame: Frame) -> PyResult { self.frames.push(frame); let filename = if let Some(source_path) = &self.current_frame().code.source_path { source_path.to_string() @@ -690,8 +691,15 @@ impl VirtualMachine { let scope = self.ctx.new_scope(Some(scope.clone())); self.fill_scope_from_args(&code_object, &scope, args, defaults)?; + // Construct frame: let frame = Frame::new(code.clone(), scope); - self.run_frame(frame) + + // If we have a generator, create a new generator + if code_object.is_generator { + objgenerator::new_generator(self, frame) + } else { + self.run_frame(frame) + } } fn fill_scope_from_args(