From ed4557fec5858417325b7ff8ee9048909548b759 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 8 Jul 2018 19:57:53 +0200 Subject: [PATCH] Added cool script to test snippets in both implementations --- parser/src/lexer.rs | 44 ++++--- py_code_object/test_snippets.py | 39 ++++-- src/compile.rs | 65 ++++++---- vm/src/bytecode.rs | 3 +- vm/src/pyobject.rs | 39 +++++- vm/src/vm.rs | 207 ++++++++++++++++++++++++++------ 6 files changed, 310 insertions(+), 87 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 4522aead5..71206b51c 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -180,21 +180,8 @@ impl<'input> Lexer<'input> { }; c } -} - -/* Implement iterator pattern for the get_tok function. - -Calling the next element in the iterator will yield the next lexical -token. -*/ -impl<'input> Iterator for Lexer<'input> { - type Item = Spanned; - - fn next(&mut self) -> Option { - // Idea: create some sort of hash map for single char tokens: - // let mut X = HashMap::new(); - // X.insert('=', Tok::Equal); + fn inner_next(&mut self) -> Option> { // Detect indentation levels loop { if self.at_begin_of_line { @@ -225,7 +212,14 @@ impl<'input> Iterator for Lexer<'input> { return Some(Ok((0, Tok::Indent, 0))); } else if col < current_indentation { // One or more dedentations + // Pop off other levels until col is found: + + let l2 = self.indentation_stack.pop().unwrap(); return Some(Ok((0, Tok::Dedent, 0))); + if col == l2 { + // TODO: handle wrong indentations + } + } } } @@ -504,6 +498,25 @@ impl<'input> Iterator for Lexer<'input> { } } +/* Implement iterator pattern for the get_tok function. + +Calling the next element in the iterator will yield the next lexical +token. +*/ +impl<'input> Iterator for Lexer<'input> { + type Item = Spanned; + + fn next(&mut self) -> Option { + // Idea: create some sort of hash map for single char tokens: + // let mut X = HashMap::new(); + // X.insert('=', Tok::Equal); + let token = self.inner_next(); + trace!("Lex token {:?}, nesting={:?}, indent stack: {:?}", token, self.nesting, self.indentation_stack); + token + } + +} + #[cfg(test)] mod tests { use super::Tok; @@ -538,7 +551,7 @@ mod tests { #[test] fn test_indentation() { - let source = String::from("def foo():\n return 99\n"); + let source = String::from("def foo():\n return 99\n\n"); let tokens = lex_source(&source); assert_eq!( tokens, @@ -556,6 +569,7 @@ mod tests { Tok::Number { value: 99 }, Tok::Newline, Tok::Dedent, + Tok::Newline, ] ); } diff --git a/py_code_object/test_snippets.py b/py_code_object/test_snippets.py index cb8aedda5..6b389db21 100644 --- a/py_code_object/test_snippets.py +++ b/py_code_object/test_snippets.py @@ -14,9 +14,10 @@ import compile_code logger = logging.getLogger('tests') -THIS_DIR = '.' -TEST_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', 'tests')) -CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(THIS_DIR)) +ROOT_DIR = '..' +TEST_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) +CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) +RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) @contextlib.contextmanager @@ -27,8 +28,17 @@ def pushd(path): os.chdir(old_dir) -def perform_test(filename): - logger.info('Running %s', filename) +def perform_test(filename, method): + logger.info('Running %s via %s', filename, method) + if method == 'cpython': + run_via_cpython(filename) + elif method == 'rustpython': + run_via_rustpython(filename) + else: + raise NotImplementedError(method) + + +def run_via_cpython(filename): # Step1: Create bytecode file: bytecode_filename = filename + '.bytecode' with open(bytecode_filename, 'w') as f: @@ -42,15 +52,23 @@ def perform_test(filename): subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) -def create_test_function(cls, filename): +def run_via_rustpython(filename): + env = os.environ.copy() + env['RUST_LOG'] = 'debug' + env['RUST_BACKTRACE'] = '1' + with pushd(RUSTPYTHON_RUNNER_DIR): + subprocess.check_call(['cargo', 'run', filename], env=env) + + +def create_test_function(cls, filename, method): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) - test_function_name = 'test_' \ + test_function_name = 'test_{}_'.format(method) \ + os.path.splitext(snippet_filename)[0] \ .replace('.', '_').replace('-', '_') def test_function(self): - perform_test(filename) + perform_test(filename, method) if hasattr(cls, test_function_name): raise ValueError('Duplicate test case {}'.format(test_function_name)) @@ -59,8 +77,9 @@ def create_test_function(cls, filename): def populate(cls): """ Decorator function which can populate a unittest.TestCase class """ - for filename in get_test_files(): - create_test_function(cls, filename) + for method in ['cpython', 'rustpython']: + for filename in get_test_files(): + create_test_function(cls, filename, method) return cls diff --git a/src/compile.rs b/src/compile.rs index 264cdf5f9..e1bcf4cb3 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -9,14 +9,13 @@ use rustpython_parser::ast; use rustpython_vm::bytecode::{self, CodeObject, Instruction}; struct Compiler { - code_object: CodeObject, + code_object_stack: Vec, nxt_label: usize, } pub fn compile(p: ast::Program) -> CodeObject { let mut compiler = Compiler::new(); - compiler.compile_program(p); - compiler.code_object + compiler.compile_program(p) } type Label = usize; @@ -24,13 +23,21 @@ type Label = usize; impl Compiler { fn new() -> Self { Compiler { - code_object: CodeObject::new(), + code_object_stack: Vec::new(), nxt_label: 0, } } - fn compile_program(&mut self, program: ast::Program) { + fn compile_program(&mut self, program: ast::Program) -> CodeObject { + self.code_object_stack.push(CodeObject::new()); self.compile_statements(program.statements); + assert!(self.code_object_stack.len() == 1); + + // Emit None at end: + self.emit(Instruction::LoadConst { value: bytecode::Constant::None }); + self.emit(Instruction::ReturnValue); + + self.code_object_stack.pop().unwrap() } fn compile_statements(&mut self, statements: Vec) { @@ -39,20 +46,6 @@ impl Compiler { } } - // Generate a new label - fn new_label(&mut self) -> Label { - let l = self.nxt_label; - self.nxt_label += 1; - l - } - - // Assign current position the given label - fn set_label(&mut self, label: Label) { - let position = self.code_object.instructions.len(); - // assert!(label not in self.label_map) - self.code_object.label_map.insert(label, position); - } - fn compile_statement(&mut self, statement: ast::Statement) { trace!("Compiling {:?}", statement); match statement { @@ -125,7 +118,16 @@ impl Compiler { self.emit(Instruction::PopBlock); } ast::Statement::FunctionDef { name, body } => { + // Create bytecode for this function: + self.code_object_stack.push(CodeObject::new()); self.compile_statements(body); + let code = self.code_object_stack.pop().unwrap(); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: code } }); + self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: name.clone() } }); + + // Turn code object into function object: + self.emit(Instruction::MakeFunction); + self.emit(Instruction::StoreName { name: name }); } ast::Statement::ClassDef { name } => { // TODO? @@ -136,6 +138,8 @@ impl Compiler { self.compile_expression(test); // if true, jump over raise: + let end_label = self.new_label(); + self.emit(Instruction::JumpIf { target: end_label } ); self.emit(Instruction::LoadName { name: String::from("AssertionError"), @@ -149,7 +153,7 @@ impl Compiler { self.emit(Instruction::CallFunction { count: 0 }); } } - // TODO? + self.set_label(end_label); } ast::Statement::Break => { self.emit(Instruction::Break); @@ -285,7 +289,26 @@ impl Compiler { } } + // Low level helper functions: fn emit(&mut self, instruction: Instruction) { - self.code_object.instructions.push(instruction); + self.current_code_object().instructions.push(instruction); + } + + fn current_code_object(&mut self) -> &mut CodeObject { + self.code_object_stack.last_mut().unwrap() + } + + // Generate a new label + fn new_label(&mut self) -> Label { + let l = self.nxt_label; + self.nxt_label += 1; + l + } + + // Assign current position the given label + fn set_label(&mut self, label: Label) { + let position = self.current_code_object().instructions.len(); + // assert!(label not in self.label_map) + self.current_code_object().label_map.insert(label, position); } } diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index faac634d2..f8dd5be87 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -46,7 +46,7 @@ pub enum Instruction { Break, Jump { target: Label }, JumpIf { target: Label }, - MakeFunction { code: CodeObject }, + MakeFunction, CallFunction { count: usize }, ForIter, ReturnValue, @@ -64,6 +64,7 @@ pub enum Constant { Integer { value: i32 }, // TODO: replace by arbitrary big int math. // TODO: Float { value: f64 }, String { value: String }, + Code { code: CodeObject }, None, } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index b23fa2f48..c2e29831c 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -28,6 +28,9 @@ pub enum PyObject { Integer { value: i32, }, + Boolean { + value: bool, + }, List { elements: Vec, }, @@ -40,11 +43,16 @@ pub enum PyObject { iterated_obj: PyObjectRef, }, Slice { - elements: Vec, + start: Option, + stop: Option, + step: Option, }, Code { code: bytecode::CodeObject, }, + Function { + code: bytecode::CodeObject, + }, None, RustFunction { function: fn(Vec), @@ -59,10 +67,11 @@ impl PyObjectRef { }*/ impl PyObject { - pub fn call(&self, args: Vec) { + pub fn call(&self, args: Vec) -> PyObjectRef { match *self { PyObject::RustFunction { ref function } => { function(args); + PyObject::None.into_ref() } _ => { println!("Not impl {:?}", self); @@ -215,6 +224,32 @@ impl<'a> Mul<&'a PyObject> for &'a PyObject { } } +// impl<'a> PartialEq<&'a PyObject> for &'a PyObject { +impl PartialEq for PyObject { + fn eq(&self, other: &PyObject) -> bool { + match (self, other) { + (&PyObject::Integer { value: ref v1i }, &PyObject::Integer { value: ref v2i }) => { + v2i == v1i + }, + /* + (&NativeType::Float(ref v1f), &NativeType::Float(ref v2f)) => { + curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == v1f))); + }, + (&NativeType::Str(ref v1s), &NativeType::Str(ref v2s)) => { + curr_frame.stack.push(Rc::new(NativeType::Boolean(v2s == v1s))); + }, + (&NativeType::Int(ref v1i), &NativeType::Float(ref v2f)) => { + curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == &(*v1i as f64)))); + }, + (&NativeType::List(ref l1), &NativeType::List(ref l2)) => { + curr_frame.stack.push(Rc::new(NativeType::Boolean(l2 == l1))); + }, + */ + _ => panic!("TypeError in COMPARE_OP: can't compare {:?} with {:?}", self, other) + } + } +} + #[cfg(test)] mod tests { use super::PyObject; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index c45fe8b6b..a045a4d7f 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -110,6 +110,18 @@ impl VirtualMachine { self.frames.pop().unwrap(); } + fn push_value(&mut self, obj: PyObjectRef) { + self.current_frame().stack.push(obj); + } + + fn pop_value(&mut self) -> PyObjectRef { + self.current_frame().stack.pop().unwrap() + } + + fn pop_multiple(&mut self, count: usize) -> Vec { + self.current_frame().pop_multiple(count) + } + fn run(&mut self, code: Rc) { let frame = Frame::new(code, HashMap::new(), None); self.run_frame(frame); @@ -123,13 +135,6 @@ impl VirtualMachine { //let mut why = None; // Change this to a loop for jump loop { - { - let curr_frame = self.current_frame(); - if curr_frame.lasti >= curr_frame.code.instructions.len() { - break; - } - } - //while curr_frame.lasti < curr_frame.code.co_code.len() { let result = self.execute_instruction(); match result { @@ -156,20 +161,89 @@ impl VirtualMachine { return_value.into_ref().clone() } + fn subscript(&mut self, a: &PyObject, b: &PyObject) -> PyObjectRef { + // debug!("tos: {:?}, tos1: {:?}", tos, tos1); + // Subscript implementation: a[b] + match (a, b) { + /* + (&NativeType::List(ref l), &NativeType::Int(ref index)) => { + let pos_index = (index + l.borrow().len() as i32) % l.borrow().len() as i32; + curr_frame.stack.push(Rc::new(l.borrow()[pos_index as usize].clone())) + }, + (&NativeType::List(ref l), &NativeType::Slice(ref opt_start, ref opt_stop, ref opt_step)) => { + let start = match opt_start { + &Some(start) => ((start + l.borrow().len() as i32) % l.borrow().len() as i32) as usize, + &None => 0, + }; + let stop = match opt_stop { + &Some(stop) => ((stop + l.borrow().len() as i32) % l.borrow().len() as i32) as usize, + &None => l.borrow().len() as usize, + }; + let step = match opt_step { + //Some(step) => step as usize, + &None => 1 as usize, + _ => unimplemented!(), + }; + // TODO: we could potentially avoid this copy and use slice + curr_frame.stack.push(Rc::new(NativeType::List(RefCell::new(l.borrow()[start..stop].to_vec())))); + }, + (&NativeType::Tuple(ref t), &NativeType::Int(ref index)) => curr_frame.stack.push(Rc::new(t[*index as usize].clone())), + (&NativeType::Str(ref s), &NativeType::Int(ref index)) => { + let idx = (index + s.len() as i32) % s.len() as i32; + curr_frame.stack.push(Rc::new(NativeType::Str(s.chars().nth(idx as usize).unwrap().to_string()))); + }, + */ + (&PyObject::String{ref value}, &PyObject::Slice { ref start, ref stop, ref step }) => { + let start2: usize = match start { + &Some(_) => panic!("Bad start index for string slicing"), + &Some(start) => start as usize, + &None => 0, + }; + let stop2: usize = match stop { + &Some(stop) => stop as usize, + &Some(_) => panic!("Bad stop index for string slicing"), + &None => value.len() as usize, + }; + let step2: usize = match step { + //Some(step) => step as usize, + &None => 1 as usize, + _ => unimplemented!(), + }; + PyObject::String { value: value[start2..stop2].to_string() }.into_ref() + }, + // TODO: implement other Slice possibilities + _ => panic!("TypeError: indexing type {:?} with index {:?} is not supported (yet?)", a, b) + } + } + fn execute_binop(&mut self, op: &bytecode::BinaryOperator) { - let frame = self.frames.last_mut().unwrap(); - let b_ref = frame.stack.pop().unwrap(); - let a_ref = frame.stack.pop().unwrap(); + let b_ref = self.pop_value(); + let a_ref = self.pop_value(); let b = &*b_ref.borrow(); let a = &*a_ref.borrow(); let result = match op { - &bytecode::BinaryOperator::Subtract => a - b, - &bytecode::BinaryOperator::Add => a + b, - &bytecode::BinaryOperator::Multiply => a * b, + &bytecode::BinaryOperator::Subtract => (a - b).into_ref(), + &bytecode::BinaryOperator::Add => (a + b).into_ref(), + &bytecode::BinaryOperator::Multiply => (a * b).into_ref(), // &bytecode::BinaryOperator::Div => a / b, + &bytecode::BinaryOperator::Subscript => self.subscript(a, b), _ => panic!("NOT IMPL {:?}", op), }; - frame.stack.push(result.into_ref()); + self.push_value(result); + } + + fn execute_compare(&mut self, op: &bytecode::ComparisonOperator) { + let b_ref = self.pop_value(); + let a_ref = self.pop_value(); + let b = &*b_ref.borrow(); + let a = &*a_ref.borrow(); + let result_bool = match op { + &bytecode::ComparisonOperator::Equal => (a == b), + &bytecode::ComparisonOperator::NotEqual => (a != b), + _ => panic!("NOT IMPL {:?}", op), + }; + let result = PyObject::Boolean { value: result_bool }.into_ref(); + self.push_value(result); } // Execute a single instruction: @@ -185,9 +259,9 @@ impl VirtualMachine { }; { - debug!("stack:{:?}", self.current_frame().stack); - debug!("env :{:?}", self.current_frame().locals); - debug!("Executing op code: {:?}", instruction); + trace!("Executing op code: {:?}", instruction); + trace!(" stack:{:?}", self.current_frame().stack); + trace!(" env :{:?}", self.current_frame().locals); } match &instruction { &bytecode::Instruction::LoadConst { ref value } => { @@ -195,45 +269,46 @@ impl VirtualMachine { &bytecode::Constant::Integer { ref value } => { PyObject::Integer { value: *value } }, // &bytecode::Constant::Float &bytecode::Constant::String { ref value } => { PyObject::String { value: value.clone() } }, + &bytecode::Constant::Code { ref code } => { PyObject::Code { code: code.clone() } }, &bytecode::Constant::None => { PyObject::None }, }; - self.current_frame().stack.push(obj.into_ref().clone()); + self.push_value(obj.into_ref().clone()); None }, &bytecode::Instruction::LoadName { ref name } => { // Lookup name in scope and put it onto the stack! let obj = self.current_frame().locals[name].clone(); - self.current_frame().stack.push(obj); + self.push_value(obj); None }, &bytecode::Instruction::StoreName { ref name } => { // take top of stack and assign in scope: - let obj = self.current_frame().stack.pop().unwrap(); + let obj = self.pop_value(); self.current_frame().locals.insert(name.clone(), obj); None }, &bytecode::Instruction::Pop => { // Pop value from stack and ignore. - self.current_frame().stack.pop(); + self.pop_value(); None }, &bytecode::Instruction::BuildList { size } => { - let elements = self.current_frame().pop_multiple(size); + let elements = self.pop_multiple(size); let list_obj = PyObject::List { elements: elements }.into_ref(); - self.current_frame().stack.push(list_obj); + self.push_value(list_obj); None }, &bytecode::Instruction::BuildTuple { size } => { - let elements = self.current_frame().pop_multiple(size); + let elements = self.pop_multiple(size); let list_obj = PyObject::Tuple { elements: elements }.into_ref(); - self.current_frame().stack.push(list_obj); + self.push_value(list_obj); None }, &bytecode::Instruction::BuildMap { size } => { let mut elements = Vec::new(); for _x in 0..size { - let key = self.current_frame().stack.pop().unwrap(); - let obj = self.current_frame().stack.pop().unwrap(); + let key = self.pop_value(); + let obj = self.pop_value(); elements.push((key,obj)); } panic!("To be implemented!") @@ -242,9 +317,24 @@ impl VirtualMachine { }, &bytecode::Instruction::BuildSlice { size } => { assert!(size == 2 || size == 3); - let elements = self.current_frame().pop_multiple(size); - let list_obj = PyObject::Slice { elements: elements }.into_ref(); - self.current_frame().stack.push(list_obj); + let elements = self.pop_multiple(size); + + let mut out: Vec> = elements.into_iter().map(|x| match &*x.borrow() { + &PyObject::Integer { value } => Some(value), + &PyObject::None => None, + _ => panic!("Expect Int or None as BUILD_SLICE arguments, got {:?}", x), + }).collect(); + + let start = out[0]; + let stop = out[1]; + let step = if out.len() == 3 { + out[2] + } else { + None + }; + + let obj = PyObject::Slice { start, stop, step }.into_ref(); + self.push_value(obj); None }, &bytecode::Instruction::BinaryOperation { ref op } => { @@ -255,8 +345,12 @@ impl VirtualMachine { panic!("TODO"); // self.execute_binop(op); }, + &bytecode::Instruction::CompareOperation { ref op } => { + self.execute_compare(op); + None + }, &bytecode::Instruction::ReturnValue => { - let value = self.current_frame().stack.pop().unwrap(); + let value = self.pop_value(); Some(ExeResult::Value { value }) }, /* @@ -299,11 +393,37 @@ impl VirtualMachine { } }, */ + &bytecode::Instruction::MakeFunction => { + let qualified_name = self.pop_value(); + let code = self.pop_value(); + let code_2 = &*code.borrow(); + let code_obj = match code_2 { + &PyObject::Code { ref code } => code.clone(), + _ => panic!("Second item on the stack should be a code object") + }; + // pop argc arguments + // argument: name, args, globals + self.push_value(PyObject::Function { code: code_obj }.into_ref()); + None + }, &bytecode::Instruction::CallFunction { count } => { - let mut args: Vec = self.current_frame().pop_multiple(count); - let func_ref = self.current_frame().stack.pop().unwrap(); + let mut args: Vec = self.pop_multiple(count); + let func_ref = self.pop_value(); let f = func_ref.borrow();// = &*func_ref.borrow(); - f.call(args); + let result = match *f { + PyObject::RustFunction { ref function } => { + f.call(args) + } + PyObject::Function { ref code } => { + let frame = Frame::new(Rc::new(code.clone()), HashMap::new(), None); + self.run_frame(frame) + } + _ => { + println!("Not impl {:?}", f); + panic!("Not impl"); + } + }; + self.push_value(result); // call_stack.push(); // If a builtin function, then call directly, otherwise, execute it? // execute(function.code); @@ -315,12 +435,21 @@ impl VirtualMachine { } */ &bytecode::Instruction::JumpIf { target } => { - let obj = self.current_frame().stack.pop().unwrap(); + let obj = self.pop_value(); // TODO: determine if this value is True-ish: //if *v == NativeType::Boolean(true) { // curr_frame.lasti = curr_frame.labels.get(target).unwrap().clone(); //} - let result: bool = true; + let x = obj.borrow(); + let result: bool = match *x { + PyObject::Boolean { ref value } => { + *value + }, + _ => { + panic!("Not impl {:?}", x); + } + + }; if result { self.jump(target); } @@ -359,7 +488,9 @@ impl VirtualMachine { } fn jump(&mut self, label: bytecode::Label) { - // let current_frame = self.call_stack.last().unwrap(); - // self.program_counter = current_frame.label_map[&label]; + let current_frame = self.current_frame(); + let target_pc = current_frame.code.label_map[&label]; + trace!("program counter from {:?} to {:?}", current_frame.lasti, target_pc); + current_frame.lasti = target_pc; } }