Added cool script to test snippets in both implementations

This commit is contained in:
Windel Bouwman committed 2018-07-08 19:57:53 +02:00
1 parent 62cb3e2474
commit ed4557fec5
6 files changed
+310 -87

No files matched your search

+29 -15
View File
@@ -180,21 +180,8 @@ impl<'input> Lexer<'input> {
};
c
}
}
/* Implement iterator pattern for the get_tok function.
Calling the next element in the iterator will yield the next lexical
token.
*/
impl<'input> Iterator for Lexer<'input> {
type Item = Spanned<Tok>;
fn next(&mut self) -> Option<Self::Item> {
// Idea: create some sort of hash map for single char tokens:
// let mut X = HashMap::new();
// X.insert('=', Tok::Equal);
fn inner_next(&mut self) -> Option<Spanned<Tok>> {
// Detect indentation levels
loop {
if self.at_begin_of_line {
@@ -225,7 +212,14 @@ impl<'input> Iterator for Lexer<'input> {
return Some(Ok((0, Tok::Indent, 0)));
} else if col < current_indentation {
// One or more dedentations
// Pop off other levels until col is found:
let l2 = self.indentation_stack.pop().unwrap();
return Some(Ok((0, Tok::Dedent, 0)));
if col == l2 {
// TODO: handle wrong indentations
}
}
}
}
@@ -504,6 +498,25 @@ impl<'input> Iterator for Lexer<'input> {
}
}
/* Implement iterator pattern for the get_tok function.
Calling the next element in the iterator will yield the next lexical
token.
*/
impl<'input> Iterator for Lexer<'input> {
type Item = Spanned<Tok>;
fn next(&mut self) -> Option<Self::Item> {
// Idea: create some sort of hash map for single char tokens:
// let mut X = HashMap::new();
// X.insert('=', Tok::Equal);
let token = self.inner_next();
trace!("Lex token {:?}, nesting={:?}, indent stack: {:?}", token, self.nesting, self.indentation_stack);
token
}
}
#[cfg(test)]
mod tests {
use super::Tok;
@@ -538,7 +551,7 @@ mod tests {
#[test]
fn test_indentation() {
let source = String::from("def foo():\n return 99\n");
let source = String::from("def foo():\n return 99\n\n");
let tokens = lex_source(&source);
assert_eq!(
tokens,
@@ -556,6 +569,7 @@ mod tests {
Tok::Number { value: 99 },
Tok::Newline,
Tok::Dedent,
Tok::Newline,
]
);
}
+29 -10
View File
@@ -14,9 +14,10 @@ import compile_code
logger = logging.getLogger('tests')
THIS_DIR = '.'
TEST_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', 'tests'))
CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(THIS_DIR))
ROOT_DIR = '..'
TEST_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'tests'))
CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object'))
RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR))
@contextlib.contextmanager
@@ -27,8 +28,17 @@ def pushd(path):
os.chdir(old_dir)
def perform_test(filename):
logger.info('Running %s', filename)
def perform_test(filename, method):
logger.info('Running %s via %s', filename, method)
if method == 'cpython':
run_via_cpython(filename)
elif method == 'rustpython':
run_via_rustpython(filename)
else:
raise NotImplementedError(method)
def run_via_cpython(filename):
# Step1: Create bytecode file:
bytecode_filename = filename + '.bytecode'
with open(bytecode_filename, 'w') as f:
@@ -42,15 +52,23 @@ def perform_test(filename):
subprocess.check_call(['cargo', 'run', bytecode_filename], env=env)
def create_test_function(cls, filename):
def run_via_rustpython(filename):
env = os.environ.copy()
env['RUST_LOG'] = 'debug'
env['RUST_BACKTRACE'] = '1'
with pushd(RUSTPYTHON_RUNNER_DIR):
subprocess.check_call(['cargo', 'run', filename], env=env)
def create_test_function(cls, filename, method):
""" Create a test function for a single snippet """
core_test_directory, snippet_filename = os.path.split(filename)
test_function_name = 'test_' \
test_function_name = 'test_{}_'.format(method) \
+ os.path.splitext(snippet_filename)[0] \
.replace('.', '_').replace('-', '_')
def test_function(self):
perform_test(filename)
perform_test(filename, method)
if hasattr(cls, test_function_name):
raise ValueError('Duplicate test case {}'.format(test_function_name))
@@ -59,8 +77,9 @@ def create_test_function(cls, filename):
def populate(cls):
""" Decorator function which can populate a unittest.TestCase class """
for filename in get_test_files():
create_test_function(cls, filename)
for method in ['cpython', 'rustpython']:
for filename in get_test_files():
create_test_function(cls, filename, method)
return cls
+44 -21
View File
@@ -9,14 +9,13 @@ use rustpython_parser::ast;
use rustpython_vm::bytecode::{self, CodeObject, Instruction};
struct Compiler {
code_object: CodeObject,
code_object_stack: Vec<CodeObject>,
nxt_label: usize,
}
pub fn compile(p: ast::Program) -> CodeObject {
let mut compiler = Compiler::new();
compiler.compile_program(p);
compiler.code_object
compiler.compile_program(p)
}
type Label = usize;
@@ -24,13 +23,21 @@ type Label = usize;
impl Compiler {
fn new() -> Self {
Compiler {
code_object: CodeObject::new(),
code_object_stack: Vec::new(),
nxt_label: 0,
}
}
fn compile_program(&mut self, program: ast::Program) {
fn compile_program(&mut self, program: ast::Program) -> CodeObject {
self.code_object_stack.push(CodeObject::new());
self.compile_statements(program.statements);
assert!(self.code_object_stack.len() == 1);
// Emit None at end:
self.emit(Instruction::LoadConst { value: bytecode::Constant::None });
self.emit(Instruction::ReturnValue);
self.code_object_stack.pop().unwrap()
}
fn compile_statements(&mut self, statements: Vec<ast::Statement>) {
@@ -39,20 +46,6 @@ impl Compiler {
}
}
// Generate a new label
fn new_label(&mut self) -> Label {
let l = self.nxt_label;
self.nxt_label += 1;
l
}
// Assign current position the given label
fn set_label(&mut self, label: Label) {
let position = self.code_object.instructions.len();
// assert!(label not in self.label_map)
self.code_object.label_map.insert(label, position);
}
fn compile_statement(&mut self, statement: ast::Statement) {
trace!("Compiling {:?}", statement);
match statement {
@@ -125,7 +118,16 @@ impl Compiler {
self.emit(Instruction::PopBlock);
}
ast::Statement::FunctionDef { name, body } => {
// Create bytecode for this function:
self.code_object_stack.push(CodeObject::new());
self.compile_statements(body);
let code = self.code_object_stack.pop().unwrap();
self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: code } });
self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: name.clone() } });
// Turn code object into function object:
self.emit(Instruction::MakeFunction);
self.emit(Instruction::StoreName { name: name });
}
ast::Statement::ClassDef { name } => {
// TODO?
@@ -136,6 +138,8 @@ impl Compiler {
self.compile_expression(test);
// if true, jump over raise:
let end_label = self.new_label();
self.emit(Instruction::JumpIf { target: end_label } );
self.emit(Instruction::LoadName {
name: String::from("AssertionError"),
@@ -149,7 +153,7 @@ impl Compiler {
self.emit(Instruction::CallFunction { count: 0 });
}
}
// TODO?
self.set_label(end_label);
}
ast::Statement::Break => {
self.emit(Instruction::Break);
@@ -285,7 +289,26 @@ impl Compiler {
}
}
// Low level helper functions:
fn emit(&mut self, instruction: Instruction) {
self.code_object.instructions.push(instruction);
self.current_code_object().instructions.push(instruction);
}
fn current_code_object(&mut self) -> &mut CodeObject {
self.code_object_stack.last_mut().unwrap()
}
// Generate a new label
fn new_label(&mut self) -> Label {
let l = self.nxt_label;
self.nxt_label += 1;
l
}
// Assign current position the given label
fn set_label(&mut self, label: Label) {
let position = self.current_code_object().instructions.len();
// assert!(label not in self.label_map)
self.current_code_object().label_map.insert(label, position);
}
}
+2 -1
View File
@@ -46,7 +46,7 @@ pub enum Instruction {
Break,
Jump { target: Label },
JumpIf { target: Label },
MakeFunction { code: CodeObject },
MakeFunction,
CallFunction { count: usize },
ForIter,
ReturnValue,
@@ -64,6 +64,7 @@ pub enum Constant {
Integer { value: i32 }, // TODO: replace by arbitrary big int math.
// TODO: Float { value: f64 },
String { value: String },
Code { code: CodeObject },
None,
}
+37 -2
View File
@@ -28,6 +28,9 @@ pub enum PyObject {
Integer {
value: i32,
},
Boolean {
value: bool,
},
List {
elements: Vec<PyObjectRef>,
},
@@ -40,11 +43,16 @@ pub enum PyObject {
iterated_obj: PyObjectRef,
},
Slice {
elements: Vec<PyObjectRef>,
start: Option<i32>,
stop: Option<i32>,
step: Option<i32>,
},
Code {
code: bytecode::CodeObject,
},
Function {
code: bytecode::CodeObject,
},
None,
RustFunction {
function: fn(Vec<PyObjectRef>),
@@ -59,10 +67,11 @@ impl PyObjectRef {
}*/
impl PyObject {
pub fn call(&self, args: Vec<PyObjectRef>) {
pub fn call(&self, args: Vec<PyObjectRef>) -> PyObjectRef {
match *self {
PyObject::RustFunction { ref function } => {
function(args);
PyObject::None.into_ref()
}
_ => {
println!("Not impl {:?}", self);
@@ -215,6 +224,32 @@ impl<'a> Mul<&'a PyObject> for &'a PyObject {
}
}
// impl<'a> PartialEq<&'a PyObject> for &'a PyObject {
impl PartialEq for PyObject {
fn eq(&self, other: &PyObject) -> bool {
match (self, other) {
(&PyObject::Integer { value: ref v1i }, &PyObject::Integer { value: ref v2i }) => {
v2i == v1i
},
/*
(&NativeType::Float(ref v1f), &NativeType::Float(ref v2f)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == v1f)));
},
(&NativeType::Str(ref v1s), &NativeType::Str(ref v2s)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2s == v1s)));
},
(&NativeType::Int(ref v1i), &NativeType::Float(ref v2f)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == &(*v1i as f64))));
},
(&NativeType::List(ref l1), &NativeType::List(ref l2)) => {
curr_frame.stack.push(Rc::new(NativeType::Boolean(l2 == l1)));
},
*/
_ => panic!("TypeError in COMPARE_OP: can't compare {:?} with {:?}", self, other)
}
}
}
#[cfg(test)]
mod tests {
use super::PyObject;
+169 -38
View File
@@ -110,6 +110,18 @@ impl VirtualMachine {
self.frames.pop().unwrap();
}
fn push_value(&mut self, obj: PyObjectRef) {
self.current_frame().stack.push(obj);
}
fn pop_value(&mut self) -> PyObjectRef {
self.current_frame().stack.pop().unwrap()
}
fn pop_multiple(&mut self, count: usize) -> Vec<PyObjectRef> {
self.current_frame().pop_multiple(count)
}
fn run(&mut self, code: Rc<bytecode::CodeObject>) {
let frame = Frame::new(code, HashMap::new(), None);
self.run_frame(frame);
@@ -123,13 +135,6 @@ impl VirtualMachine {
//let mut why = None;
// Change this to a loop for jump
loop {
{
let curr_frame = self.current_frame();
if curr_frame.lasti >= curr_frame.code.instructions.len() {
break;
}
}
//while curr_frame.lasti < curr_frame.code.co_code.len() {
let result = self.execute_instruction();
match result {
@@ -156,20 +161,89 @@ impl VirtualMachine {
return_value.into_ref().clone()
}
fn subscript(&mut self, a: &PyObject, b: &PyObject) -> PyObjectRef {
// debug!("tos: {:?}, tos1: {:?}", tos, tos1);
// Subscript implementation: a[b]
match (a, b) {
/*
(&NativeType::List(ref l), &NativeType::Int(ref index)) => {
let pos_index = (index + l.borrow().len() as i32) % l.borrow().len() as i32;
curr_frame.stack.push(Rc::new(l.borrow()[pos_index as usize].clone()))
},
(&NativeType::List(ref l), &NativeType::Slice(ref opt_start, ref opt_stop, ref opt_step)) => {
let start = match opt_start {
&Some(start) => ((start + l.borrow().len() as i32) % l.borrow().len() as i32) as usize,
&None => 0,
};
let stop = match opt_stop {
&Some(stop) => ((stop + l.borrow().len() as i32) % l.borrow().len() as i32) as usize,
&None => l.borrow().len() as usize,
};
let step = match opt_step {
//Some(step) => step as usize,
&None => 1 as usize,
_ => unimplemented!(),
};
// TODO: we could potentially avoid this copy and use slice
curr_frame.stack.push(Rc::new(NativeType::List(RefCell::new(l.borrow()[start..stop].to_vec()))));
},
(&NativeType::Tuple(ref t), &NativeType::Int(ref index)) => curr_frame.stack.push(Rc::new(t[*index as usize].clone())),
(&NativeType::Str(ref s), &NativeType::Int(ref index)) => {
let idx = (index + s.len() as i32) % s.len() as i32;
curr_frame.stack.push(Rc::new(NativeType::Str(s.chars().nth(idx as usize).unwrap().to_string())));
},
*/
(&PyObject::String{ref value}, &PyObject::Slice { ref start, ref stop, ref step }) => {
let start2: usize = match start {
&Some(_) => panic!("Bad start index for string slicing"),
&Some(start) => start as usize,
&None => 0,
};
let stop2: usize = match stop {
&Some(stop) => stop as usize,
&Some(_) => panic!("Bad stop index for string slicing"),
&None => value.len() as usize,
};
let step2: usize = match step {
//Some(step) => step as usize,
&None => 1 as usize,
_ => unimplemented!(),
};
PyObject::String { value: value[start2..stop2].to_string() }.into_ref()
},
// TODO: implement other Slice possibilities
_ => panic!("TypeError: indexing type {:?} with index {:?} is not supported (yet?)", a, b)
}
}
fn execute_binop(&mut self, op: &bytecode::BinaryOperator) {
let frame = self.frames.last_mut().unwrap();
let b_ref = frame.stack.pop().unwrap();
let a_ref = frame.stack.pop().unwrap();
let b_ref = self.pop_value();
let a_ref = self.pop_value();
let b = &*b_ref.borrow();
let a = &*a_ref.borrow();
let result = match op {
&bytecode::BinaryOperator::Subtract => a - b,
&bytecode::BinaryOperator::Add => a + b,
&bytecode::BinaryOperator::Multiply => a * b,
&bytecode::BinaryOperator::Subtract => (a - b).into_ref(),
&bytecode::BinaryOperator::Add => (a + b).into_ref(),
&bytecode::BinaryOperator::Multiply => (a * b).into_ref(),
// &bytecode::BinaryOperator::Div => a / b,
&bytecode::BinaryOperator::Subscript => self.subscript(a, b),
_ => panic!("NOT IMPL {:?}", op),
};
frame.stack.push(result.into_ref());
self.push_value(result);
}
fn execute_compare(&mut self, op: &bytecode::ComparisonOperator) {
let b_ref = self.pop_value();
let a_ref = self.pop_value();
let b = &*b_ref.borrow();
let a = &*a_ref.borrow();
let result_bool = match op {
&bytecode::ComparisonOperator::Equal => (a == b),
&bytecode::ComparisonOperator::NotEqual => (a != b),
_ => panic!("NOT IMPL {:?}", op),
};
let result = PyObject::Boolean { value: result_bool }.into_ref();
self.push_value(result);
}
// Execute a single instruction:
@@ -185,9 +259,9 @@ impl VirtualMachine {
};
{
debug!("stack:{:?}", self.current_frame().stack);
debug!("env :{:?}", self.current_frame().locals);
debug!("Executing op code: {:?}", instruction);
trace!("Executing op code: {:?}", instruction);
trace!(" stack:{:?}", self.current_frame().stack);
trace!(" env :{:?}", self.current_frame().locals);
}
match &instruction {
&bytecode::Instruction::LoadConst { ref value } => {
@@ -195,45 +269,46 @@ impl VirtualMachine {
&bytecode::Constant::Integer { ref value } => { PyObject::Integer { value: *value } },
// &bytecode::Constant::Float
&bytecode::Constant::String { ref value } => { PyObject::String { value: value.clone() } },
&bytecode::Constant::Code { ref code } => { PyObject::Code { code: code.clone() } },
&bytecode::Constant::None => { PyObject::None },
};
self.current_frame().stack.push(obj.into_ref().clone());
self.push_value(obj.into_ref().clone());
None
},
&bytecode::Instruction::LoadName { ref name } => {
// Lookup name in scope and put it onto the stack!
let obj = self.current_frame().locals[name].clone();
self.current_frame().stack.push(obj);
self.push_value(obj);
None
},
&bytecode::Instruction::StoreName { ref name } => {
// take top of stack and assign in scope:
let obj = self.current_frame().stack.pop().unwrap();
let obj = self.pop_value();
self.current_frame().locals.insert(name.clone(), obj);
None
},
&bytecode::Instruction::Pop => {
// Pop value from stack and ignore.
self.current_frame().stack.pop();
self.pop_value();
None
},
&bytecode::Instruction::BuildList { size } => {
let elements = self.current_frame().pop_multiple(size);
let elements = self.pop_multiple(size);
let list_obj = PyObject::List { elements: elements }.into_ref();
self.current_frame().stack.push(list_obj);
self.push_value(list_obj);
None
},
&bytecode::Instruction::BuildTuple { size } => {
let elements = self.current_frame().pop_multiple(size);
let elements = self.pop_multiple(size);
let list_obj = PyObject::Tuple { elements: elements }.into_ref();
self.current_frame().stack.push(list_obj);
self.push_value(list_obj);
None
},
&bytecode::Instruction::BuildMap { size } => {
let mut elements = Vec::new();
for _x in 0..size {
let key = self.current_frame().stack.pop().unwrap();
let obj = self.current_frame().stack.pop().unwrap();
let key = self.pop_value();
let obj = self.pop_value();
elements.push((key,obj));
}
panic!("To be implemented!")
@@ -242,9 +317,24 @@ impl VirtualMachine {
},
&bytecode::Instruction::BuildSlice { size } => {
assert!(size == 2 || size == 3);
let elements = self.current_frame().pop_multiple(size);
let list_obj = PyObject::Slice { elements: elements }.into_ref();
self.current_frame().stack.push(list_obj);
let elements = self.pop_multiple(size);
let mut out: Vec<Option<i32>> = elements.into_iter().map(|x| match &*x.borrow() {
&PyObject::Integer { value } => Some(value),
&PyObject::None => None,
_ => panic!("Expect Int or None as BUILD_SLICE arguments, got {:?}", x),
}).collect();
let start = out[0];
let stop = out[1];
let step = if out.len() == 3 {
out[2]
} else {
None
};
let obj = PyObject::Slice { start, stop, step }.into_ref();
self.push_value(obj);
None
},
&bytecode::Instruction::BinaryOperation { ref op } => {
@@ -255,8 +345,12 @@ impl VirtualMachine {
panic!("TODO");
// self.execute_binop(op);
},
&bytecode::Instruction::CompareOperation { ref op } => {
self.execute_compare(op);
None
},
&bytecode::Instruction::ReturnValue => {
let value = self.current_frame().stack.pop().unwrap();
let value = self.pop_value();
Some(ExeResult::Value { value })
},
/*
@@ -299,11 +393,37 @@ impl VirtualMachine {
}
},
*/
&bytecode::Instruction::MakeFunction => {
let qualified_name = self.pop_value();
let code = self.pop_value();
let code_2 = &*code.borrow();
let code_obj = match code_2 {
&PyObject::Code { ref code } => code.clone(),
_ => panic!("Second item on the stack should be a code object")
};
// pop argc arguments
// argument: name, args, globals
self.push_value(PyObject::Function { code: code_obj }.into_ref());
None
},
&bytecode::Instruction::CallFunction { count } => {
let mut args: Vec<PyObjectRef> = self.current_frame().pop_multiple(count);
let func_ref = self.current_frame().stack.pop().unwrap();
let mut args: Vec<PyObjectRef> = self.pop_multiple(count);
let func_ref = self.pop_value();
let f = func_ref.borrow();// = &*func_ref.borrow();
f.call(args);
let result = match *f {
PyObject::RustFunction { ref function } => {
f.call(args)
}
PyObject::Function { ref code } => {
let frame = Frame::new(Rc::new(code.clone()), HashMap::new(), None);
self.run_frame(frame)
}
_ => {
println!("Not impl {:?}", f);
panic!("Not impl");
}
};
self.push_value(result);
// call_stack.push();
// If a builtin function, then call directly, otherwise, execute it?
// execute(function.code);
@@ -315,12 +435,21 @@ impl VirtualMachine {
}
*/
&bytecode::Instruction::JumpIf { target } => {
let obj = self.current_frame().stack.pop().unwrap();
let obj = self.pop_value();
// TODO: determine if this value is True-ish:
//if *v == NativeType::Boolean(true) {
// curr_frame.lasti = curr_frame.labels.get(target).unwrap().clone();
//}
let result: bool = true;
let x = obj.borrow();
let result: bool = match *x {
PyObject::Boolean { ref value } => {
*value
},
_ => {
panic!("Not impl {:?}", x);
}
};
if result {
self.jump(target);
}
@@ -359,7 +488,9 @@ impl VirtualMachine {
}
fn jump(&mut self, label: bytecode::Label) {
// let current_frame = self.call_stack.last().unwrap();
// self.program_counter = current_frame.label_map[&label];
let current_frame = self.current_frame();
let target_pc = current_frame.code.label_map[&label];
trace!("program counter from {:?} to {:?}", current_frame.lasti, target_pc);
current_frame.lasti = target_pc;
}
}