diff --git a/parser/src/ast.rs b/parser/src/ast.rs index a84d9ac2c..c1f952298 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -2,12 +2,9 @@ * Implement abstract syntax tree nodes for the python language. */ +pub use super::lexer::Location; /* #[derive(Debug)] -pub struct Location { - pub row: i32, - pub column: i32, -} #[derive(Debug)] pub struct Node { @@ -29,7 +26,13 @@ pub struct SingleImport { } #[derive(Debug, PartialEq)] -pub enum Statement { +pub struct Statement { + pub location: Location, + pub statement: StatementType, +} + +#[derive(Debug, PartialEq)] +pub enum StatementType { Break, Continue, Return { diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index a78a2fdc0..684593a39 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -18,7 +18,7 @@ pub enum LexicalError { StringError, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Location { row: usize, column: usize, @@ -49,6 +49,9 @@ impl<'input> Lexer<'input> { }; lxr.next_char(); lxr.next_char(); + // Start at top row (=1) left column (=1) + lxr.location.row = 1; + lxr.location.column = 1; lxr } @@ -137,9 +140,11 @@ impl<'input> Lexer<'input> { self.next_char(); match self.chr0 { Some('\n') => { + self.new_line(); return; } Some('\r') => { + self.new_line(); return; } Some(_) => {} @@ -268,6 +273,11 @@ impl<'input> Lexer<'input> { self.location.clone() } + fn new_line(&mut self) { + self.location.row += 1; + self.location.column = 1; + } + fn inner_next(&mut self) -> Option> { if !self.pending.is_empty() { return Some(self.pending.remove(0)); @@ -299,12 +309,14 @@ impl<'input> Lexer<'input> { self.next_char(); } self.at_begin_of_line = true; + self.new_line(); continue 'top_loop; } Some('\n') => { // Empty line! self.next_char(); self.at_begin_of_line = true; + self.new_line(); continue 'top_loop; } _ => { @@ -666,6 +678,7 @@ impl<'input> Lexer<'input> { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); + self.new_line(); // Depending on the nesting level, we emit newline or not: if self.nesting == 0 { @@ -679,6 +692,7 @@ impl<'input> Lexer<'input> { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); + self.new_line(); // Depending on the nesting level, we emit newline or not: if self.nesting == 0 { diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 333e91997..550d87b95 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -88,14 +88,17 @@ mod tests { assert_eq!( parse_ast, ast::Program { - statements: vec![ast::Statement::Expression { - expression: ast::Expression::Call { - function: Box::new(ast::Expression::Identifier { - name: String::from("print"), - }), - args: vec![ast::Expression::String { - value: String::from("Hello world"), - },], + statements: vec![ast::Statement { + location: ast::Location::new(1, 1), + statement: ast::StatementType::Expression { + expression: ast::Expression::Call { + function: Box::new(ast::Expression::Identifier { + name: String::from("print"), + }), + args: vec![ast::Expression::String { + value: String::from("Hello world"), + },], + }, }, },], } @@ -109,19 +112,22 @@ mod tests { assert_eq!( parse_ast, ast::Program { - statements: vec![ast::Statement::Expression { - expression: ast::Expression::Call { - function: Box::new(ast::Expression::Identifier { - name: String::from("print"), - }), - args: vec![ - ast::Expression::String { - value: String::from("Hello world"), - }, - ast::Expression::Number { - value: ast::Number::Integer { value: 2 }, - }, - ], + statements: vec![ast::Statement { + location: ast::Location::new(1, 1), + statement: ast::StatementType::Expression { + expression: ast::Expression::Call { + function: Box::new(ast::Expression::Identifier { + name: String::from("print"), + }), + args: vec![ + ast::Expression::String { + value: String::from("Hello world"), + }, + ast::Expression::Number { + value: ast::Number::Integer { value: 2 }, + }, + ], + }, }, },], } @@ -134,30 +140,45 @@ mod tests { let parse_ast = parse_statement(&source).unwrap(); assert_eq!( parse_ast, - ast::Statement::If { - test: ast::Expression::Number { - value: ast::Number::Integer { value: 1 }, - }, - body: vec![ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { value: 10 }, - }, - },], - orelse: Some(vec![ast::Statement::If { + ast::Statement { + location: ast::Location::new(1, 1), + statement: ast::StatementType::If { test: ast::Expression::Number { - value: ast::Number::Integer { value: 2 }, + value: ast::Number::Integer { value: 1 }, }, - body: vec![ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { value: 20 }, + body: vec![ast::Statement { + location: ast::Location::new(1, 7), + statement: ast::StatementType::Expression { + expression: ast::Expression::Number { + value: ast::Number::Integer { value: 10 }, + } }, },], - orelse: Some(vec![ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { value: 30 }, - }, + orelse: Some(vec![ast::Statement { + location: ast::Location::new(2, 1), + statement: ast::StatementType::If { + test: ast::Expression::Number { + value: ast::Number::Integer { value: 2 }, + }, + body: vec![ast::Statement { + location: ast::Location::new(2, 9), + statement: ast::StatementType::Expression { + expression: ast::Expression::Number { + value: ast::Number::Integer { value: 20 }, + }, + }, + },], + orelse: Some(vec![ast::Statement { + location: ast::Location::new(3, 7), + statement: ast::StatementType::Expression { + expression: ast::Expression::Number { + value: ast::Number::Integer { value: 30 }, + }, + }, + },]), + } },]), - },]), + } } ); } @@ -168,18 +189,21 @@ mod tests { let parse_ast = parse_statement(&source); assert_eq!( parse_ast, - Ok(ast::Statement::Expression { - expression: ast::Expression::Lambda { - args: vec![String::from("x"), String::from("y")], - body: Box::new(ast::Expression::Binop { - a: Box::new(ast::Expression::Identifier { - name: String::from("x"), - }), - op: ast::Operator::Mult, - b: Box::new(ast::Expression::Identifier { - name: String::from("y"), + Ok(ast::Statement { + location: ast::Location::new(1, 1), + statement: ast::StatementType::Expression { + expression: ast::Expression::Lambda { + args: vec![String::from("x"), String::from("y")], + body: Box::new(ast::Expression::Binop { + a: Box::new(ast::Expression::Identifier { + name: String::from("x"), + }), + op: ast::Operator::Mult, + b: Box::new(ast::Expression::Identifier { + name: String::from("y"), + }) }) - }) + } } }) ) @@ -190,14 +214,23 @@ mod tests { let source = String::from("class Foo(A, B):\n def __init__(self):\n pass\n"); assert_eq!( parse_statement(&source), - Ok(ast::Statement::ClassDef { - name: String::from("Foo"), - args: vec![String::from("A"), String::from("B")], - body: vec![ast::Statement::FunctionDef { - name: String::from("__init__"), - args: vec![String::from("self")], - body: vec![ast::Statement::Pass], - }], + Ok(ast::Statement { + location: ast::Location::new(1, 1), + statement: ast::StatementType::ClassDef { + name: String::from("Foo"), + args: vec![String::from("A"), String::from("B")], + body: vec![ast::Statement { + location: ast::Location::new(2, 2), + statement: ast::StatementType::FunctionDef { + name: String::from("__init__"), + args: vec![String::from("self")], + body: vec![ast::Statement { + location: ast::Location::new(3, 3), + statement: ast::StatementType::Pass, + }], + } + }], + } }) ) } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index c36638aab..b02d635e4 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -34,14 +34,19 @@ SimpleStatement: ast::Statement = { SmallStatement: ast::Statement = { // => ast::Statement::Expression { expression: e }, ExpressionStatement, - "pass" => ast::Statement::Pass, + "pass" => { + ast::Statement { + location: loc, + statement: ast::StatementType::Pass, + } + }, FlowStatement, ImportStatement, AssertStatement, }; ExpressionStatement: ast::Statement = { - => { + => { //match e2 { // None => ast::Statement::Expression { expression: e }, // Some(e3) => ast::Statement::Expression { expression: e }, @@ -52,21 +57,30 @@ ExpressionStatement: ast::Statement = { let rhs = e2.into_iter().next().unwrap(); // ast::Expression::Tuple { elements: e2.into_iter().next().unwrap() let v = rhs.into_iter().next().unwrap(); - let lhs = ast::Statement::Assign { targets: e, value: v }; + let lhs = ast::Statement { + location: loc.clone(), + statement: ast::StatementType::Assign { targets: e, value: v }, + }; lhs } else { if e.len() > 1 { panic!("Not good?"); // ast::Statement::Expression { expression: e[0] } } else { - ast::Statement::Expression { expression: e.into_iter().next().unwrap() } + ast::Statement { + location: loc.clone(), + statement: ast::StatementType::Expression { expression: e.into_iter().next().unwrap() }, + } } } }, - => { + => { // TODO: this works in most cases: let rhs = e2.into_iter().next().unwrap(); - ast::Statement::AugAssign { target: e1, op: op, value: rhs } + ast::Statement { + location: loc, + statement: ast::StatementType::AugAssign { target: e1, op: op, value: rhs }, + } }, }; @@ -91,16 +105,33 @@ AugAssign: ast::Operator = { }; FlowStatement: ast::Statement = { - "break" => ast::Statement::Break, - "continue" => ast::Statement::Continue, - "return" => ast::Statement::Return { value: t}, + "break" => { + ast::Statement { + location: loc, + statement: ast::StatementType::Break, + } + }, + "continue" => { + ast::Statement { + location: loc, + statement: ast::StatementType::Continue, + } + }, + "return" => { + ast::Statement { + location: loc, + statement: ast::StatementType::Return { value: t}, + } + }, // raise // yield }; ImportStatement: ast::Statement = { - "import" >>> => { - ast::Statement::Import { + "import" >>> => { + ast::Statement { + location: loc, + statement: ast::StatementType::Import { import_parts: i .iter() .map(|(n, a)| @@ -110,10 +141,13 @@ ImportStatement: ast::Statement = { alias: a.clone() }) .collect() + }, } }, - "from" "import" >> => { - ast::Statement::Import { + "from" "import" >> => { + ast::Statement { + location: loc, + statement: ast::StatementType::Import { import_parts: i .iter() .map(|(i, a)| @@ -123,6 +157,7 @@ ImportStatement: ast::Statement = { alias: a.clone() }) .collect() + }, } }, }; @@ -137,11 +172,16 @@ DottedName: String = { }; AssertStatement: ast::Statement = { - "assert" => ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, + "assert" => { + ast::Statement { + location: loc, + statement: ast::StatementType::Assert { + test: t, + msg: match m { + Some(e) => Some(e.1), + None => None, + } + } } }, }; @@ -156,7 +196,7 @@ CompoundStatement: ast::Statement = { }; IfStatement: ast::Statement = { - "if" ":" => { + "if" ":" => { // Determine last else: let mut last = match s3 { Some(s) => Some(s.2), @@ -165,39 +205,62 @@ IfStatement: ast::Statement = { // handle elif: for i in s2.into_iter().rev() { - last = Some(vec![ast::Statement::If { test: i.1, body: i.3, orelse: last }]); + let x = ast::Statement { + location: i.0, + statement: ast::StatementType::If { test: i.2, body: i.4, orelse: last }, + }; + last = Some(vec![x]); } - ast::Statement::If { test: t, body: s1, orelse: last } + ast::Statement { + location: loc, + statement: ast::StatementType::If { test: t, body: s1, orelse: last } + } }, }; WhileStatement: ast::Statement = { - "while" ":" => { + "while" ":" => { let or_else = match s2 { Some(s) => Some(s.2), None => None, }; - ast::Statement::While { test: e, body: s, orelse: or_else } + ast::Statement { + location: loc, + statement: ast::StatementType::While { test: e, body: s, orelse: or_else }, + } }, }; ForStatement: ast::Statement = { - "for" "in" ":" => { + "for" "in" ":" => { let or_else = match s2 { Some(s) => Some(s.2), None => None, }; - ast::Statement::For { target: e, iter: t, body: s, orelse: or_else } + ast::Statement { + location: loc, + statement: ast::StatementType::For { target: e, iter: t, body: s, orelse: or_else }, + } }, }; WithStatement: ast::Statement = { - "with" "as" <_e:Expression> ":" => ast::Statement::With { items: t, body: s }, + "with" "as" <_e:Expression> ":" => { + ast::Statement { + location: loc, + statement: ast::StatementType::With { items: t, body: s }, + } + }, }; FuncDef: ast::Statement = { - "def" ":" => ast::Statement::FunctionDef { name: i, args: a, body: s }, + "def" ":" => { + ast::Statement { + location: loc, + statement: ast::StatementType::FunctionDef { name: i, args: a, body: s } + } + }, }; Parameters: Vec = { @@ -209,10 +272,16 @@ TypedArgsList: Vec = { }; ClassDef: ast::Statement = { - "class" ":" => ast::Statement::ClassDef { - name: n, - args: a.unwrap_or(vec![]), - body: s}, + "class" ":" => { + ast::Statement { + location: loc, + statement: ast::StatementType::ClassDef { + name: n, + args: a.unwrap_or(vec![]), + body: s + }, + } + }, }; Test: ast::Expression = { diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 58c3ad23b..5bf77de4b 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -30,7 +30,7 @@ pub fn compile( }, Mode::Eval => match parser::parse_statement(source) { Ok(statement) => { - if let &ast::Statement::Expression { ref expression } = &statement { + if let &ast::StatementType::Expression { ref expression } = &statement.statement { compiler.compile_expression(expression); compiler.emit(Instruction::ReturnValue); } else { @@ -42,7 +42,7 @@ pub fn compile( Mode::Single => match parser::parse_program(source) { Ok(ast) => { for statement in ast.statements { - if let &ast::Statement::Expression { ref expression } = &statement { + if let &ast::StatementType::Expression { ref expression } = &statement.statement { compiler.compile_expression(expression); compiler.emit(Instruction::PrintExpr); } else { @@ -109,8 +109,8 @@ impl Compiler { fn compile_statement(&mut self, statement: &ast::Statement) { trace!("Compiling {:?}", statement); - match statement { - ast::Statement::Import { import_parts } => { + match &statement.statement { + ast::StatementType::Import { import_parts } => { for ast::SingleImport { module, symbol, @@ -132,13 +132,13 @@ impl Compiler { }); } } - ast::Statement::Expression { expression } => { + ast::StatementType::Expression { expression } => { self.compile_expression(expression); // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::If { test, body, orelse } => { + ast::StatementType::If { test, body, orelse } => { let end_label = self.new_label(); match orelse { None => { @@ -160,7 +160,7 @@ impl Compiler { } self.set_label(end_label); } - ast::Statement::While { + ast::StatementType::While { test, body, orelse: _, @@ -182,10 +182,10 @@ impl Compiler { }); self.set_label(end_label); } - ast::Statement::With { items: _, body: _ } => { + ast::StatementType::With { items: _, body: _ } => { // TODO } - ast::Statement::For { + ast::StatementType::For { target, iter, body, @@ -230,7 +230,7 @@ impl Compiler { self.set_label(end_label); self.emit(Instruction::PopBlock); } - ast::Statement::FunctionDef { name, args, body } => { + ast::StatementType::FunctionDef { name, args, body } => { // Create bytecode for this function: self.code_object_stack.push(CodeObject::new(args.to_vec())); self.compile_statements(body); @@ -257,7 +257,7 @@ impl Compiler { name: name.to_string(), }); } - ast::Statement::ClassDef { name, body, args } => { + ast::StatementType::ClassDef { name, body, args } => { self.emit(Instruction::LoadBuildClass); self.code_object_stack .push(CodeObject::new(vec![String::from("__locals__")])); @@ -300,7 +300,7 @@ impl Compiler { name: name.to_string(), }); } - ast::Statement::Assert { test, msg } => { + ast::StatementType::Assert { test, msg } => { // TODO: if some flag, ignore all assert statements! self.compile_expression(test); @@ -323,13 +323,13 @@ impl Compiler { } self.set_label(end_label); } - ast::Statement::Break => { + ast::StatementType::Break => { self.emit(Instruction::Break); } - ast::Statement::Continue => { + ast::StatementType::Continue => { self.emit(Instruction::Continue); } - ast::Statement::Return { value } => { + ast::StatementType::Return { value } => { match value { Some(e) => { let size = e.len(); @@ -349,14 +349,14 @@ impl Compiler { self.emit(Instruction::ReturnValue); } - ast::Statement::Assign { targets, value } => { + ast::StatementType::Assign { targets, value } => { self.compile_expression(value); for target in targets { self.compile_store(target); } } - ast::Statement::AugAssign { target, op, value } => { + ast::StatementType::AugAssign { target, op, value } => { self.compile_expression(target); self.compile_expression(value); @@ -364,11 +364,11 @@ impl Compiler { self.compile_op(op); self.compile_store(target); } - ast::Statement::Delete { targets: _ } => { + ast::StatementType::Delete { targets: _ } => { // TODO: Remove the given names from the scope // self.emit(Instruction::DeleteName); } - ast::Statement::Pass => { + ast::StatementType::Pass => { self.emit(Instruction::Pass); } }