Add location to statement ast type

parser/src/ast.rs

@@ -2,12 +2,9 @@
  * Implement abstract syntax tree nodes for the python language.
  */
 
+pub use super::lexer::Location;
 /*
 #[derive(Debug)]
-pub struct Location {
-    pub row: i32,
-    pub column: i32,
-}
 
 #[derive(Debug)]
 pub struct Node {
@@ -29,7 +26,13 @@ pub struct SingleImport {
 }
 
 #[derive(Debug, PartialEq)]
-pub enum Statement {
+pub struct Statement {
+    pub location: Location,
+    pub statement: StatementType,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum StatementType {
     Break,
     Continue,
     Return {

parser/src/lexer.rs

@@ -18,7 +18,7 @@ pub enum LexicalError {
     StringError,
 }
 
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, PartialEq)]
 pub struct Location {
     row: usize,
     column: usize,
@@ -49,6 +49,9 @@ impl<'input> Lexer<'input> {
         };
         lxr.next_char();
         lxr.next_char();
+        // Start at top row (=1) left column (=1)
+        lxr.location.row = 1;
+        lxr.location.column = 1;
         lxr
     }
 
@@ -137,9 +140,11 @@ impl<'input> Lexer<'input> {
             self.next_char();
             match self.chr0 {
                 Some('\n') => {
+                    self.new_line();
                     return;
                 }
                 Some('\r') => {
+                    self.new_line();
                     return;
                 }
                 Some(_) => {}
@@ -268,6 +273,11 @@ impl<'input> Lexer<'input> {
         self.location.clone()
     }
 
+    fn new_line(&mut self) {
+        self.location.row += 1;
+        self.location.column = 1;
+    }
+
     fn inner_next(&mut self) -> Option<Spanned<Tok>> {
         if !self.pending.is_empty() {
             return Some(self.pending.remove(0));
@@ -299,12 +309,14 @@ impl<'input> Lexer<'input> {
                                 self.next_char();
                             }
                             self.at_begin_of_line = true;
+                            self.new_line();
                             continue 'top_loop;
                         }
                         Some('\n') => {
                             // Empty line!
                             self.next_char();
                             self.at_begin_of_line = true;
+                            self.new_line();
                             continue 'top_loop;
                         }
                         _ => {
@@ -666,6 +678,7 @@ impl<'input> Lexer<'input> {
                     let tok_start = self.get_pos();
                     self.next_char();
                     let tok_end = self.get_pos();
+                    self.new_line();
 
                     // Depending on the nesting level, we emit newline or not:
                     if self.nesting == 0 {
@@ -679,6 +692,7 @@ impl<'input> Lexer<'input> {
                     let tok_start = self.get_pos();
                     self.next_char();
                     let tok_end = self.get_pos();
+                    self.new_line();
 
                     // Depending on the nesting level, we emit newline or not:
                     if self.nesting == 0 {

parser/src/parser.rs

@@ -88,14 +88,17 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Program {
-                statements: vec![ast::Statement::Expression {
-                    expression: ast::Expression::Call {
-                        function: Box::new(ast::Expression::Identifier {
-                            name: String::from("print"),
-                        }),
-                        args: vec![ast::Expression::String {
-                            value: String::from("Hello world"),
-                        },],
+                statements: vec![ast::Statement {
+                    location: ast::Location::new(1, 1),
+                    statement: ast::StatementType::Expression {
+                        expression: ast::Expression::Call {
+                            function: Box::new(ast::Expression::Identifier {
+                                name: String::from("print"),
+                            }),
+                            args: vec![ast::Expression::String {
+                                value: String::from("Hello world"),
+                            },],
+                        },
                     },
                 },],
             }
@@ -109,19 +112,22 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Program {
-                statements: vec![ast::Statement::Expression {
-                    expression: ast::Expression::Call {
-                        function: Box::new(ast::Expression::Identifier {
-                            name: String::from("print"),
-                        }),
-                        args: vec![
-                            ast::Expression::String {
-                                value: String::from("Hello world"),
-                            },
-                            ast::Expression::Number {
-                                value: ast::Number::Integer { value: 2 },
-                            },
-                        ],
+                statements: vec![ast::Statement {
+                    location: ast::Location::new(1, 1),
+                    statement: ast::StatementType::Expression {
+                        expression: ast::Expression::Call {
+                            function: Box::new(ast::Expression::Identifier {
+                                name: String::from("print"),
+                            }),
+                            args: vec![
+                                ast::Expression::String {
+                                    value: String::from("Hello world"),
+                                },
+                                ast::Expression::Number {
+                                    value: ast::Number::Integer { value: 2 },
+                                },
+                            ],
+                        },
                     },
                 },],
             }
@@ -134,30 +140,45 @@ mod tests {
         let parse_ast = parse_statement(&source).unwrap();
         assert_eq!(
             parse_ast,
-            ast::Statement::If {
-                test: ast::Expression::Number {
-                    value: ast::Number::Integer { value: 1 },
-                },
-                body: vec![ast::Statement::Expression {
-                    expression: ast::Expression::Number {
-                        value: ast::Number::Integer { value: 10 },
-                    },
-                },],
-                orelse: Some(vec![ast::Statement::If {
+            ast::Statement {
+                location: ast::Location::new(1, 1),
+                statement: ast::StatementType::If {
                     test: ast::Expression::Number {
-                        value: ast::Number::Integer { value: 2 },
+                        value: ast::Number::Integer { value: 1 },
                     },
-                    body: vec![ast::Statement::Expression {
-                        expression: ast::Expression::Number {
-                            value: ast::Number::Integer { value: 20 },
+                    body: vec![ast::Statement {
+                        location: ast::Location::new(1, 7),
+                        statement: ast::StatementType::Expression {
+                            expression: ast::Expression::Number {
+                                value: ast::Number::Integer { value: 10 },
+                            }
                         },
                     },],
-                    orelse: Some(vec![ast::Statement::Expression {
-                        expression: ast::Expression::Number {
-                            value: ast::Number::Integer { value: 30 },
-                        },
+                    orelse: Some(vec![ast::Statement {
+                        location: ast::Location::new(2, 1),
+                        statement: ast::StatementType::If {
+                            test: ast::Expression::Number {
+                                value: ast::Number::Integer { value: 2 },
+                            },
+                            body: vec![ast::Statement {
+                                location: ast::Location::new(2, 9),
+                                statement: ast::StatementType::Expression {
+                                    expression: ast::Expression::Number {
+                                        value: ast::Number::Integer { value: 20 },
+                                    },
+                                },
+                            },],
+                            orelse: Some(vec![ast::Statement {
+                                location: ast::Location::new(3, 7),
+                                statement: ast::StatementType::Expression {
+                                    expression: ast::Expression::Number {
+                                        value: ast::Number::Integer { value: 30 },
+                                    },
+                                },
+                            },]),
+                        }
                     },]),
-                },]),
+                }
             }
         );
     }
@@ -168,18 +189,21 @@ mod tests {
         let parse_ast = parse_statement(&source);
         assert_eq!(
             parse_ast,
-            Ok(ast::Statement::Expression {
-                expression: ast::Expression::Lambda {
-                    args: vec![String::from("x"), String::from("y")],
-                    body: Box::new(ast::Expression::Binop {
-                        a: Box::new(ast::Expression::Identifier {
-                            name: String::from("x"),
-                        }),
-                        op: ast::Operator::Mult,
-                        b: Box::new(ast::Expression::Identifier {
-                            name: String::from("y"),
+            Ok(ast::Statement {
+                location: ast::Location::new(1, 1),
+                statement: ast::StatementType::Expression {
+                    expression: ast::Expression::Lambda {
+                        args: vec![String::from("x"), String::from("y")],
+                        body: Box::new(ast::Expression::Binop {
+                            a: Box::new(ast::Expression::Identifier {
+                                name: String::from("x"),
+                            }),
+                            op: ast::Operator::Mult,
+                            b: Box::new(ast::Expression::Identifier {
+                                name: String::from("y"),
+                            })
                         })
-                    })
+                    }
                 }
             })
         )
@@ -190,14 +214,23 @@ mod tests {
         let source = String::from("class Foo(A, B):\n def __init__(self):\n  pass\n");
         assert_eq!(
             parse_statement(&source),
-            Ok(ast::Statement::ClassDef {
-                name: String::from("Foo"),
-                args: vec![String::from("A"), String::from("B")],
-                body: vec![ast::Statement::FunctionDef {
-                    name: String::from("__init__"),
-                    args: vec![String::from("self")],
-                    body: vec![ast::Statement::Pass],
-                }],
+            Ok(ast::Statement {
+                location: ast::Location::new(1, 1),
+                statement: ast::StatementType::ClassDef {
+                    name: String::from("Foo"),
+                    args: vec![String::from("A"), String::from("B")],
+                    body: vec![ast::Statement {
+                        location: ast::Location::new(2, 2),
+                        statement: ast::StatementType::FunctionDef {
+                            name: String::from("__init__"),
+                            args: vec![String::from("self")],
+                            body: vec![ast::Statement {
+                                location: ast::Location::new(3, 3),
+                                statement: ast::StatementType::Pass,
+                            }],
+                        }
+                    }],
+                }
             })
         )
     }

parser/src/python.lalrpop

@@ -34,14 +34,19 @@ SimpleStatement: ast::Statement = {
 SmallStatement: ast::Statement = {
   // <e:Expression> => ast::Statement::Expression { expression: e },
   ExpressionStatement,
-  "pass" => ast::Statement::Pass,
+  <loc:@L> "pass" => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Pass,
+    }
+  },
   FlowStatement,
   ImportStatement,
   AssertStatement,
 };
 
 ExpressionStatement: ast::Statement = {
-  <e:TestList> <e2:AssignSuffix*> => {
+  <loc:@L> <e:TestList> <e2:AssignSuffix*> => {
     //match e2 {
     //  None => ast::Statement::Expression { expression: e },
     //  Some(e3) => ast::Statement::Expression { expression: e },
@@ -52,21 +57,30 @@ ExpressionStatement: ast::Statement = {
        let rhs = e2.into_iter().next().unwrap();
        // ast::Expression::Tuple { elements: e2.into_iter().next().unwrap()
        let v = rhs.into_iter().next().unwrap();
-       let lhs = ast::Statement::Assign { targets: e, value: v };
+       let lhs = ast::Statement {
+         location: loc.clone(),
+         statement: ast::StatementType::Assign { targets: e, value: v },
+       };
       lhs
     } else {
       if e.len() > 1 {
         panic!("Not good?");
         // ast::Statement::Expression { expression: e[0] }
       } else {
-        ast::Statement::Expression { expression: e.into_iter().next().unwrap() }
+        ast::Statement {
+          location: loc.clone(),
+          statement: ast::StatementType::Expression { expression: e.into_iter().next().unwrap() },
+        }
       }
     }
   },
-  <e1:Test> <op:AugAssign> <e2:TestList> => {
+  <loc:@L> <e1:Test> <op:AugAssign> <e2:TestList> => {
     // TODO: this works in most cases:
     let rhs = e2.into_iter().next().unwrap();
-    ast::Statement::AugAssign { target: e1, op: op, value: rhs }
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::AugAssign { target: e1, op: op, value: rhs },
+    }
   },
 };
 
@@ -91,16 +105,33 @@ AugAssign: ast::Operator = {
 };
 
 FlowStatement: ast::Statement = {
-  "break" => ast::Statement::Break,
-  "continue" => ast::Statement::Continue,
-  "return" <t:TestList?> => ast::Statement::Return { value: t},
+  <loc:@L> "break" => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Break,
+    }
+  },
+  <loc:@L> "continue" => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Continue,
+    }
+  },
+  <loc:@L> "return" <t:TestList?> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Return { value: t},
+    }
+  },
   // raise
   // yield
 };
 
 ImportStatement: ast::Statement = {
-  "import" <i: Comma<ImportPart<<DottedName>>>> => {
-    ast::Statement::Import {
+  <loc:@L> "import" <i: Comma<ImportPart<<DottedName>>>> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Import {
         import_parts: i
             .iter()
             .map(|(n, a)|
@@ -110,10 +141,13 @@ ImportStatement: ast::Statement = {
                     alias: a.clone()
                 })
             .collect()
+      },
     }
   },
-  "from" <n:DottedName> "import" <i: Comma<ImportPart<Identifier>>> => {
-    ast::Statement::Import {
+  <loc:@L> "from" <n:DottedName> "import" <i: Comma<ImportPart<Identifier>>> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Import {
         import_parts: i
             .iter()
             .map(|(i, a)|
@@ -123,6 +157,7 @@ ImportStatement: ast::Statement = {
                     alias: a.clone()
                 })
             .collect()
+      },
     }
   },
 };
@@ -137,11 +172,16 @@ DottedName: String = {
 };
 
 AssertStatement: ast::Statement = {
-  "assert" <t:Test> <m: ("," Test)?> => ast::Statement::Assert {
-    test: t,
-    msg: match m {
-      Some(e) => Some(e.1),
-      None => None,
+  <loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::Assert {
+        test: t,
+        msg: match m {
+          Some(e) => Some(e.1),
+          None => None,
+        }
+      }
     }
   },
 };
@@ -156,7 +196,7 @@ CompoundStatement: ast::Statement = {
 };
 
 IfStatement: ast::Statement = {
-  "if" <t:Test> ":" <s1:Suite> <s2:("elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
+  <loc:@L> "if" <t:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
     // Determine last else:
     let mut last = match s3 {
       Some(s) => Some(s.2),
@@ -165,39 +205,62 @@ IfStatement: ast::Statement = {
 
     // handle elif:
     for i in s2.into_iter().rev() {
-      last = Some(vec![ast::Statement::If { test: i.1, body: i.3, orelse: last }]);
+      let x = ast::Statement {
+        location: i.0,
+        statement: ast::StatementType::If { test: i.2, body: i.4, orelse: last },
+      };
+      last = Some(vec![x]);
     }
 
-    ast::Statement::If { test: t, body: s1, orelse: last }
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::If { test: t, body: s1, orelse: last }
+    }
   },
 };
 
 WhileStatement: ast::Statement = {
-  "while" <e:Test> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
+  <loc:@L> "while" <e:Test> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
     let or_else = match s2 {
       Some(s) => Some(s.2),
       None => None,
     };
-    ast::Statement::While { test: e, body: s, orelse: or_else }
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::While { test: e, body: s, orelse: or_else },
+    }
   },
 };
 
 ForStatement: ast::Statement = {
-  "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
+  <loc:@L> "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
     let or_else = match s2 {
       Some(s) => Some(s.2),
       None => None,
     };
-    ast::Statement::For { target: e, iter: t, body: s, orelse: or_else }
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::For { target: e, iter: t, body: s, orelse: or_else },
+    }
   },
 };
 
 WithStatement: ast::Statement = {
-  "with" <t:Test> "as" <_e:Expression> ":" <s:Suite> => ast::Statement::With { items: t, body: s },
+  <loc:@L> "with" <t:Test> "as" <_e:Expression> ":" <s:Suite> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::With { items: t, body: s },
+    }
+  },
 };
 
 FuncDef: ast::Statement = {
-  "def" <i:Identifier> <a:Parameters> ":" <s:Suite> => ast::Statement::FunctionDef { name: i, args: a, body: s },
+  <loc:@L> "def" <i:Identifier> <a:Parameters> ":" <s:Suite> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::FunctionDef { name: i, args: a, body: s }
+    }
+  },
 };
 
 Parameters: Vec<String> = {
@@ -209,10 +272,16 @@ TypedArgsList: Vec<String> = {
 };
 
 ClassDef: ast::Statement = {
-  "class" <n:Identifier> <a:Parameters?> ":" <s:Suite> => ast::Statement::ClassDef {
-	  name: n,
-	  args: a.unwrap_or(vec![]),
-	  body: s},
+  <loc:@L> "class" <n:Identifier> <a:Parameters?> ":" <s:Suite> => {
+    ast::Statement {
+      location: loc,
+      statement: ast::StatementType::ClassDef {
+          name: n,
+          args: a.unwrap_or(vec![]),
+          body: s
+      },
+    }
+  },
 };
 
 Test: ast::Expression = {

vm/src/compile.rs

@@ -30,7 +30,7 @@ pub fn compile(
         },
         Mode::Eval => match parser::parse_statement(source) {
             Ok(statement) => {
-                if let &ast::Statement::Expression { ref expression } = &statement {
+                if let &ast::StatementType::Expression { ref expression } = &statement.statement {
                     compiler.compile_expression(expression);
                     compiler.emit(Instruction::ReturnValue);
                 } else {
@@ -42,7 +42,7 @@ pub fn compile(
         Mode::Single => match parser::parse_program(source) {
             Ok(ast) => {
                 for statement in ast.statements {
-                    if let &ast::Statement::Expression { ref expression } = &statement {
+                    if let &ast::StatementType::Expression { ref expression } = &statement.statement {
                         compiler.compile_expression(expression);
                         compiler.emit(Instruction::PrintExpr);
                     } else {
@@ -109,8 +109,8 @@ impl Compiler {
 
     fn compile_statement(&mut self, statement: &ast::Statement) {
         trace!("Compiling {:?}", statement);
-        match statement {
-            ast::Statement::Import { import_parts } => {
+        match &statement.statement {
+            ast::StatementType::Import { import_parts } => {
                 for ast::SingleImport {
                     module,
                     symbol,
@@ -132,13 +132,13 @@ impl Compiler {
                     });
                 }
             }
-            ast::Statement::Expression { expression } => {
+            ast::StatementType::Expression { expression } => {
                 self.compile_expression(expression);
 
                 // Pop result of stack, since we not use it:
                 self.emit(Instruction::Pop);
             }
-            ast::Statement::If { test, body, orelse } => {
+            ast::StatementType::If { test, body, orelse } => {
                 let end_label = self.new_label();
                 match orelse {
                     None => {
@@ -160,7 +160,7 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            ast::Statement::While {
+            ast::StatementType::While {
                 test,
                 body,
                 orelse: _,
@@ -182,10 +182,10 @@ impl Compiler {
                 });
                 self.set_label(end_label);
             }
-            ast::Statement::With { items: _, body: _ } => {
+            ast::StatementType::With { items: _, body: _ } => {
                 // TODO
             }
-            ast::Statement::For {
+            ast::StatementType::For {
                 target,
                 iter,
                 body,
@@ -230,7 +230,7 @@ impl Compiler {
                 self.set_label(end_label);
                 self.emit(Instruction::PopBlock);
             }
-            ast::Statement::FunctionDef { name, args, body } => {
+            ast::StatementType::FunctionDef { name, args, body } => {
                 // Create bytecode for this function:
                 self.code_object_stack.push(CodeObject::new(args.to_vec()));
                 self.compile_statements(body);
@@ -257,7 +257,7 @@ impl Compiler {
                     name: name.to_string(),
                 });
             }
-            ast::Statement::ClassDef { name, body, args } => {
+            ast::StatementType::ClassDef { name, body, args } => {
                 self.emit(Instruction::LoadBuildClass);
                 self.code_object_stack
                     .push(CodeObject::new(vec![String::from("__locals__")]));
@@ -300,7 +300,7 @@ impl Compiler {
                     name: name.to_string(),
                 });
             }
-            ast::Statement::Assert { test, msg } => {
+            ast::StatementType::Assert { test, msg } => {
                 // TODO: if some flag, ignore all assert statements!
 
                 self.compile_expression(test);
@@ -323,13 +323,13 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            ast::Statement::Break => {
+            ast::StatementType::Break => {
                 self.emit(Instruction::Break);
             }
-            ast::Statement::Continue => {
+            ast::StatementType::Continue => {
                 self.emit(Instruction::Continue);
             }
-            ast::Statement::Return { value } => {
+            ast::StatementType::Return { value } => {
                 match value {
                     Some(e) => {
                         let size = e.len();
@@ -349,14 +349,14 @@ impl Compiler {
 
                 self.emit(Instruction::ReturnValue);
             }
-            ast::Statement::Assign { targets, value } => {
+            ast::StatementType::Assign { targets, value } => {
                 self.compile_expression(value);
 
                 for target in targets {
                     self.compile_store(target);
                 }
             }
-            ast::Statement::AugAssign { target, op, value } => {
+            ast::StatementType::AugAssign { target, op, value } => {
                 self.compile_expression(target);
                 self.compile_expression(value);
 
@@ -364,11 +364,11 @@ impl Compiler {
                 self.compile_op(op);
                 self.compile_store(target);
             }
-            ast::Statement::Delete { targets: _ } => {
+            ast::StatementType::Delete { targets: _ } => {
                 // TODO: Remove the given names from the scope
                 // self.emit(Instruction::DeleteName);
             }
-            ast::Statement::Pass => {
+            ast::StatementType::Pass => {
                 self.emit(Instruction::Pass);
             }
         }