From d6c19c1c6a2490bcb502282fbe5e756ea71e25e0 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sat, 20 Jul 2019 20:44:38 +0200 Subject: [PATCH] Add location to expressions. Change symboltable to use flags for symbols. --- compiler/src/compile.rs | 196 ++++++------ compiler/src/symboltable.rs | 270 +++++++++-------- crawl_sourcecode.py | 18 +- parser/src/ast.rs | 48 +-- parser/src/fstring.rs | 13 +- parser/src/parser.rs | 448 ++++++++++++---------------- parser/src/python.lalrpop | 575 ++++++++++++++++++++++-------------- vm/src/stdlib/ast.rs | 128 ++++---- vm/src/stdlib/mod.rs | 7 +- vm/src/stdlib/symtable.rs | 60 +--- 10 files changed, 935 insertions(+), 828 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 8f3d695cb..b06262c60 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -6,7 +6,7 @@ //! https://github.com/micropython/micropython/blob/master/py/compile.c use crate::error::{CompileError, CompileErrorType}; -use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope}; +use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope}; use num_complex::Complex64; use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs}; use rustpython_parser::{ast, parser}; @@ -75,7 +75,7 @@ pub fn compile_program( /// Compile a single Python expression to bytecode pub fn compile_statement_eval( - statement: Vec, + statement: Vec, source_path: String, optimize: u8, ) -> Result { @@ -180,7 +180,7 @@ impl Compiler { for (i, statement) in program.statements.iter().enumerate() { let is_last = i == program.statements.len() - 1; - if let ast::Statement::Expression { ref expression } = statement.node { + if let ast::StatementType::Expression { ref expression } = statement.node { self.compile_expression(expression)?; if is_last { @@ -209,12 +209,12 @@ impl Compiler { // Compile statement in eval mode: fn compile_statement_eval( &mut self, - statements: &[ast::LocatedStatement], + statements: &[ast::Statement], symbol_table: SymbolScope, ) -> Result<(), CompileError> { self.scope_stack.push(symbol_table); for statement in statements { - if let ast::Statement::Expression { ref expression } = statement.node { + if let ast::StatementType::Expression { ref expression } = statement.node { self.compile_expression(expression)?; } else { return Err(CompileError { @@ -227,10 +227,7 @@ impl Compiler { Ok(()) } - fn compile_statements( - &mut self, - statements: &[ast::LocatedStatement], - ) -> Result<(), CompileError> { + fn compile_statements(&mut self, statements: &[ast::Statement]) -> Result<(), CompileError> { for statement in statements { self.compile_statement(statement)? } @@ -238,11 +235,13 @@ impl Compiler { } fn scope_for_name(&self, name: &str) -> bytecode::NameScope { - let role = self.lookup_name(name); - match role { - SymbolRole::Global => bytecode::NameScope::Global, - SymbolRole::Nonlocal => bytecode::NameScope::NonLocal, - _ => bytecode::NameScope::Local, + let symbol = self.lookup_name(name); + if symbol.is_global { + bytecode::NameScope::Global + } else if symbol.is_nonlocal { + bytecode::NameScope::NonLocal + } else { + bytecode::NameScope::Local } } @@ -262,12 +261,13 @@ impl Compiler { }); } - fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> { + fn compile_statement(&mut self, statement: &ast::Statement) -> Result<(), CompileError> { trace!("Compiling {:?}", statement); self.set_source_location(&statement.location); + use ast::StatementType::*; match &statement.node { - ast::Statement::Import { names } => { + Import { names } => { // import a, b, c as d for name in names { self.emit(Instruction::Import { @@ -283,7 +283,7 @@ impl Compiler { } } } - ast::Statement::ImportFrom { + ImportFrom { level, module, names, @@ -326,16 +326,16 @@ impl Compiler { self.emit(Instruction::Pop); } } - ast::Statement::Expression { expression } => { + Expression { expression } => { self.compile_expression(expression)?; // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => { + Global { .. } | Nonlocal { .. } => { // Handled during symbol table construction. } - ast::Statement::If { test, body, orelse } => { + If { test, body, orelse } => { let end_label = self.new_label(); match orelse { None => { @@ -358,7 +358,7 @@ impl Compiler { } self.set_label(end_label); } - ast::Statement::While { test, body, orelse } => { + While { test, body, orelse } => { let start_label = self.new_label(); let else_label = self.new_label(); let end_label = self.new_label(); @@ -385,7 +385,7 @@ impl Compiler { } self.set_label(end_label); } - ast::Statement::With { items, body } => { + With { items, body } => { let end_label = self.new_label(); for item in items { self.compile_expression(&item.context_expr)?; @@ -406,16 +406,16 @@ impl Compiler { } self.set_label(end_label); } - ast::Statement::For { + For { target, iter, body, orelse, } => self.compile_for(target, iter, body, orelse)?, - ast::Statement::AsyncFor { .. } => { + AsyncFor { .. } => { unimplemented!("async for"); } - ast::Statement::Raise { exception, cause } => match exception { + Raise { exception, cause } => match exception { Some(value) => { self.compile_expression(value)?; match cause { @@ -432,30 +432,30 @@ impl Compiler { self.emit(Instruction::Raise { argc: 0 }); } }, - ast::Statement::Try { + Try { body, handlers, orelse, finalbody, } => self.compile_try_statement(body, handlers, orelse, finalbody)?, - ast::Statement::FunctionDef { + FunctionDef { name, args, body, decorator_list, returns, } => self.compile_function_def(name, args, body, decorator_list, returns)?, - ast::Statement::AsyncFunctionDef { .. } => { + AsyncFunctionDef { .. } => { unimplemented!("async def"); } - ast::Statement::ClassDef { + ClassDef { name, body, bases, keywords, decorator_list, } => self.compile_class_def(name, body, bases, keywords, decorator_list)?, - ast::Statement::Assert { test, msg } => { + Assert { test, msg } => { // if some flag, ignore all assert statements! if self.optimize == 0 { let end_label = self.new_label(); @@ -481,7 +481,7 @@ impl Compiler { self.set_label(end_label); } } - ast::Statement::Break => { + Break => { if !self.in_loop { return Err(CompileError { error: CompileErrorType::InvalidBreak, @@ -490,7 +490,7 @@ impl Compiler { } self.emit(Instruction::Break); } - ast::Statement::Continue => { + Continue => { if !self.in_loop { return Err(CompileError { error: CompileErrorType::InvalidContinue, @@ -499,7 +499,7 @@ impl Compiler { } self.emit(Instruction::Continue); } - ast::Statement::Return { value } => { + Return { value } => { if !self.in_function_def { return Err(CompileError { error: CompileErrorType::InvalidReturn, @@ -519,7 +519,7 @@ impl Compiler { self.emit(Instruction::ReturnValue); } - ast::Statement::Assign { targets, value } => { + Assign { targets, value } => { self.compile_expression(value)?; for (i, target) in targets.iter().enumerate() { @@ -529,7 +529,7 @@ impl Compiler { self.compile_store(target)?; } } - ast::Statement::AugAssign { target, op, value } => { + AugAssign { target, op, value } => { self.compile_expression(target)?; self.compile_expression(value)?; @@ -537,12 +537,12 @@ impl Compiler { self.compile_op(op, true); self.compile_store(target)?; } - ast::Statement::Delete { targets } => { + Delete { targets } => { for target in targets { self.compile_delete(target)?; } } - ast::Statement::Pass => { + Pass => { self.emit(Instruction::Pass); } } @@ -550,24 +550,24 @@ impl Compiler { } fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> { - match expression { - ast::Expression::Identifier { name } => { + match &expression.node { + ast::ExpressionType::Identifier { name } => { self.emit(Instruction::DeleteName { name: name.to_string(), }); } - ast::Expression::Attribute { value, name } => { + ast::ExpressionType::Attribute { value, name } => { self.compile_expression(value)?; self.emit(Instruction::DeleteAttr { name: name.to_string(), }); } - ast::Expression::Subscript { a, b } => { + ast::ExpressionType::Subscript { a, b } => { self.compile_expression(a)?; self.compile_expression(b)?; self.emit(Instruction::DeleteSubscript); } - ast::Expression::Tuple { elements } => { + ast::ExpressionType::Tuple { elements } => { for element in elements { self.compile_delete(element)?; } @@ -663,10 +663,10 @@ impl Compiler { fn compile_try_statement( &mut self, - body: &[ast::LocatedStatement], + body: &[ast::Statement], handlers: &[ast::ExceptHandler], - orelse: &Option>, - finalbody: &Option>, + orelse: &Option>, + finalbody: &Option>, ) -> Result<(), CompileError> { let mut handler_label = self.new_label(); let finally_label = self.new_label(); @@ -764,7 +764,7 @@ impl Compiler { &mut self, name: &str, args: &ast::Parameters, - body: &[ast::LocatedStatement], + body: &[ast::Statement], decorator_list: &[ast::Expression], returns: &Option, // TODO: use type hint somehow.. ) -> Result<(), CompileError> { @@ -858,7 +858,7 @@ impl Compiler { fn compile_class_def( &mut self, name: &str, - body: &[ast::LocatedStatement], + body: &[ast::Statement], bases: &[ast::Expression], keywords: &[ast::Keyword], decorator_list: &[ast::Expression], @@ -989,8 +989,8 @@ impl Compiler { &mut self, target: &ast::Expression, iter: &ast::Expression, - body: &[ast::LocatedStatement], - orelse: &Option>, + body: &[ast::Statement], + orelse: &Option>, ) -> Result<(), CompileError> { // Start loop let start_label = self.new_label(); @@ -1104,27 +1104,27 @@ impl Compiler { } fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { - match target { - ast::Expression::Identifier { name } => { + match &target.node { + ast::ExpressionType::Identifier { name } => { self.store_name(name); } - ast::Expression::Subscript { a, b } => { + ast::ExpressionType::Subscript { a, b } => { self.compile_expression(a)?; self.compile_expression(b)?; self.emit(Instruction::StoreSubscript); } - ast::Expression::Attribute { value, name } => { + ast::ExpressionType::Attribute { value, name } => { self.compile_expression(value)?; self.emit(Instruction::StoreAttr { name: name.to_string(), }); } - ast::Expression::List { elements } | ast::Expression::Tuple { elements } => { + ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => { let mut seen_star = false; // Scan for star args: for (i, element) in elements.iter().enumerate() { - if let ast::Expression::Starred { .. } = element { + if let ast::ExpressionType::Starred { .. } = &element.node { if seen_star { return Err(CompileError { error: CompileErrorType::StarArgs, @@ -1147,7 +1147,7 @@ impl Compiler { } for element in elements { - if let ast::Expression::Starred { value } = element { + if let ast::ExpressionType::Starred { value } = &element.node { self.compile_store(value)?; } else { self.compile_store(element)?; @@ -1192,8 +1192,8 @@ impl Compiler { context: EvalContext, ) -> Result<(), CompileError> { // Compile expression for test, and jump to label if false - match expression { - ast::Expression::BoolOp { a, op, b } => match op { + match &expression.node { + ast::ExpressionType::BoolOp { a, op, b } => match op { ast::BooleanOperator::And => { let f = false_label.unwrap_or_else(|| self.new_label()); self.compile_test(a, None, Some(f), context)?; @@ -1246,23 +1246,27 @@ impl Compiler { fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> { trace!("Compiling {:?}", expression); - match expression { - ast::Expression::Call { + use ast::ExpressionType::*; + match &expression.node { + Call { function, args, keywords, } => self.compile_call(function, args, keywords)?, - ast::Expression::BoolOp { .. } => { - self.compile_test(expression, None, None, EvalContext::Expression)? - } - ast::Expression::Binop { a, op, b } => { + BoolOp { .. } => self.compile_test( + expression, + Option::None, + Option::None, + EvalContext::Expression, + )?, + Binop { a, op, b } => { self.compile_expression(a)?; self.compile_expression(b)?; // Perform operation: self.compile_op(op, false); } - ast::Expression::Subscript { a, b } => { + Subscript { a, b } => { self.compile_expression(a)?; self.compile_expression(b)?; self.emit(Instruction::BinaryOperation { @@ -1270,7 +1274,7 @@ impl Compiler { inplace: false, }); } - ast::Expression::Unop { op, a } => { + Unop { op, a } => { self.compile_expression(a)?; // Perform operation: @@ -1283,16 +1287,16 @@ impl Compiler { let i = Instruction::UnaryOperation { op: i }; self.emit(i); } - ast::Expression::Attribute { value, name } => { + Attribute { value, name } => { self.compile_expression(value)?; self.emit(Instruction::LoadAttr { name: name.to_string(), }); } - ast::Expression::Compare { vals, ops } => { + Compare { vals, ops } => { self.compile_chained_comparison(vals, ops)?; } - ast::Expression::Number { value } => { + Number { value } => { let const_value = match value { ast::Number::Integer { value } => bytecode::Constant::Integer { value: value.clone(), @@ -1304,7 +1308,7 @@ impl Compiler { }; self.emit(Instruction::LoadConst { value: const_value }); } - ast::Expression::List { elements } => { + List { elements } => { let size = elements.len(); let must_unpack = self.gather_elements(elements)?; self.emit(Instruction::BuildList { @@ -1312,7 +1316,7 @@ impl Compiler { unpack: must_unpack, }); } - ast::Expression::Tuple { elements } => { + Tuple { elements } => { let size = elements.len(); let must_unpack = self.gather_elements(elements)?; self.emit(Instruction::BuildTuple { @@ -1320,7 +1324,7 @@ impl Compiler { unpack: must_unpack, }); } - ast::Expression::Set { elements } => { + Set { elements } => { let size = elements.len(); let must_unpack = self.gather_elements(elements)?; self.emit(Instruction::BuildSet { @@ -1328,7 +1332,7 @@ impl Compiler { unpack: must_unpack, }); } - ast::Expression::Dict { elements } => { + Dict { elements } => { let size = elements.len(); let has_double_star = elements.iter().any(|e| e.0.is_none()); for (key, value) in elements { @@ -1351,14 +1355,14 @@ impl Compiler { unpack: has_double_star, }); } - ast::Expression::Slice { elements } => { + Slice { elements } => { let size = elements.len(); for element in elements { self.compile_expression(element)?; } self.emit(Instruction::BuildSlice { size }); } - ast::Expression::Yield { value } => { + Yield { value } => { if !self.in_function_def { return Err(CompileError { error: CompileErrorType::InvalidYield, @@ -1368,16 +1372,16 @@ impl Compiler { self.mark_generator(); match value { Some(expression) => self.compile_expression(expression)?, - None => self.emit(Instruction::LoadConst { + Option::None => self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }), }; self.emit(Instruction::YieldValue); } - ast::Expression::Await { .. } => { + Await { .. } => { unimplemented!("await"); } - ast::Expression::YieldFrom { value } => { + YieldFrom { value } => { self.mark_generator(); self.compile_expression(value)?; self.emit(Instruction::GetIter); @@ -1386,40 +1390,40 @@ impl Compiler { }); self.emit(Instruction::YieldFrom); } - ast::Expression::True => { + True => { self.emit(Instruction::LoadConst { value: bytecode::Constant::Boolean { value: true }, }); } - ast::Expression::False => { + False => { self.emit(Instruction::LoadConst { value: bytecode::Constant::Boolean { value: false }, }); } - ast::Expression::None => { + None => { self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); } - ast::Expression::Ellipsis => { + Ellipsis => { self.emit(Instruction::LoadConst { value: bytecode::Constant::Ellipsis, }); } - ast::Expression::String { value } => { + String { value } => { self.compile_string(value)?; } - ast::Expression::Bytes { value } => { + Bytes { value } => { self.emit(Instruction::LoadConst { value: bytecode::Constant::Bytes { value: value.clone(), }, }); } - ast::Expression::Identifier { name } => { + Identifier { name } => { self.load_name(name); } - ast::Expression::Lambda { args, body } => { + Lambda { args, body } => { let name = "".to_string(); // no need to worry about the self.loop_depth because there are no loops in lambda expressions let flags = self.enter_function(&name, args)?; @@ -1438,18 +1442,18 @@ impl Compiler { // Turn code object into function object: self.emit(Instruction::MakeFunction { flags }); } - ast::Expression::Comprehension { kind, generators } => { + Comprehension { kind, generators } => { self.compile_comprehension(kind, generators)?; } - ast::Expression::Starred { value } => { + Starred { value } => { self.compile_expression(value)?; self.emit(Instruction::Unpack); panic!("We should not just unpack a starred args, since the size is unknown."); } - ast::Expression::IfExpression { test, body, orelse } => { + IfExpression { test, body, orelse } => { let no_label = self.new_label(); let end_label = self.new_label(); - self.compile_test(test, None, None, EvalContext::Expression)?; + self.compile_test(test, Option::None, Option::None, EvalContext::Expression)?; self.emit(Instruction::JumpIfFalse { target: no_label }); // True case self.compile_expression(body)?; @@ -1557,7 +1561,7 @@ impl Compiler { fn gather_elements(&mut self, elements: &[ast::Expression]) -> Result { // First determine if we have starred elements: let has_stars = elements.iter().any(|e| { - if let ast::Expression::Starred { .. } = e { + if let ast::ExpressionType::Starred { .. } = &e.node { true } else { false @@ -1565,7 +1569,7 @@ impl Compiler { }); for element in elements { - if let ast::Expression::Starred { value } = element { + if let ast::ExpressionType::Starred { value } = &element.node { self.compile_expression(value)?; } else { self.compile_expression(element)?; @@ -1792,7 +1796,7 @@ impl Compiler { assert!(scope.sub_scopes.is_empty()); } - fn lookup_name(&self, name: &str) -> &SymbolRole { + fn lookup_name(&self, name: &str) -> &Symbol { // println!("Looking up {:?}", name); let scope = self.scope_stack.last().unwrap(); scope.lookup(name).unwrap() @@ -1846,10 +1850,10 @@ impl Compiler { } } -fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option) { +fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option) { if let Some(val) = body.get(0) { - if let ast::Statement::Expression { ref expression } = val.node { - if let ast::Expression::String { ref value } = expression { + if let ast::StatementType::Expression { ref expression } = val.node { + if let ast::ExpressionType::String { value } = &expression.node { if let ast::StringGroup::Constant { ref value } = value { if let Some((_, body_rest)) = body.split_first() { return (body_rest, Some(value.to_string())); diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 9d4d67721..d80755d33 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -24,7 +24,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result Result { let mut builder: SymbolTableBuilder = Default::default(); builder.enter_scope(); @@ -36,25 +36,46 @@ pub fn statements_to_symbol_table( Ok(symbol_table) } -#[derive(Debug, Clone)] -pub enum SymbolRole { - Global, - Nonlocal, - Used, - Assigned, -} - /// Captures all symbols in the current scope, and has a list of subscopes in this scope. -#[derive(Clone)] +#[derive(Clone, Default)] pub struct SymbolScope { /// A set of symbols present on this scope level. - pub symbols: IndexMap, + pub symbols: IndexMap, /// A list of subscopes in the order as found in the /// AST nodes. pub sub_scopes: Vec, } +#[derive(Debug, Clone)] +pub struct Symbol { + pub name: String, + pub is_global: bool, + pub is_local: bool, + pub is_nonlocal: bool, + pub is_param: bool, + pub is_referenced: bool, + pub is_assigned: bool, + pub is_parameter: bool, + pub is_free: bool, +} + +impl Symbol { + fn new(name: &str) -> Self { + Symbol { + name: name.to_string(), + is_global: false, + is_local: false, + is_nonlocal: false, + is_param: false, + is_referenced: false, + is_assigned: false, + is_parameter: false, + is_free: false, + } + } +} + #[derive(Debug)] pub struct SymbolTableError { error: String, @@ -73,20 +94,11 @@ impl From for CompileError { type SymbolTableResult = Result<(), SymbolTableError>; impl SymbolScope { - pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { + pub fn lookup(&self, name: &str) -> Option<&Symbol> { self.symbols.get(name) } } -impl Default for SymbolScope { - fn default() -> Self { - SymbolScope { - symbols: Default::default(), - sub_scopes: Default::default(), - } - } -} - impl std::fmt::Debug for SymbolScope { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -111,74 +123,73 @@ fn analyze_symbol_table( } // Analyze symbols: - for (symbol_name, symbol_role) in &symbol_scope.symbols { - analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?; + for symbol in symbol_scope.symbols.values() { + analyze_symbol(symbol, parent_symbol_scope)?; } Ok(()) } -#[allow(clippy::single_match)] -fn analyze_symbol( - symbol_name: &str, - symbol_role: &SymbolRole, - parent_symbol_scope: Option<&SymbolScope>, -) -> SymbolTableResult { - match symbol_role { - SymbolRole::Nonlocal => { - // check if name is defined in parent scope! - if let Some(parent_symbol_scope) = parent_symbol_scope { - if !parent_symbol_scope.symbols.contains_key(symbol_name) { - return Err(SymbolTableError { - error: format!("no binding for nonlocal '{}' found", symbol_name), - location: Default::default(), - }); - } - } else { +fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult { + if symbol.is_nonlocal { + // check if name is defined in parent scope! + if let Some(parent_symbol_scope) = parent_symbol_scope { + if !parent_symbol_scope.symbols.contains_key(&symbol.name) { return Err(SymbolTableError { - error: format!( - "nonlocal {} defined at place without an enclosing scope", - symbol_name - ), + error: format!("no binding for nonlocal '{}' found", symbol.name), location: Default::default(), }); } + } else { + return Err(SymbolTableError { + error: format!( + "nonlocal {} defined at place without an enclosing scope", + symbol.name + ), + location: Default::default(), + }); } - // TODO: add more checks for globals - _ => {} } + + // TODO: add more checks for globals + Ok(()) } -pub struct SymbolTableBuilder { - // Scope stack. - pub scopes: Vec, +#[derive(Debug, Clone)] +enum SymbolRole { + Global, + Nonlocal, + Used, + Assigned, } -impl Default for SymbolTableBuilder { - fn default() -> Self { - SymbolTableBuilder { scopes: vec![] } - } +#[derive(Default)] +struct SymbolTableBuilder { + // Scope stack. + scopes: Vec, } impl SymbolTableBuilder { - pub fn enter_scope(&mut self) { + fn enter_scope(&mut self) { let scope = Default::default(); self.scopes.push(scope); + // self.work_scopes.push(Default::default()); } fn leave_scope(&mut self) { // Pop scope and add to subscopes of parent scope. + // let work_scope = self.work_scopes.pop().unwrap(); let scope = self.scopes.pop().unwrap(); self.scopes.last_mut().unwrap().sub_scopes.push(scope); } - pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { + fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { self.scan_statements(&program.statements)?; Ok(()) } - pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult { + fn scan_statements(&mut self, statements: &[ast::Statement]) -> SymbolTableResult { for statement in statements { self.scan_statement(statement)?; } @@ -210,26 +221,27 @@ impl SymbolTableBuilder { Ok(()) } - fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult { + fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult { + use ast::StatementType::*; match &statement.node { - ast::Statement::Global { names } => { + Global { names } => { for name in names { self.register_name(name, SymbolRole::Global)?; } } - ast::Statement::Nonlocal { names } => { + Nonlocal { names } => { for name in names { self.register_name(name, SymbolRole::Nonlocal)?; } } - ast::Statement::FunctionDef { + FunctionDef { name, body, args, decorator_list, returns, } - | ast::Statement::AsyncFunctionDef { + | AsyncFunctionDef { name, body, args, @@ -247,7 +259,7 @@ impl SymbolTableBuilder { } self.leave_scope(); } - ast::Statement::ClassDef { + ClassDef { name, body, bases, @@ -264,21 +276,21 @@ impl SymbolTableBuilder { } self.scan_expressions(decorator_list)?; } - ast::Statement::Expression { expression } => self.scan_expression(expression)?, - ast::Statement::If { test, body, orelse } => { + Expression { expression } => self.scan_expression(expression)?, + If { test, body, orelse } => { self.scan_expression(test)?; self.scan_statements(body)?; if let Some(code) = orelse { self.scan_statements(code)?; } } - ast::Statement::For { + For { target, iter, body, orelse, } - | ast::Statement::AsyncFor { + | AsyncFor { target, iter, body, @@ -291,17 +303,17 @@ impl SymbolTableBuilder { self.scan_statements(code)?; } } - ast::Statement::While { test, body, orelse } => { + While { test, body, orelse } => { self.scan_expression(test)?; self.scan_statements(body)?; if let Some(code) = orelse { self.scan_statements(code)?; } } - ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { + Break | Continue | Pass => { // No symbols here. } - ast::Statement::Import { names } | ast::Statement::ImportFrom { names, .. } => { + Import { names } | ImportFrom { names, .. } => { for name in names { if let Some(alias) = &name.alias { // `import mymodule as myalias` @@ -312,29 +324,29 @@ impl SymbolTableBuilder { } } } - ast::Statement::Return { value } => { + Return { value } => { if let Some(expression) = value { self.scan_expression(expression)?; } } - ast::Statement::Assert { test, msg } => { + Assert { test, msg } => { self.scan_expression(test)?; if let Some(expression) = msg { self.scan_expression(expression)?; } } - ast::Statement::Delete { targets } => { + Delete { targets } => { self.scan_expressions(targets)?; } - ast::Statement::Assign { targets, value } => { + Assign { targets, value } => { self.scan_expressions(targets)?; self.scan_expression(value)?; } - ast::Statement::AugAssign { target, value, .. } => { + AugAssign { target, value, .. } => { self.scan_expression(target)?; self.scan_expression(value)?; } - ast::Statement::With { items, body } => { + With { items, body } => { for item in items { self.scan_expression(&item.context_expr)?; if let Some(expression) = &item.optional_vars { @@ -343,7 +355,7 @@ impl SymbolTableBuilder { } self.scan_statements(body)?; } - ast::Statement::Try { + Try { body, handlers, orelse, @@ -366,7 +378,7 @@ impl SymbolTableBuilder { self.scan_statements(code)?; } } - ast::Statement::Raise { exception, cause } => { + Raise { exception, cause } => { if let Some(expression) = exception { self.scan_expression(expression)?; } @@ -386,26 +398,27 @@ impl SymbolTableBuilder { } fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { - match expression { - ast::Expression::Binop { a, b, .. } => { + use ast::ExpressionType::*; + match &expression.node { + Binop { a, b, .. } => { self.scan_expression(a)?; self.scan_expression(b)?; } - ast::Expression::BoolOp { a, b, .. } => { + BoolOp { a, b, .. } => { self.scan_expression(a)?; self.scan_expression(b)?; } - ast::Expression::Compare { vals, .. } => { + Compare { vals, .. } => { self.scan_expressions(vals)?; } - ast::Expression::Subscript { a, b } => { + Subscript { a, b } => { self.scan_expression(a)?; self.scan_expression(b)?; } - ast::Expression::Attribute { value, .. } => { + Attribute { value, .. } => { self.scan_expression(value)?; } - ast::Expression::Dict { elements } => { + Dict { elements } => { for (key, value) in elements { if let Some(key) = key { self.scan_expression(key)?; @@ -415,36 +428,30 @@ impl SymbolTableBuilder { self.scan_expression(value)?; } } - ast::Expression::Await { value } => { + Await { value } => { self.scan_expression(value)?; } - ast::Expression::Yield { value } => { + Yield { value } => { if let Some(expression) = value { self.scan_expression(expression)?; } } - ast::Expression::YieldFrom { value } => { + YieldFrom { value } => { self.scan_expression(value)?; } - ast::Expression::Unop { a, .. } => { + Unop { a, .. } => { self.scan_expression(a)?; } - ast::Expression::True - | ast::Expression::False - | ast::Expression::None - | ast::Expression::Ellipsis => {} - ast::Expression::Number { .. } => {} - ast::Expression::Starred { value } => { + True | False | None | Ellipsis => {} + Number { .. } => {} + Starred { value } => { self.scan_expression(value)?; } - ast::Expression::Bytes { .. } => {} - ast::Expression::Tuple { elements } - | ast::Expression::Set { elements } - | ast::Expression::List { elements } - | ast::Expression::Slice { elements } => { + Bytes { .. } => {} + Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => { self.scan_expressions(elements)?; } - ast::Expression::Comprehension { kind, generators } => { + Comprehension { kind, generators } => { match **kind { ast::ComprehensionKind::GeneratorExpression { ref element } | ast::ComprehensionKind::List { ref element } @@ -465,7 +472,7 @@ impl SymbolTableBuilder { } } } - ast::Expression::Call { + Call { function, args, keywords, @@ -476,18 +483,18 @@ impl SymbolTableBuilder { self.scan_expression(&keyword.value)?; } } - ast::Expression::String { value } => { + String { value } => { self.scan_string_group(value)?; } - ast::Expression::Identifier { name } => { + Identifier { name } => { self.register_name(name, SymbolRole::Used)?; } - ast::Expression::Lambda { args, body } => { + Lambda { args, body } => { self.enter_function(args)?; self.scan_expression(body)?; self.leave_scope(); } - ast::Expression::IfExpression { test, body, orelse } => { + IfExpression { test, body, orelse } => { self.scan_expression(test)?; self.scan_expression(body)?; self.scan_expression(orelse)?; @@ -549,6 +556,8 @@ impl SymbolTableBuilder { let scope_depth = self.scopes.len(); let current_scope = self.scopes.last_mut().unwrap(); let location = Default::default(); + + // Some checks: if current_scope.symbols.contains_key(name) { // Role already set.. match role { @@ -568,22 +577,47 @@ impl SymbolTableBuilder { // Ok? } } - } else { - match role { - SymbolRole::Nonlocal => { - if scope_depth < 2 { - return Err(SymbolTableError { - error: format!("cannot define nonlocal '{}' at top level.", name), - location, - }); - } - } - _ => { - // Ok! + } + + // Some more checks: + match role { + SymbolRole::Nonlocal => { + if scope_depth < 2 { + return Err(SymbolTableError { + error: format!("cannot define nonlocal '{}' at top level.", name), + location, + }); } } - current_scope.symbols.insert(name.to_string(), role); + _ => { + // Ok! + } } + + // Insert symbol when required: + if !current_scope.symbols.contains_key(name) { + let symbol = Symbol::new(name); + current_scope.symbols.insert(name.to_string(), symbol); + } + + // Set proper flags on symbol: + let symbol = current_scope.symbols.get_mut(name).unwrap(); + match role { + SymbolRole::Nonlocal => { + symbol.is_nonlocal = true; + } + SymbolRole::Assigned => { + symbol.is_assigned = true; + // symbol.is_local = true; + } + SymbolRole::Global => { + symbol.is_global = true; + } + SymbolRole::Used => { + symbol.is_referenced = true; + } + } + Ok(()) } } diff --git a/crawl_sourcecode.py b/crawl_sourcecode.py index c58672ec1..bdb7a15b0 100644 --- a/crawl_sourcecode.py +++ b/crawl_sourcecode.py @@ -26,7 +26,8 @@ print(t) shift = 3 def print_node(node, indent=0): if isinstance(node, ast.AST): - print(' '*indent, "NODE", node.__class__.__name__) + lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else '' + print(' '*indent, "NODE", node.__class__.__name__, lineno) for field in node._fields: print(' '*indent,'-', field) f = getattr(node, field) @@ -41,12 +42,25 @@ def print_node(node, indent=0): print_node(t) # print(ast.dump(t)) +flag_names = [ + 'is_referenced', + 'is_assigned', + 'is_global', + 'is_local', + 'is_parameter', + 'is_free', +] def print_table(table, indent=0): print(' '*indent, 'table:', table.get_name()) print(' '*indent, ' ', 'Syms:') for sym in table.get_symbols(): - print(' '*indent, ' ', sym.get_name(), 'is_referenced=', sym.is_referenced(), 'is_assigned=', sym.is_assigned()) + flags = [] + for flag_name in flag_names: + func = getattr(sym, flag_name) + if func(): + flags.append(flag_name) + print(' '*indent, ' sym:', sym.get_name(), 'flags:', ' '.join(flags)) print(' '*indent, ' ', 'Child tables:') for child in table.get_children(): print_table(child, indent=indent+shift) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index b54a528df..a892e5f56 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -17,13 +17,13 @@ pub struct Node { #[derive(Debug, PartialEq)] pub enum Top { Program(Program), - Statement(Vec), + Statement(Vec), Expression(Expression), } #[derive(Debug, PartialEq)] pub struct Program { - pub statements: Vec, + pub statements: Vec, } #[derive(Debug, PartialEq)] @@ -38,12 +38,12 @@ pub struct Located { pub node: T, } -pub type LocatedStatement = Located; +pub type Statement = Located; /// Abstract syntax tree nodes for python statements. #[allow(clippy::large_enum_variant)] #[derive(Debug, PartialEq)] -pub enum Statement { +pub enum StatementType { Break, Continue, Return { @@ -85,43 +85,43 @@ pub enum Statement { }, If { test: Expression, - body: Vec, - orelse: Option>, + body: Vec, + orelse: Option>, }, While { test: Expression, - body: Vec, - orelse: Option>, + body: Vec, + orelse: Option>, }, With { items: Vec, - body: Vec, + body: Vec, }, For { target: Expression, iter: Expression, - body: Vec, - orelse: Option>, + body: Vec, + orelse: Option>, }, AsyncFor { target: Expression, iter: Expression, - body: Vec, - orelse: Option>, + body: Vec, + orelse: Option>, }, Raise { exception: Option, cause: Option, }, Try { - body: Vec, + body: Vec, handlers: Vec, - orelse: Option>, - finalbody: Option>, + orelse: Option>, + finalbody: Option>, }, ClassDef { name: String, - body: Vec, + body: Vec, bases: Vec, keywords: Vec, decorator_list: Vec, @@ -129,14 +129,14 @@ pub enum Statement { FunctionDef { name: String, args: Parameters, - body: Vec, + body: Vec, decorator_list: Vec, returns: Option, }, AsyncFunctionDef { name: String, args: Parameters, - body: Vec, + body: Vec, decorator_list: Vec, returns: Option, }, @@ -148,8 +148,10 @@ pub struct WithItem { pub optional_vars: Option, } +pub type Expression = Located; + #[derive(Debug, PartialEq)] -pub enum Expression { +pub enum ExpressionType { BoolOp { a: Box, op: BooleanOperator, @@ -242,10 +244,10 @@ pub enum Expression { impl Expression { /// Returns a short name for the node suitable for use in error messages. pub fn name(&self) -> &'static str { - use self::Expression::*; + use self::ExpressionType::*; use self::StringGroup::*; - match self { + match &self.node { BoolOp { .. } | Binop { .. } | Unop { .. } => "operator", Subscript { .. } => "subscript", Await { .. } => "await expression", @@ -330,7 +332,7 @@ pub struct Keyword { pub struct ExceptHandler { pub typ: Option, pub name: Option, - pub body: Vec, + pub body: Vec, } #[derive(Debug, PartialEq)] diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 6b90a9fcd..e836c8c4b 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -173,9 +173,12 @@ mod tests { use super::*; - fn mk_ident(name: &str) -> ast::Expression { - ast::Expression::Identifier { - name: name.to_owned(), + fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { + ast::Expression { + location: ast::Location::new(row, col), + node: ast::ExpressionType::Identifier { + name: name.to_owned(), + }, } } @@ -189,12 +192,12 @@ mod tests { Joined { values: vec![ FormattedValue { - value: Box::new(mk_ident("a")), + value: Box::new(mk_ident("a", 1, 1)), conversion: None, spec: String::new(), }, FormattedValue { - value: Box::new(mk_ident("b")), + value: Box::new(mk_ident("b", 1, 1)), conversion: None, spec: String::new(), }, diff --git a/parser/src/parser.rs b/parser/src/parser.rs index eee419310..a3ca48c7a 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result { do_lalr_parsing!(source, Program, StartProgram) } -pub fn parse_statement(source: &str) -> Result, ParseError> { +pub fn parse_statement(source: &str) -> Result, ParseError> { do_lalr_parsing!(source, Statement, StartStatement) } @@ -46,16 +46,25 @@ pub fn parse_statement(source: &str) -> Result, Parse /// extern crate num_bigint; /// use num_bigint::BigInt; /// use rustpython_parser::{parser, ast}; -/// let expr = parser::parse_expression("1+2").unwrap(); +/// let expr = parser::parse_expression("1 + 2").unwrap(); /// -/// assert_eq!(ast::Expression::Binop { -/// a: Box::new(ast::Expression::Number { -/// value: ast::Number::Integer { value: BigInt::from(1) } -/// }), -/// op: ast::Operator::Add, -/// b: Box::new(ast::Expression::Number { -/// value: ast::Number::Integer { value: BigInt::from(2) } -/// }) +/// assert_eq!(ast::Expression { +/// location: ast::Location::new(1, 3), +/// node: ast::ExpressionType::Binop { +/// a: Box::new(ast::Expression { +/// location: ast::Location::new(1, 1), +/// node: ast::ExpressionType::Number { +/// value: ast::Number::Integer { value: BigInt::from(1) } +/// } +/// }), +/// op: ast::Operator::Add, +/// b: Box::new(ast::Expression { +/// location: ast::Location::new(1, 5), +/// node: ast::ExpressionType::Number { +/// value: ast::Number::Integer { value: BigInt::from(2) } +/// } +/// }) +/// } /// }, /// expr); /// @@ -72,6 +81,44 @@ mod tests { use super::parse_statement; use num_bigint::BigInt; + fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { + ast::Expression { + location: ast::Location::new(row, col), + node: ast::ExpressionType::Identifier { + name: name.to_owned(), + }, + } + } + + fn make_int(value: i32, row: usize, col: usize) -> ast::Expression { + ast::Expression { + location: ast::Location::new(row, col), + node: ast::ExpressionType::Number { + value: ast::Number::Integer { + value: BigInt::from(value), + }, + }, + } + } + + fn make_string(value: &str, row: usize, col: usize) -> ast::Expression { + ast::Expression { + location: ast::Location::new(row, col), + node: ast::ExpressionType::String { + value: ast::StringGroup::Constant { + value: String::from(value), + }, + }, + } + } + + fn as_statement(expr: ast::Expression) -> ast::Statement { + ast::Statement { + location: expr.location.clone(), + node: ast::StatementType::Expression { expression: expr }, + } + } + #[test] fn test_parse_empty() { let parse_ast = parse_program(&String::from("\n")); @@ -85,19 +132,16 @@ mod tests { assert_eq!( parse_ast, ast::Program { - statements: vec![ast::LocatedStatement { + statements: vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::Expression { - expression: ast::Expression::Call { - function: Box::new(ast::Expression::Identifier { - name: String::from("print"), - }), - args: vec![ast::Expression::String { - value: ast::StringGroup::Constant { - value: String::from("Hello world") - } - }], - keywords: vec![], + node: ast::StatementType::Expression { + expression: ast::Expression { + location: ast::Location::new(1, 6), + node: ast::ExpressionType::Call { + function: Box::new(mk_ident("print", 1, 1)), + args: vec![make_string("Hello world", 1, 8)], + keywords: vec![], + } }, }, },], @@ -112,26 +156,16 @@ mod tests { assert_eq!( parse_ast, ast::Program { - statements: vec![ast::LocatedStatement { + statements: vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::Expression { - expression: ast::Expression::Call { - function: Box::new(ast::Expression::Identifier { - name: String::from("print"), - }), - args: vec![ - ast::Expression::String { - value: ast::StringGroup::Constant { - value: String::from("Hello world"), - } - }, - ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(2) - }, - } - ], - keywords: vec![], + node: ast::StatementType::Expression { + expression: ast::Expression { + location: ast::Location::new(1, 6), + node: ast::ExpressionType::Call { + function: Box::new(mk_ident("print", 1, 1)), + args: vec![make_string("Hello world", 1, 8), make_int(2, 1, 22),], + keywords: vec![], + }, }, }, },], @@ -146,26 +180,19 @@ mod tests { assert_eq!( parse_ast, ast::Program { - statements: vec![ast::LocatedStatement { + statements: vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::Expression { - expression: ast::Expression::Call { - function: Box::new(ast::Expression::Identifier { - name: String::from("my_func"), - }), - args: vec![ast::Expression::String { - value: ast::StringGroup::Constant { - value: String::from("positional"), - } - }], - keywords: vec![ast::Keyword { - name: Some("keyword".to_string()), - value: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(2) - }, - } - }], + node: ast::StatementType::Expression { + expression: ast::Expression { + location: ast::Location::new(1, 8), + node: ast::ExpressionType::Call { + function: Box::new(mk_ident("my_func", 1, 1)), + args: vec![make_string("positional", 1, 10)], + keywords: vec![ast::Keyword { + name: Some("keyword".to_string()), + value: make_int(2, 1, 31), + }], + } }, }, },], @@ -179,52 +206,17 @@ mod tests { let parse_ast = parse_statement(&source).unwrap(); assert_eq!( parse_ast, - vec![ast::LocatedStatement { + vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::If { - test: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(1) - }, - }, - body: vec![ast::LocatedStatement { - location: ast::Location::new(1, 7), - node: ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(10) - }, - } - }, - },], - orelse: Some(vec![ast::LocatedStatement { + node: ast::StatementType::If { + test: make_int(1, 1, 4), + body: vec![as_statement(make_int(10, 1, 7))], + orelse: Some(vec![ast::Statement { location: ast::Location::new(2, 1), - node: ast::Statement::If { - test: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(2) - }, - }, - body: vec![ast::LocatedStatement { - location: ast::Location::new(2, 9), - node: ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(20) - }, - }, - }, - },], - orelse: Some(vec![ast::LocatedStatement { - location: ast::Location::new(3, 7), - node: ast::Statement::Expression { - expression: ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(30) - }, - }, - }, - },]), + node: ast::StatementType::If { + test: make_int(2, 2, 6), + body: vec![as_statement(make_int(20, 2, 9))], + orelse: Some(vec![as_statement(make_int(30, 3, 7))]), } },]), } @@ -238,39 +230,36 @@ mod tests { let parse_ast = parse_statement(&source); assert_eq!( parse_ast, - Ok(vec![ast::LocatedStatement { + Ok(vec![as_statement(ast::Expression { location: ast::Location::new(1, 1), - node: ast::Statement::Expression { - expression: ast::Expression::Lambda { - args: ast::Parameters { - args: vec![ - ast::Parameter { - arg: String::from("x"), - annotation: None, - }, - ast::Parameter { - arg: String::from("y"), - annotation: None, - } - ], - kwonlyargs: vec![], - vararg: ast::Varargs::None, - kwarg: ast::Varargs::None, - defaults: vec![], - kw_defaults: vec![], - }, - body: Box::new(ast::Expression::Binop { - a: Box::new(ast::Expression::Identifier { - name: String::from("x"), - }), + node: ast::ExpressionType::Lambda { + args: ast::Parameters { + args: vec![ + ast::Parameter { + arg: String::from("x"), + annotation: None, + }, + ast::Parameter { + arg: String::from("y"), + annotation: None, + } + ], + kwonlyargs: vec![], + vararg: ast::Varargs::None, + kwarg: ast::Varargs::None, + defaults: vec![], + kw_defaults: vec![], + }, + body: Box::new(ast::Expression { + location: ast::Location::new(1, 16), + node: ast::ExpressionType::Binop { + a: Box::new(mk_ident("x", 1, 14)), op: ast::Operator::Mult, - b: Box::new(ast::Expression::Identifier { - name: String::from("y"), - }) - }) - } + b: Box::new(mk_ident("y", 1, 18)) + } + }) } - }]) + })]) ) } @@ -280,32 +269,20 @@ mod tests { assert_eq!( parse_statement(&source), - Ok(vec![ast::LocatedStatement { + Ok(vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::Assign { - targets: vec![ast::Expression::Tuple { - elements: vec![ - ast::Expression::Identifier { - name: "a".to_string() - }, - ast::Expression::Identifier { - name: "b".to_string() - } - ] + node: ast::StatementType::Assign { + targets: vec![ast::Expression { + location: ast::Location::new(1, 1), + node: ast::ExpressionType::Tuple { + elements: vec![mk_ident("a", 1, 1), mk_ident("b", 1, 4),] + } }], - value: ast::Expression::Tuple { - elements: vec![ - ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(4) - } - }, - ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(5) - } - } - ] + value: ast::Expression { + location: ast::Location::new(1, 8), + node: ast::ExpressionType::Tuple { + elements: vec![make_int(4, 1, 8), make_int(5, 1, 11),] + } } } }]) @@ -319,23 +296,16 @@ mod tests { ); assert_eq!( parse_statement(&source), - Ok(vec![ast::LocatedStatement { + Ok(vec![ast::Statement { location: ast::Location::new(1, 1), - node: ast::Statement::ClassDef { + node: ast::StatementType::ClassDef { name: String::from("Foo"), - bases: vec![ - ast::Expression::Identifier { - name: String::from("A") - }, - ast::Expression::Identifier { - name: String::from("B") - } - ], + bases: vec![mk_ident("A", 1, 11), mk_ident("B", 1, 14)], keywords: vec![], body: vec![ - ast::LocatedStatement { + ast::Statement { location: ast::Location::new(2, 2), - node: ast::Statement::FunctionDef { + node: ast::StatementType::FunctionDef { name: String::from("__init__"), args: ast::Parameters { args: vec![ast::Parameter { @@ -348,17 +318,17 @@ mod tests { defaults: vec![], kw_defaults: vec![], }, - body: vec![ast::LocatedStatement { + body: vec![ast::Statement { location: ast::Location::new(3, 3), - node: ast::Statement::Pass, + node: ast::StatementType::Pass, }], decorator_list: vec![], returns: None, } }, - ast::LocatedStatement { + ast::Statement { location: ast::Location::new(4, 2), - node: ast::Statement::FunctionDef { + node: ast::StatementType::FunctionDef { name: String::from("method_with_default"), args: ast::Parameters { args: vec![ @@ -374,16 +344,12 @@ mod tests { kwonlyargs: vec![], vararg: ast::Varargs::None, kwarg: ast::Varargs::None, - defaults: vec![ast::Expression::String { - value: ast::StringGroup::Constant { - value: "default".to_string() - } - }], + defaults: vec![make_string("default", 4, 37)], kw_defaults: vec![], }, - body: vec![ast::LocatedStatement { + body: vec![ast::Statement { location: ast::Location::new(5, 3), - node: ast::Statement::Pass, + node: ast::StatementType::Pass, }], decorator_list: vec![], returns: None, @@ -402,21 +368,18 @@ mod tests { let parse_ast = parse_expression(&source).unwrap(); assert_eq!( parse_ast, - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { - element: ast::Expression::Identifier { - name: "x".to_string() - } - }), - generators: vec![ast::Comprehension { - target: ast::Expression::Identifier { - name: "y".to_string() - }, - iter: ast::Expression::Identifier { - name: "z".to_string() - }, - ifs: vec![], - }], + ast::Expression { + location: ast::Location::new(1, 2), + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::List { + element: mk_ident("x", 1, 2), + }), + generators: vec![ast::Comprehension { + target: mk_ident("y", 1, 8), + iter: mk_ident("z", 1, 13), + ifs: vec![], + }], + } } ); } @@ -427,66 +390,45 @@ mod tests { let parse_ast = parse_expression(&source).unwrap(); assert_eq!( parse_ast, - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { - element: ast::Expression::Identifier { - name: "x".to_string() - } - }), - generators: vec![ - ast::Comprehension { - target: ast::Expression::Tuple { - elements: vec![ - ast::Expression::Identifier { - name: "y".to_string() + ast::Expression { + location: ast::Location::new(1, 2), + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::List { + element: mk_ident("x", 1, 2) + }), + generators: vec![ + ast::Comprehension { + target: ast::Expression { + location: ast::Location::new(1, 8), + node: ast::ExpressionType::Tuple { + elements: vec![mk_ident("y", 1, 8), mk_ident("y2", 1, 11),], + } + }, + iter: mk_ident("z", 1, 17), + ifs: vec![], + }, + ast::Comprehension { + target: mk_ident("a", 1, 23), + iter: mk_ident("b", 1, 28), + ifs: vec![ + ast::Expression { + location: ast::Location::new(1, 35), + node: ast::ExpressionType::Compare { + vals: vec![mk_ident("a", 1, 33), make_int(5, 1, 37),], + ops: vec![ast::Comparison::Less], + } }, - ast::Expression::Identifier { - name: "y2".to_string() + ast::Expression { + location: ast::Location::new(1, 44), + node: ast::ExpressionType::Compare { + vals: vec![mk_ident("a", 1, 42), make_int(10, 1, 46),], + ops: vec![ast::Comparison::Greater], + }, }, ], - }, - iter: ast::Expression::Identifier { - name: "z".to_string() - }, - ifs: vec![], - }, - ast::Comprehension { - target: ast::Expression::Identifier { - name: "a".to_string() - }, - iter: ast::Expression::Identifier { - name: "b".to_string() - }, - ifs: vec![ - ast::Expression::Compare { - vals: vec![ - ast::Expression::Identifier { - name: "a".to_string() - }, - ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(5) - } - } - ], - ops: vec![ast::Comparison::Less], - }, - ast::Expression::Compare { - vals: vec![ - ast::Expression::Identifier { - name: "a".to_string() - }, - ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(10) - } - } - ], - ops: vec![ast::Comparison::Greater], - }, - ], - } - ], + } + ], + } } ); } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 8132ad9f3..d08efc041 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -31,22 +31,22 @@ Program: ast::Program = { }; // A file line either has a declaration, or an empty newline: -FileLine: Vec = { +FileLine: Vec = { Statement, "\n" => vec![], }; -Suite: Vec = { +Suite: Vec = { SimpleStatement, "\n" indent dedent => s.into_iter().flatten().collect(), }; -Statement: Vec = { +Statement: Vec = { SimpleStatement, => vec![s], }; -SimpleStatement: Vec = { +SimpleStatement: Vec = { ";"? "\n" => { let mut statements = vec![s1]; statements.extend(s2.into_iter().map(|e| e.1)); @@ -54,7 +54,7 @@ SimpleStatement: Vec = { } }; -SmallStatement: ast::LocatedStatement = { +SmallStatement: ast::Statement = { ExpressionStatement, PassStatement, DelStatement, @@ -65,52 +65,52 @@ SmallStatement: ast::LocatedStatement = { AssertStatement, }; -PassStatement: ast::LocatedStatement = { +PassStatement: ast::Statement = { "pass" => { - ast::LocatedStatement { + ast::Statement { location: loc, - node: ast::Statement::Pass, + node: ast::StatementType::Pass, } }, }; -DelStatement: ast::LocatedStatement = { +DelStatement: ast::Statement = { "del" => { - ast::LocatedStatement { + ast::Statement { location: loc, - node: ast::Statement::Delete { targets: e }, + node: ast::StatementType::Delete { targets: e }, } }, }; -ExpressionStatement: ast::LocatedStatement = { - => { +ExpressionStatement: ast::Statement = { + => { // Just an expression, no assignment: if suffix.is_empty() { - ast::LocatedStatement { - location: loc.clone(), - node: ast::Statement::Expression { expression: expr } + ast::Statement { + location, + node: ast::StatementType::Expression { expression: expr } } } else { - let mut targets = vec![expr]; - let mut values = suffix; + let mut targets = vec![expr]; + let mut values = suffix; - while values.len() > 1 { + while values.len() > 1 { targets.push(values.remove(0)); - } + } - let value = values.into_iter().next().unwrap(); + let value = values.into_iter().next().unwrap(); - ast::LocatedStatement { - location: loc.clone(), - node: ast::Statement::Assign { targets, value }, - } + ast::Statement { + location, + node: ast::StatementType::Assign { targets, value }, + } } }, - => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::AugAssign { + => { + ast::Statement { + location, + node: ast::StatementType::AugAssign { target: Box::new(expr), op, value: Box::new(rhs) @@ -125,11 +125,14 @@ AssignSuffix: ast::Expression = { }; TestOrStarExprList: ast::Expression = { - > => { + > => { if elements.len() == 1 && comma.is_none() { elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements } + ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements } + } } } }; @@ -155,60 +158,60 @@ AugAssign: ast::Operator = { "//=" => ast::Operator::FloorDiv, }; -FlowStatement: ast::LocatedStatement = { - "break" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Break, +FlowStatement: ast::Statement = { + "break" => { + ast::Statement { + location, + node: ast::StatementType::Break, } }, - "continue" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Continue, + "continue" => { + ast::Statement { + location, + node: ast::StatementType::Continue, } }, - "return" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Return { value }, + "return" => { + ast::Statement { + location, + node: ast::StatementType::Return { value }, } }, - => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Expression { expression: y }, + => { + ast::Statement { + location, + node: ast::StatementType::Expression { expression: y }, } }, RaiseStatement, }; -RaiseStatement: ast::LocatedStatement = { - "raise" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Raise { exception: None, cause: None }, +RaiseStatement: ast::Statement = { + "raise" => { + ast::Statement { + location, + node: ast::StatementType::Raise { exception: None, cause: None }, } }, - "raise" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Raise { exception: Some(t), cause: c.map(|x| x.1) }, + "raise" => { + ast::Statement { + location, + node: ast::StatementType::Raise { exception: Some(t), cause: c.map(|x| x.1) }, } }, }; -ImportStatement: ast::LocatedStatement = { - "import" >>> => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Import { names }, +ImportStatement: ast::Statement = { + "import" >>> => { + ast::Statement { + location, + node: ast::StatementType::Import { names }, } }, - "from" "import" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::ImportFrom { + "from" "import" => { + ast::Statement { + location, + node: ast::StatementType::ImportFrom { level: n.0, module: n.1, names @@ -259,36 +262,36 @@ DottedName: String = { }, }; -GlobalStatement: ast::LocatedStatement = { - "global" > => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Global { names } +GlobalStatement: ast::Statement = { + "global" > => { + ast::Statement { + location, + node: ast::StatementType::Global { names } } }, }; -NonlocalStatement: ast::LocatedStatement = { - "nonlocal" > => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Nonlocal { names } +NonlocalStatement: ast::Statement = { + "nonlocal" > => { + ast::Statement { + location, + node: ast::StatementType::Nonlocal { names } } }, }; -AssertStatement: ast::LocatedStatement = { - "assert" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::Assert { +AssertStatement: ast::Statement = { + "assert" => { + ast::Statement { + location, + node: ast::StatementType::Assert { test, msg: msg.map(|e| e.1) } } }, }; -CompoundStatement: ast::LocatedStatement = { +CompoundStatement: ast::Statement = { IfStatement, WhileStatement, ForStatement, @@ -298,58 +301,58 @@ CompoundStatement: ast::LocatedStatement = { ClassDef, }; -IfStatement: ast::LocatedStatement = { - "if" ":" => { +IfStatement: ast::Statement = { + "if" ":" => { // Determine last else: let mut last = s3.map(|s| s.2); // handle elif: for i in s2.into_iter().rev() { - let x = ast::LocatedStatement { + let x = ast::Statement { location: i.0, - node: ast::Statement::If { test: i.2, body: i.4, orelse: last }, + node: ast::StatementType::If { test: i.2, body: i.4, orelse: last }, }; last = Some(vec![x]); } - ast::LocatedStatement { - location: loc, - node: ast::Statement::If { test, body: s1, orelse: last } + ast::Statement { + location, + node: ast::StatementType::If { test, body: s1, orelse: last } } }, }; -WhileStatement: ast::LocatedStatement = { - "while" ":" => { +WhileStatement: ast::Statement = { + "while" ":" => { let or_else = s2.map(|s| s.2); - ast::LocatedStatement { - location: loc, - node: ast::Statement::While { test, body, orelse: or_else }, + ast::Statement { + location, + node: ast::StatementType::While { test, body, orelse: or_else }, } }, }; -ForStatement: ast::LocatedStatement = { - "for" "in" ":" => { +ForStatement: ast::Statement = { + "for" "in" ":" => { let orelse = s2.map(|s| s.2); - ast::LocatedStatement { - location: loc, + ast::Statement { + location, node: if is_async.is_some() { - ast::Statement::AsyncFor { target, iter, body, orelse } + ast::StatementType::AsyncFor { target, iter, body, orelse } } else { - ast::Statement::For { target, iter, body, orelse } + ast::StatementType::For { target, iter, body, orelse } }, } }, }; -TryStatement: ast::LocatedStatement = { - "try" ":" => { +TryStatement: ast::Statement = { + "try" ":" => { let or_else = else_suite.map(|s| s.2); let finalbody = finally.map(|s| s.2); - ast::LocatedStatement { - location: loc, - node: ast::Statement::Try { + ast::Statement { + location, + node: ast::StatementType::Try { body: body, handlers: handlers, orelse: or_else, @@ -376,11 +379,11 @@ ExceptClause: ast::ExceptHandler = { }, }; -WithStatement: ast::LocatedStatement = { - "with" > ":" => { - ast::LocatedStatement { - location: loc, - node: ast::Statement::With { items: items, body: s }, +WithStatement: ast::Statement = { + "with" > ":" => { + ast::Statement { + location, + node: ast::StatementType::With { items: items, body: s }, } }, }; @@ -392,12 +395,12 @@ WithItem: ast::WithItem = { }, }; -FuncDef: ast::LocatedStatement = { - "def" " Test)?> ":" => { - ast::LocatedStatement { - location: loc, +FuncDef: ast::Statement = { + "def" " Test)?> ":" => { + ast::Statement { + location, node: if is_async.is_some() { - ast::Statement::AsyncFunctionDef { + ast::StatementType::AsyncFunctionDef { name: i, args: a, body: s, @@ -405,7 +408,7 @@ FuncDef: ast::LocatedStatement = { returns: r.map(|x| x.1), } } else { - ast::Statement::FunctionDef { + ast::StatementType::FunctionDef { name: i, args: a, body: s, @@ -552,43 +555,55 @@ KwargParameter: Option = { } }; -ClassDef: ast::LocatedStatement = { - "class" ":" => { +ClassDef: ast::Statement = { + "class" ":" => { let (bases, keywords) = match a { Some((_, args, _)) => args, None => (vec![], vec![]), }; - ast::LocatedStatement { - location: loc, - node: ast::Statement::ClassDef { - name: n, - bases: bases, - keywords: keywords, - body: s, - decorator_list: d, + ast::Statement { + location, + node: ast::StatementType::ClassDef { + name, + bases, + keywords, + body, + decorator_list: d, }, } }, }; Path: ast::Expression = { - => ast::Expression::Identifier { name: n }, - "." => { - ast::Expression::Attribute { - value: Box::new(p), - name: n, + => ast::Expression { + location, + node: ast::ExpressionType::Identifier { name: n } + }, + "." => { + ast::Expression { + location, + node: ast::ExpressionType::Attribute { + value: Box::new(p), + name: n, + } } }, }; // Decorators: Decorator: ast::Expression = { - "@" "\n" => { + "@" "\n" => { match a { - Some((_, args, _)) => ast::Expression::Call { - function: Box::new(p), - args: args.0, - keywords: args.1, + Some((location, _, args, _)) => { + let (args, keywords) = args; + ast::Expression { + location, + node: ast::ExpressionType::Call { + function: Box::new(p), + args, + keywords, + } + } }, None => p, } @@ -596,17 +611,26 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => ast::Expression::Yield { value: value.map(Box::new) }, - "yield" "from" => ast::Expression::YieldFrom { value: Box::new(e) }, + "yield" => ast::Expression { + location, + node: ast::ExpressionType::Yield { value: value.map(Box::new) } + }, + "yield" "from" => ast::Expression { + location, + node: ast::ExpressionType::YieldFrom { value: Box::new(e) } + }, }; Test: ast::Expression = { - => { + => { if let Some(c) = condition { - ast::Expression::IfExpression { - test: Box::new(c.1), - body: Box::new(expr), - orelse: Box::new(c.3), + ast::Expression { + location: c.0, + node: ast::ExpressionType::IfExpression { + test: Box::new(c.2), + body: Box::new(expr), + orelse: Box::new(c.4), + } } } else { expr @@ -616,37 +640,52 @@ Test: ast::Expression = { }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => - ast::Expression::Lambda { - args: p.unwrap_or(Default::default()), - body: Box::new(body) + "lambda" ?> ":" => + ast::Expression { + location, + node: ast::ExpressionType::Lambda { + args: p.unwrap_or(Default::default()), + body: Box::new(body) + } } } OrTest: ast::Expression = { AndTest, - "or" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }, + "or" => ast::Expression { + location, + node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) } + }, }; AndTest: ast::Expression = { NotTest, - "and" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }, + "and" => ast::Expression { + location, + node: ast::ExpressionType::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) } + }, }; NotTest: ast::Expression = { - "not" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }, + "not" => ast::Expression { + location, + node: ast::ExpressionType::Unop { a: Box::new(e), op: ast::UnaryOperator::Not } + }, Comparison, }; Comparison: ast::Expression = { - => { + => { let mut vals = vec![e]; let mut ops = vec![]; for x in comparisons { ops.push(x.0); vals.push(x.1); } - ast::Expression::Compare { vals, ops } + ast::Expression { + location, + node: ast::ExpressionType::Compare { vals, ops } + } }, Expression, }; @@ -665,22 +704,34 @@ CompOp: ast::Comparison = { }; Expression: ast::Expression = { - "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, + "|" => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) } + }, XorExpression, }; XorExpression: ast::Expression = { - "^" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) }, + "^" => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) } + }, AndExpression, }; AndExpression: ast::Expression = { - "&" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) }, + "&" => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) } + }, ShiftExpression, }; ShiftExpression: ast::Expression = { - => ast::Expression::Binop { a: Box::new(e1), op: op, b: Box::new(e2) }, + => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(e1), op: op, b: Box::new(e2) } + }, ArithmaticExpression, }; @@ -690,7 +741,10 @@ ShiftOp: ast::Operator = { }; ArithmaticExpression: ast::Expression = { - => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) }, + => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(a), op: op, b: Box::new(b) } + }, Term, }; @@ -700,7 +754,10 @@ AddOp: ast::Operator = { }; Term: ast::Expression = { - => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) }, + => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(a), op, b: Box::new(b) } + }, Factor, }; @@ -713,25 +770,38 @@ MulOp: ast::Operator = { }; Factor: ast::Expression = { - "+" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos }, - "-" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg }, - "~" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv }, + => ast::Expression { + location, + node: ast::ExpressionType::Unop { a: Box::new(e), op } + }, Power, }; +UnOp: ast::UnaryOperator = { + "+" => ast::UnaryOperator::Pos, + "-" => ast::UnaryOperator::Neg, + "~" => ast::UnaryOperator::Inv, +}; + Power: ast::Expression = { - => { + => { match e2 { None => e, - Some(x) => ast::Expression::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(x.1) }, + Some((location, _, b)) => ast::Expression { + location, + node: ast::ExpressionType::Binop { a: Box::new(e), op: ast::Operator::Pow, b: Box::new(b) } + }, } } }; AtomExpr: ast::Expression = { - => { + => { if is_await.is_some() { - ast::Expression::Await { value: Box::new(atom) } + ast::Expression { + location, + node: ast::ExpressionType::Await { value: Box::new(atom) } + } } else { atom } @@ -740,13 +810,25 @@ AtomExpr: ast::Expression = { AtomExpr2: ast::Expression = { Atom, - "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, - "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, - "." => ast::Expression::Attribute { value: Box::new(e), name: n }, + "(" ")" => { + let (args, keywords) = a; + ast::Expression { + location, + node: ast::ExpressionType::Call { function: Box::new(f), args, keywords } + } + }, + "[" "]" => ast::Expression { + location, + node: ast::ExpressionType::Subscript { a: Box::new(e), b: Box::new(s) } + }, + "." => ast::Expression { + location, + node: ast::ExpressionType::Attribute { value: Box::new(e), name } + }, }; SubscriptList: ast::Expression = { - ","? => { + ","? => { if s2.is_empty() { s1 } else { @@ -754,52 +836,86 @@ SubscriptList: ast::Expression = { for x in s2 { dims.push(x.1) } - ast::Expression::Tuple { elements: dims } + + ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements: dims }, + } } } }; Subscript: ast::Expression = { Test, - ":" => { - let s1 = e1.unwrap_or(ast::Expression::None); - let s2 = e2.unwrap_or(ast::Expression::None); - let s3 = e3.unwrap_or(ast::Expression::None); - ast::Expression::Slice { elements: vec![s1, s2, s3] } + ":" => { + let s1 = e1.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None }); + let s2 = e2.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None }); + let s3 = e3.unwrap_or(ast::Expression { location: location.clone(), node: ast::ExpressionType::None }); + ast::Expression { + location, + node: ast::ExpressionType::Slice { elements: vec![s1, s2, s3] } + } } }; SliceOp: ast::Expression = { - ":" => e.unwrap_or(ast::Expression::None) + ":" => e.unwrap_or(ast::Expression {location, node: ast::ExpressionType::None}) } Atom: ast::Expression = { - => ast::Expression::String { value }, - => ast::Expression::Bytes { value }, - => ast::Expression::Number { value }, - => ast::Expression::Identifier { name }, - "[" "]" => { + => ast::Expression { + location, + node: ast::ExpressionType::String { value } + }, + => ast::Expression { + location, + node: ast::ExpressionType::Bytes { value } + }, + => ast::Expression { + location, + node: ast::ExpressionType::Number { value } + }, + => ast::Expression { + location, + node: ast::ExpressionType::Identifier { name } + }, + "[" "]" => { let elements = e.unwrap_or(Vec::new()); - ast::Expression::List { elements } - }, - "[" "]" => e, - "(" ")" => { - elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() }) - }, - "(" ")" => { - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }), - generators: c, + ast::Expression { + location, + node: ast::ExpressionType::List { elements } } }, - "{" "}" => ast::Expression::Dict { elements: e.unwrap_or(Vec::new()) }, + "[" "]" => e, + "(" ")" => { + elements.unwrap_or(ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements: Vec::new() } + }) + }, + "(" ")" => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }), + generators: c, + } + } + }, + "{" "}" => ast::Expression { + location, + node: ast::ExpressionType::Dict { elements: e.unwrap_or(Vec::new()) } + }, "{" "}" => e, - "{" "}" => ast::Expression::Set { elements: e }, + "{" "}" => ast::Expression { + location, + node: ast::ExpressionType::Set { elements: e } + }, "{" "}" => e, - "True" => ast::Expression::True, - "False" => ast::Expression::False, - "None" => ast::Expression::None, - "..." => ast::Expression::Ellipsis, + "True" => ast::Expression { location, node: ast::ExpressionType::True }, + "False" => ast::Expression { location, node: ast::ExpressionType::False }, + "None" => ast::Expression { location, node: ast::ExpressionType::None }, + "..." => ast::Expression { location, node: ast::ExpressionType::Ellipsis }, }; TestListComp: Vec = { @@ -807,10 +923,13 @@ TestListComp: Vec = { }; TestListComp2: ast::Expression = { - => { - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { element: e }), - generators: c, + => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::List { element: e }), + generators: c, + } } }, }; @@ -820,10 +939,13 @@ TestDict: Vec<(Option, ast::Expression)> = { }; TestDictComp: ast::Expression = { - => { - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }), - generators: c, + => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }), + generators: c, + } } } }; @@ -842,10 +964,13 @@ TestSet: Vec = { }; TestSetComp: ast::Expression = { - => { - ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::Set { element: e1 }), - generators: c, + => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::Set { element: e1 }), + generators: c, + } } } }; @@ -856,11 +981,14 @@ ExpressionOrStarExpression = { }; ExpressionList: ast::Expression = { - > => { + > => { if elements.len() == 1 && trailing_comma.is_none() { elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements } + ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements }, + } } }, }; @@ -874,18 +1002,24 @@ ExpressionList2: Vec = { // - a single expression // - a single expression followed by a trailing comma TestList: ast::Expression = { - > => { + > => { if elements.len() == 1 && trailing_comma.is_none() { elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements } + ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements }, + } } } }; // Test StarExpr: ast::Expression = { - "*" => ast::Expression::Starred { value: Box::new(e) }, + "*" => ast::Expression { + location, + node: ast::ExpressionType::Starred { value: Box::new(e) }, + } }; // Comprehensions: @@ -911,7 +1045,7 @@ ArgumentList: (Vec, Vec) = { }, None => { // Allow starred args after keyword arguments. - let is_starred = if let ast::Expression::Starred { .. } = &value { + let is_starred = if let ast::ExpressionType::Starred { .. } = &value.node { true } else { false @@ -931,16 +1065,19 @@ ArgumentList: (Vec, Vec) = { FunctionArgument: (Option>, ast::Expression) = { => { let expr = match c { - Some(c) => ast::Expression::Comprehension { - kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }), - generators: c, + Some(c) => ast::Expression { + location: e.location.clone(), + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element: e }), + generators: c, + } }, None => e, }; (None, expr) }, "=" => (Some(Some(i.clone())), e), - "*" => (None, ast::Expression::Starred { value: Box::new(e) }), + "*" => (None, ast::Expression { location, node: ast::ExpressionType::Starred { value: Box::new(e) } }), "**" => (Some(None), e), }; diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index c335a7f0d..9687edbd5 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -58,10 +58,7 @@ fn create_node(vm: &VirtualMachine, name: &str) -> PyResult { AstNode.into_ref_with_type(vm, vm.class("ast", name)) } -fn statements_to_ast( - vm: &VirtualMachine, - statements: &[ast::LocatedStatement], -) -> PyResult { +fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::Statement]) -> PyResult { let body: PyResult> = statements .iter() .map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object())) @@ -69,12 +66,10 @@ fn statements_to_ast( Ok(vm.ctx.new_list(body?).downcast().unwrap()) } -fn statement_to_ast( - vm: &VirtualMachine, - statement: &ast::LocatedStatement, -) -> PyResult { +fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult { + use ast::StatementType::*; let node = match &statement.node { - ast::Statement::ClassDef { + ClassDef { name, body, keywords, @@ -86,7 +81,7 @@ fn statement_to_ast( body => statements_to_ast(vm, body)?, decorator_list => expressions_to_ast(vm, decorator_list)?, }), - ast::Statement::FunctionDef { + FunctionDef { name, args, body, @@ -99,7 +94,7 @@ fn statement_to_ast( decorator_list => expressions_to_ast(vm, decorator_list)?, returns => optional_expression_to_ast(vm, returns)? }), - ast::Statement::AsyncFunctionDef { + AsyncFunctionDef { name, args, body, @@ -112,14 +107,14 @@ fn statement_to_ast( decorator_list => expressions_to_ast(vm, decorator_list)?, returns => optional_expression_to_ast(vm, returns)? }), - ast::Statement::Continue => node!(vm, Continue), - ast::Statement::Break => node!(vm, Break), - ast::Statement::Pass => node!(vm, Pass), - ast::Statement::Assert { test, msg } => node!(vm, Assert, { + Continue => node!(vm, Continue), + Break => node!(vm, Break), + Pass => node!(vm, Pass), + Assert { test, msg } => node!(vm, Assert, { test => expression_to_ast(vm, test)?, msg => optional_expression_to_ast(vm, msg)? }), - ast::Statement::Delete { targets } => { + Delete { targets } => { let targets: PyResult<_> = targets .iter() .map(|v| Ok(expression_to_ast(vm, v)?.into_object())) @@ -127,15 +122,15 @@ fn statement_to_ast( let py_targets = vm.ctx.new_tuple(targets?); node!(vm, Delete, { targets => py_targets }) } - ast::Statement::Return { value } => node!(vm, Return, { + Return { value } => node!(vm, Return, { value => optional_expression_to_ast(vm, value)? }), - ast::Statement::If { test, body, orelse } => node!(vm, If, { + If { test, body, orelse } => node!(vm, If, { test => expression_to_ast(vm, test)?, body => statements_to_ast(vm, body)?, orelse => optional_statements_to_ast(vm, orelse)? }), - ast::Statement::For { + For { target, iter, body, @@ -146,7 +141,7 @@ fn statement_to_ast( body => statements_to_ast(vm, body)?, orelse => optional_statements_to_ast(vm, orelse)? }), - ast::Statement::AsyncFor { + AsyncFor { target, iter, body, @@ -157,16 +152,16 @@ fn statement_to_ast( body => statements_to_ast(vm, body)?, orelse => optional_statements_to_ast(vm, orelse)? }), - ast::Statement::While { test, body, orelse } => node!(vm, While, { + While { test, body, orelse } => node!(vm, While, { test => expression_to_ast(vm, test)?, body => statements_to_ast(vm, body)?, orelse => optional_statements_to_ast(vm, orelse)? }), - ast::Statement::With { items, body } => node!(vm, With, { + With { items, body } => node!(vm, With, { items => map_ast(with_item_to_ast, vm, items)?, body => statements_to_ast(vm, body)? }), - ast::Statement::Try { + Try { body, handlers, orelse, @@ -177,13 +172,13 @@ fn statement_to_ast( orelse => optional_statements_to_ast(vm, orelse)?, finalbody => optional_statements_to_ast(vm, finalbody)? }), - ast::Statement::Expression { expression } => node!(vm, Expr, { + Expression { expression } => node!(vm, Expr, { value => expression_to_ast(vm, expression)? }), - ast::Statement::Import { names } => node!(vm, Import, { + Import { names } => node!(vm, Import, { names => map_ast(alias_to_ast, vm, names)? }), - ast::Statement::ImportFrom { + ImportFrom { level, module, names, @@ -192,22 +187,22 @@ fn statement_to_ast( module => optional_string_to_py_obj(vm, module), names => map_ast(alias_to_ast, vm, names)? }), - ast::Statement::Nonlocal { names } => node!(vm, Nonlocal, { + Nonlocal { names } => node!(vm, Nonlocal, { names => make_string_list(vm, names) }), - ast::Statement::Global { names } => node!(vm, Global, { + Global { names } => node!(vm, Global, { names => make_string_list(vm, names) }), - ast::Statement::Assign { targets, value } => node!(vm, Assign, { + Assign { targets, value } => node!(vm, Assign, { targets => expressions_to_ast(vm, targets)?, value => expression_to_ast(vm, value)?, }), - ast::Statement::AugAssign { target, op, value } => node!(vm, AugAssign, { + AugAssign { target, op, value } => node!(vm, AugAssign, { target => expression_to_ast(vm, target)?, op => vm.ctx.new_str(operator_string(op)), value => expression_to_ast(vm, value)?, }), - ast::Statement::Raise { exception, cause } => node!(vm, Raise, { + Raise { exception, cause } => node!(vm, Raise, { exc => optional_expression_to_ast(vm, exception)?, cause => optional_expression_to_ast(vm, cause)?, }), @@ -229,7 +224,7 @@ fn alias_to_ast(vm: &VirtualMachine, alias: &ast::ImportSymbol) -> PyResult>, + statements: &Option>, ) -> PyResult { let statements = if let Some(statements) = statements { statements_to_ast(vm, statements)?.into_object() @@ -283,8 +278,9 @@ fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> P } fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult { - let node = match &expression { - ast::Expression::Call { + use ast::ExpressionType::*; + let node = match &expression.node { + Call { function, args, keywords, @@ -293,7 +289,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes args => expressions_to_ast(vm, args)?, keywords => map_ast(keyword_to_ast, vm, keywords)?, }), - ast::Expression::Binop { a, op, b } => { + Binop { a, op, b } => { // Operator: node!(vm, BinOp, { left => expression_to_ast(vm, a)?, @@ -301,7 +297,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes right => expression_to_ast(vm, b)?, }) } - ast::Expression::Unop { op, a } => { + Unop { op, a } => { let op = match op { ast::UnaryOperator::Not => "Not", ast::UnaryOperator::Inv => "Invert", @@ -313,7 +309,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes operand => expression_to_ast(vm, a)?, }) } - ast::Expression::BoolOp { a, op, b } => { + BoolOp { a, op, b } => { // Attach values: let py_a = expression_to_ast(vm, a)?.into_object(); let py_b = expression_to_ast(vm, b)?.into_object(); @@ -330,7 +326,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes values => py_values, }) } - ast::Expression::Compare { vals, ops } => { + Compare { vals, ops } => { let left = expression_to_ast(vm, &vals[0])?; // Operator: @@ -364,20 +360,20 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes comparators => comparators, }) } - ast::Expression::Identifier { name } => node!(vm, Name, { + Identifier { name } => node!(vm, Name, { id => vm.ctx.new_str(name.clone()), ctx => vm.ctx.none() // TODO: add context. }), - ast::Expression::Lambda { args, body } => node!(vm, Lambda, { + Lambda { args, body } => node!(vm, Lambda, { args => parameters_to_ast(vm, args)?, body => expression_to_ast(vm, body)?, }), - ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { + IfExpression { test, body, orelse } => node!(vm, IfExp, { text => expression_to_ast(vm, test)?, body => expression_to_ast(vm, body)?, or_else => expression_to_ast(vm, orelse)?, }), - ast::Expression::Number { value } => { + Number { value } => { let py_n = match value { ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), ast::Number::Float { value } => vm.ctx.new_float(*value), @@ -389,26 +385,26 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes n => py_n }) } - ast::Expression::True => node!(vm, NameConstant, { + True => node!(vm, NameConstant, { value => vm.ctx.new_bool(true) }), - ast::Expression::False => node!(vm, NameConstant, { + False => node!(vm, NameConstant, { value => vm.ctx.new_bool(false) }), - ast::Expression::None => node!(vm, NameConstant, { + None => node!(vm, NameConstant, { value => vm.ctx.none() }), - ast::Expression::Ellipsis => node!(vm, Ellipsis), - ast::Expression::List { elements } => node!(vm, List, { + Ellipsis => node!(vm, Ellipsis), + List { elements } => node!(vm, List, { elts => expressions_to_ast(vm, &elements)? }), - ast::Expression::Tuple { elements } => node!(vm, Tuple, { + Tuple { elements } => node!(vm, Tuple, { elts => expressions_to_ast(vm, &elements)? }), - ast::Expression::Set { elements } => node!(vm, Set, { + Set { elements } => node!(vm, Set, { elts => expressions_to_ast(vm, &elements)? }), - ast::Expression::Dict { elements } => { + Dict { elements } => { let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { @@ -425,7 +421,7 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes values => vm.ctx.new_list(values), }) } - ast::Expression::Comprehension { kind, generators } => { + Comprehension { kind, generators } => { let py_generators = map_ast(comprehension_to_ast, vm, generators)?; match kind.deref() { @@ -443,50 +439,48 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes } } } - ast::Expression::Await { value } => { + Await { value } => { let py_value = expression_to_ast(vm, value)?; node!(vm, Await, { value => py_value }) } - ast::Expression::Yield { value } => { - let py_value = match value { - Some(value) => expression_to_ast(vm, value)?.into_object(), - None => vm.ctx.none(), + Yield { value } => { + let py_value = if let Some(value) = value { + expression_to_ast(vm, value)?.into_object() + } else { + vm.ctx.none() }; node!(vm, Yield, { value => py_value }) } - ast::Expression::YieldFrom { value } => { + YieldFrom { value } => { let py_value = expression_to_ast(vm, value)?; node!(vm, YieldFrom, { value => py_value }) } - ast::Expression::Subscript { a, b } => node!(vm, Subscript, { + Subscript { a, b } => node!(vm, Subscript, { value => expression_to_ast(vm, a)?, slice => expression_to_ast(vm, b)?, }), - ast::Expression::Attribute { value, name } => node!(vm, Attribute, { + Attribute { value, name } => node!(vm, Attribute, { value => expression_to_ast(vm, value)?, attr => vm.ctx.new_str(name.to_string()), ctx => vm.ctx.none() }), - ast::Expression::Starred { value } => node!(vm, Starred, { + Starred { value } => node!(vm, Starred, { value => expression_to_ast(vm, value)? }), - ast::Expression::Slice { elements } => node!(vm, Slice, { + Slice { elements } => node!(vm, Slice, { bounds => expressions_to_ast(vm, elements)? }), - ast::Expression::String { value } => string_to_ast(vm, value)?, - ast::Expression::Bytes { value } => { - node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) - } + String { value } => string_to_ast(vm, value)?, + Bytes { value } => node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }), }; - // TODO: retrieve correct lineno: - let lineno = vm.ctx.new_int(1); + let lineno = vm.ctx.new_int(expression.location.row()); vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); Ok(node) } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index b189ad8db..dea27dfe0 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -61,7 +61,6 @@ pub fn get_module_inits() -> HashMap { "random".to_string() => Box::new(random::make_module), "_string".to_string() => Box::new(string::make_module), "struct".to_string() => Box::new(pystruct::make_module), - "symtable".to_string() => Box::new(symtable::make_module), "_thread".to_string() => Box::new(thread::make_module), "time".to_string() => Box::new(time_module::make_module), "_weakref".to_string() => Box::new(weakref::make_module), @@ -81,6 +80,12 @@ pub fn get_module_inits() -> HashMap { modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); } + // Insert compiler related modules: + #[cfg(feature = "rustpython-compiler")] + { + modules.insert("symtable".to_string(), Box::new(symtable::make_module)); + } + // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] { diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 8d6149835..692e1ce3b 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -89,11 +89,9 @@ impl PySymbolTable { #[pymethod(name = "lookup")] fn lookup(&self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name.value; - if self.symtable.symbols.contains_key(name) { - let role = self.symtable.symbols.get(name).unwrap().clone(); + if let Some(symbol) = self.symtable.symbols.get(name) { Ok(PySymbol { - name: name.to_string(), - role, + symbol: symbol.clone(), } .into_ref(vm)) } else { @@ -106,15 +104,8 @@ impl PySymbolTable { let symbols = self .symtable .symbols - .iter() - .map(|s| { - (PySymbol { - name: s.0.clone(), - role: s.1.clone(), - }) - .into_ref(vm) - .into_object() - }) + .values() + .map(|s| (PySymbol { symbol: s.clone() }).into_ref(vm).into_object()) .collect(); Ok(vm.ctx.new_list(symbols)) } @@ -137,8 +128,7 @@ impl PySymbolTable { #[pyclass(name = "Symbol")] struct PySymbol { - name: String, - role: symboltable::SymbolRole, + symbol: symboltable::Symbol, } impl fmt::Debug for PySymbol { @@ -157,55 +147,37 @@ impl PyValue for PySymbol { impl PySymbol { #[pymethod(name = "get_name")] fn get_name(&self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_str(self.name.clone())) + Ok(vm.ctx.new_str(self.symbol.name.clone())) } #[pymethod(name = "is_global")] fn is_global(&self, vm: &VirtualMachine) -> PyResult { - // TODO: figure out how this is determined. - let is_local = if let symboltable::SymbolRole::Used = self.role { - true - } else { - false - }; - Ok(vm.ctx.new_bool(is_local)) + Ok(vm.ctx.new_bool(self.symbol.is_global)) } #[pymethod(name = "is_local")] fn is_local(&self, vm: &VirtualMachine) -> PyResult { - // TODO: figure out how this is determined. - let is_local = if let symboltable::SymbolRole::Used = self.role { - true - } else { - false - }; - Ok(vm.ctx.new_bool(is_local)) + Ok(vm.ctx.new_bool(self.symbol.is_local)) } #[pymethod(name = "is_referenced")] fn is_referenced(&self, vm: &VirtualMachine) -> PyResult { - let is_local = if let symboltable::SymbolRole::Used = self.role { - true - } else { - false - }; - Ok(vm.ctx.new_bool(is_local)) + Ok(vm.ctx.new_bool(self.symbol.is_referenced)) } #[pymethod(name = "is_assigned")] fn is_assigned(&self, vm: &VirtualMachine) -> PyResult { - let is_local = if let symboltable::SymbolRole::Assigned = self.role { - true - } else { - false - }; - Ok(vm.ctx.new_bool(is_local)) + Ok(vm.ctx.new_bool(self.symbol.is_assigned)) } #[pymethod(name = "is_parameter")] fn is_parameter(&self, vm: &VirtualMachine) -> PyResult { - // TODO - Ok(vm.ctx.new_bool(false)) + Ok(vm.ctx.new_bool(self.symbol.is_parameter)) + } + + #[pymethod(name = "is_free")] + fn is_free(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bool(self.symbol.is_free)) } #[pymethod(name = "is_namespace")]