From ded9d5ba5b5480b3eb455eb6ba7104cc10a11948 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 7 Aug 2019 15:31:24 +0200 Subject: [PATCH 1/3] Add variable annotation syntax. Simplify grammar for comprehensions. Add capital J suffix for complex numbers. --- compiler/src/compile.rs | 14 ++++++ compiler/src/symboltable.rs | 11 +++++ parser/src/ast.rs | 5 ++ parser/src/lexer.rs | 4 +- parser/src/parser.rs | 4 +- parser/src/python.lalrpop | 96 ++++++++++++++++++------------------ tests/snippets/assignment.py | 7 +++ vm/src/stdlib/ast.rs | 10 ++++ 8 files changed, 98 insertions(+), 53 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 3fa6bdbe8..8d5c4c47d 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -553,6 +553,20 @@ impl Compiler { self.compile_op(op, true); self.compile_store(target)?; } + AnnAssign { + target, + annotation, + value, + } => { + // Throw away annotation: + self.compile_expression(annotation)?; + self.emit(Instruction::Pop); + + if let Some(value) = value { + self.compile_expression(value)?; + self.compile_store(target)?; + } + } Delete { targets } => { for target in targets { self.compile_delete(target)?; diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index b12e0f92c..c1db24bf3 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -335,6 +335,17 @@ impl SymbolTableBuilder { self.scan_expression(target)?; self.scan_expression(value)?; } + AnnAssign { + target, + annotation, + value, + } => { + self.scan_expression(target)?; + self.scan_expression(annotation)?; + if let Some(value) = value { + self.scan_expression(value)?; + } + } With { items, body, .. } => { for item in items { self.scan_expression(&item.context_expr)?; diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 56fc51ed8..dcded9fc9 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -74,6 +74,11 @@ pub enum StatementType { op: Operator, value: Box, }, + AnnAssign { + target: Box, + annotation: Box, + value: Option, + }, Expression { expression: Expression, }, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 49cc93543..ae83a1d8d 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -396,7 +396,7 @@ where let value = f64::from_str(&value_text).unwrap(); // Parse trailing 'j': - if self.chr0 == Some('j') { + if self.chr0 == Some('j') || self.chr0 == Some('J') { self.next_char(); let end_pos = self.get_pos(); Ok(( @@ -413,7 +413,7 @@ where } } else { // Parse trailing 'j': - if self.chr0 == Some('j') { + if self.chr0 == Some('j') || self.chr0 == Some('J') { self.next_char(); let end_pos = self.get_pos(); let imag = f64::from_str(&value_text).unwrap(); diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 11dd61803..e5287f8d8 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -376,7 +376,7 @@ mod tests { assert_eq!( parse_ast, ast::Expression { - location: ast::Location::new(1, 2), + location: ast::Location::new(1, 1), node: ast::ExpressionType::Comprehension { kind: Box::new(ast::ComprehensionKind::List { element: mk_ident("x", 1, 2), @@ -399,7 +399,7 @@ mod tests { assert_eq!( parse_ast, ast::Expression { - location: ast::Location::new(1, 2), + location: ast::Location::new(1, 1), node: ast::ExpressionType::Comprehension { kind: Box::new(ast::ComprehensionKind::List { element: mk_ident("x", 1, 2) diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index db1a33312..20b4a4168 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -107,16 +107,26 @@ ExpressionStatement: ast::Statement = { } } }, - => { + => { ast::Statement { location, node: ast::StatementType::AugAssign { - target: Box::new(expr), + target: Box::new(target), op, value: Box::new(rhs) }, } }, + ":" => { + ast::Statement { + location, + node: ast::StatementType::AnnAssign { + target: Box::new(target), + annotation: Box::new(annotation), + value: rhs.map(|e| e.1) + }, + } + }, }; AssignSuffix: ast::Expression = { @@ -876,77 +886,77 @@ Atom: ast::Expression = { location, node: ast::ExpressionType::Identifier { name } }, - "[" "]" => { + "[" "]" => { let elements = e.unwrap_or_default(); ast::Expression { location, node: ast::ExpressionType::List { elements } } }, - "[" "]" => e, + "[" "]" => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::List { element }), + generators, + } + } + }, "(" ")" => { elements.unwrap_or(ast::Expression { location, node: ast::ExpressionType::Tuple { elements: Vec::new() } }) }, - "(" ")" => { + "(" ")" => { ast::Expression { location, node: ast::ExpressionType::Comprehension { kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element }), - generators: c, + generators, } } }, - "{" "}" => ast::Expression { + "{" "}" => ast::Expression { location, node: ast::ExpressionType::Dict { elements: e.unwrap_or_default() } }, - "{" "}" => e, - "{" "}" => ast::Expression { + "{" "}" => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }), + generators, + } + } + }, + "{" "}" => ast::Expression { location, node: ast::ExpressionType::Set { elements } }, - "{" "}" => e, + "{" "}" => { + ast::Expression { + location, + node: ast::ExpressionType::Comprehension { + kind: Box::new(ast::ComprehensionKind::Set { element }), + generators, + } + } + }, "True" => ast::Expression { location, node: ast::ExpressionType::True }, "False" => ast::Expression { location, node: ast::ExpressionType::False }, "None" => ast::Expression { location, node: ast::ExpressionType::None }, "..." => ast::Expression { location, node: ast::ExpressionType::Ellipsis }, }; -TestListComp: Vec = { +ListLiteralValues: Vec = { > <_trailing_comma:","?> => e, }; -TestListComp2: ast::Expression = { - => { - ast::Expression { - location, - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::List { element: e }), - generators: c, - } - } - }, -}; - -TestDict: Vec<(Option, ast::Expression)> = { +DictLiteralValues: Vec<(Option, ast::Expression)> = { > <_trailing_comma:","?> => elements, }; -TestDictComp: ast::Expression = { - => { - ast::Expression { - location, - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }), - generators: c, - } - } - } -}; - DictEntry: (ast::Expression, ast::Expression) = { ":" => (e1, e2), }; @@ -956,22 +966,10 @@ DictElement: (Option, ast::Expression) = { "**" => (None, e), }; -TestSet: Vec = { +SetLiteralValues: Vec = { > ","? => e1 }; -TestSetComp: ast::Expression = { - => { - ast::Expression { - location, - node: ast::ExpressionType::Comprehension { - kind: Box::new(ast::ComprehensionKind::Set { element: e1 }), - generators: c, - } - } - } -}; - ExpressionOrStarExpression = { Expression, StarExpr diff --git a/tests/snippets/assignment.py b/tests/snippets/assignment.py index 01e487e0d..e32c4fcbf 100644 --- a/tests/snippets/assignment.py +++ b/tests/snippets/assignment.py @@ -51,3 +51,10 @@ def g(): a, b = g() assert a == 1337 assert b == 42 + +# Variable annotations: +a: bool +b: bool = False + +assert a == 1337 +assert b == False diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index ac41ca0fa..138b11c6b 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -216,6 +216,15 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult op => vm.ctx.new_str(operator_string(op)), value => expression_to_ast(vm, value)?, }), + AnnAssign { + target, + annotation, + value, + } => node!(vm, AnnAssign, { + target => expression_to_ast(vm, target)?, + annotation => expression_to_ast(vm, annotation)?, + value => optional_expression_to_ast(vm, value)?, + }), Raise { exception, cause } => node!(vm, Raise, { exc => optional_expression_to_ast(vm, exception)?, cause => optional_expression_to_ast(vm, cause)?, @@ -607,6 +616,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "alias" => py_class!(ctx, "alias", ast_base.clone(), {}), "arg" => py_class!(ctx, "arg", ast_base.clone(), {}), "arguments" => py_class!(ctx, "arguments", ast_base.clone(), {}), + "AnnAssign" => py_class!(ctx, "AnnAssign", ast_base.clone(), {}), "Assign" => py_class!(ctx, "Assign", ast_base.clone(), {}), "AugAssign" => py_class!(ctx, "AugAssign", ast_base.clone(), {}), "AsyncFor" => py_class!(ctx, "AsyncFor", ast_base.clone(), {}), From c5e7fbab11d706a9a7d5bbaa3541be8e3acf4dfd Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 7 Aug 2019 16:13:12 +0200 Subject: [PATCH 2/3] Properly fill the __annotations__ dictionary. --- compiler/src/compile.rs | 108 +++++++++++++++++++++++------------ tests/snippets/assignment.py | 2 + vm/src/scope.rs | 8 ++- 3 files changed, 78 insertions(+), 40 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 8d5c4c47d..f7ac924bb 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -358,33 +358,7 @@ impl Compiler { } self.set_label(end_label); } - While { test, body, orelse } => { - let start_label = self.new_label(); - let else_label = self.new_label(); - let end_label = self.new_label(); - self.emit(Instruction::SetupLoop { - start: start_label, - end: end_label, - }); - - self.set_label(start_label); - - self.compile_test(test, None, Some(else_label), EvalContext::Statement)?; - - let was_in_loop = self.in_loop; - self.in_loop = true; - self.compile_statements(body)?; - self.in_loop = was_in_loop; - self.emit(Instruction::Jump { - target: start_label, - }); - self.set_label(else_label); - self.emit(Instruction::PopBlock); - if let Some(orelse) = orelse { - self.compile_statements(orelse)?; - } - self.set_label(end_label); - } + While { test, body, orelse } => self.compile_while(test, body, orelse)?, With { is_async, items, @@ -557,16 +531,7 @@ impl Compiler { target, annotation, value, - } => { - // Throw away annotation: - self.compile_expression(annotation)?; - self.emit(Instruction::Pop); - - if let Some(value) = value { - self.compile_expression(value)?; - self.compile_store(target)?; - } - } + } => self.compile_annotated_assign(target, annotation, value)?, Delete { targets } => { for target in targets { self.compile_delete(target)?; @@ -1015,6 +980,40 @@ impl Compiler { } } + fn compile_while( + &mut self, + test: &ast::Expression, + body: &[ast::Statement], + orelse: &Option>, + ) -> Result<(), CompileError> { + let start_label = self.new_label(); + let else_label = self.new_label(); + let end_label = self.new_label(); + self.emit(Instruction::SetupLoop { + start: start_label, + end: end_label, + }); + + self.set_label(start_label); + + self.compile_test(test, None, Some(else_label), EvalContext::Statement)?; + + let was_in_loop = self.in_loop; + self.in_loop = true; + self.compile_statements(body)?; + self.in_loop = was_in_loop; + self.emit(Instruction::Jump { + target: start_label, + }); + self.set_label(else_label); + self.emit(Instruction::PopBlock); + if let Some(orelse) = orelse { + self.compile_statements(orelse)?; + } + self.set_label(end_label); + Ok(()) + } + fn compile_for( &mut self, target: &ast::Expression, @@ -1133,6 +1132,39 @@ impl Compiler { Ok(()) } + fn compile_annotated_assign( + &mut self, + target: &ast::Expression, + annotation: &ast::Expression, + value: &Option, + ) -> Result<(), CompileError> { + if let Some(value) = value { + self.compile_expression(value)?; + self.compile_store(target)?; + } + + // Compile annotation: + self.compile_expression(annotation)?; + + if let ast::ExpressionType::Identifier { name } = &target.node { + // Store as dict entry in __annotations__ dict: + self.emit(Instruction::LoadName { + name: String::from("__annotations__"), + scope: bytecode::NameScope::Local, + }); + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: name.to_string(), + }, + }); + self.emit(Instruction::StoreSubscript); + } else { + // Drop annotation if not assigned to simple identifier. + self.emit(Instruction::Pop); + } + Ok(()) + } + fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match &target.node { ast::ExpressionType::Identifier { name } => { @@ -1276,6 +1308,8 @@ impl Compiler { fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> { trace!("Compiling {:?}", expression); + self.set_source_location(&expression.location); + use ast::ExpressionType::*; match &expression.node { Call { diff --git a/tests/snippets/assignment.py b/tests/snippets/assignment.py index e32c4fcbf..9613bcd3e 100644 --- a/tests/snippets/assignment.py +++ b/tests/snippets/assignment.py @@ -58,3 +58,5 @@ b: bool = False assert a == 1337 assert b == False + +assert __annotations__['a'] == bool diff --git a/vm/src/scope.rs b/vm/src/scope.rs index 7bc34dd0c..9e5baadb7 100644 --- a/vm/src/scope.rs +++ b/vm/src/scope.rs @@ -72,12 +72,14 @@ impl fmt::Debug for Scope { } impl Scope { - pub fn new(locals: Option, globals: PyDictRef) -> Scope { + pub fn new(locals: Option, globals: PyDictRef, vm: &VirtualMachine) -> Scope { let locals = match locals { Some(dict) => RcList::new().insert(dict), None => RcList::new(), }; - Scope { locals, globals } + let scope = Scope { locals, globals }; + scope.store_name(vm, "__annotations__", vm.ctx.new_dict().into_object()); + scope } pub fn with_builtins( @@ -91,7 +93,7 @@ impl Scope { .set_item("__builtins__", vm.builtins.clone(), vm) .unwrap(); } - Scope::new(locals, globals) + Scope::new(locals, globals, vm) } pub fn get_locals(&self) -> PyDictRef { From 2fdfcb96a55bbd6fc428ccde5e8b04c235889774 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 7 Aug 2019 20:11:27 +0200 Subject: [PATCH 3/3] Add parser support for async for loop. Add yield expression inside parentheses. --- compiler/src/compile.rs | 4 +++ examples/parse_folder.rs | 70 +++++++++++++++++++++++++++++++----- parser/src/ast.rs | 1 + parser/src/parser.rs | 3 ++ parser/src/python.lalrpop | 6 ++-- tests/snippets/generators.py | 23 ++++++++++++ vm/src/stdlib/ast.rs | 1 + 7 files changed, 97 insertions(+), 11 deletions(-) diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index f7ac924bb..c3870a3a9 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1702,6 +1702,10 @@ impl Compiler { let mut loop_labels = vec![]; for generator in generators { + if generator.is_async { + unimplemented!("async for comprehensions"); + } + if loop_labels.is_empty() { // Load iterator onto stack (passed as first argument): self.emit(Instruction::LoadName { diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 9bd3d05ae..513333b95 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -14,7 +14,8 @@ extern crate log; use clap::{App, Arg}; use rustpython_parser::{ast, parser}; -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::time::Instant; fn main() { env_logger::init(); @@ -32,14 +33,21 @@ fn main() { let folder = Path::new(matches.value_of("folder").unwrap()); if folder.exists() && folder.is_dir() { println!("Parsing folder of python code: {:?}", folder); - let res = parse_folder(&folder).unwrap(); - println!("Processed {:?} files", res.len()); + let t1 = Instant::now(); + let parsed_files = parse_folder(&folder).unwrap(); + let t2 = Instant::now(); + let results = ScanResult { + t1, + t2, + parsed_files, + }; + statistics(results); } else { println!("{:?} is not a folder.", folder); } } -fn parse_folder(path: &Path) -> std::io::Result> { +fn parse_folder(path: &Path) -> std::io::Result> { let mut res = vec![]; info!("Parsing folder of python code: {:?}", path); for entry in path.read_dir()? { @@ -49,22 +57,66 @@ fn parse_folder(path: &Path) -> std::io::Result> { let path = entry.path(); if metadata.is_dir() { - let x = parse_folder(&path)?; - res.extend(x); + res.extend(parse_folder(&path)?); } if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") { - match parse_python_file(&path) { - Ok(x) => res.push(x), + let result = parse_python_file(&path); + match &result { + Ok(_) => {} Err(y) => error!("Erreur in file {:?} {:?}", path, y), } + res.push(ParsedFile { + filename: Box::new(path), + result, + }); } } Ok(res) } -fn parse_python_file(filename: &Path) -> Result { +fn parse_python_file(filename: &Path) -> ParseResult { info!("Parsing file {:?}", filename); let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?; parser::parse_program(&source).map_err(|e| e.to_string()) } + +fn statistics(results: ScanResult) { + // println!("Processed {:?} files", res.len()); + println!("Scanned a total of {} files", results.parsed_files.len()); + let total = results.parsed_files.len(); + let failed = results + .parsed_files + .iter() + .filter(|p| p.result.is_err()) + .count(); + let passed = results + .parsed_files + .iter() + .filter(|p| p.result.is_ok()) + .count(); + println!("Passed: {} Failed: {} Total: {}", passed, failed, total); + println!( + "That is {} % success rate.", + (passed as f64 * 100.0) / total as f64 + ); + let duration = results.t2 - results.t1; + println!("Total time spend: {:?}", duration); + println!( + "File processing rate: {} files/second", + (total * 1_000_000) as f64 / duration.as_micros() as f64 + ); +} + +struct ScanResult { + t1: Instant, + t2: Instant, + parsed_files: Vec, +} + +struct ParsedFile { + filename: Box, + result: ParseResult, +} + +type ParseResult = Result; diff --git a/parser/src/ast.rs b/parser/src/ast.rs index dcded9fc9..b11e0f9c6 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -318,6 +318,7 @@ pub struct Comprehension { pub target: Expression, pub iter: Expression, pub ifs: Vec, + pub is_async: bool, } #[derive(Debug, PartialEq)] diff --git a/parser/src/parser.rs b/parser/src/parser.rs index e5287f8d8..d27fce025 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -386,6 +386,7 @@ mod tests { target: mk_ident("y", 1, 8), iter: mk_ident("z", 1, 13), ifs: vec![], + is_async: false, }], } } @@ -415,6 +416,7 @@ mod tests { }, iter: mk_ident("z", 1, 17), ifs: vec![], + is_async: false, }, ast::Comprehension { location: ast::Location::new(1, 19), @@ -436,6 +438,7 @@ mod tests { }, }, ], + is_async: false, } ], } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 20b4a4168..30831c9fc 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -908,6 +908,7 @@ Atom: ast::Expression = { node: ast::ExpressionType::Tuple { elements: Vec::new() } }) }, + "(" ")" => e, "(" ")" => { ast::Expression { location, @@ -1021,8 +1022,9 @@ StarExpr: ast::Expression = { CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { - ast::Comprehension { location, target, iter, ifs } + "for" "in" => { + let is_async = is_async.is_some(); + ast::Comprehension { location, target, iter, ifs, is_async } } }; diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 3b7dc77c3..ddab1d68d 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -64,3 +64,26 @@ assert next(g) == 1 with assertRaises(KeyError): assert g.throw(KeyError, KeyError(), None) == 2 + + +r = [] +def p(a, b, c): + # print(a, b, c) + r.append(a) + r.append(b) + r.append(c) + + +def g5(): + p('a', (yield 2), (yield 5)) + yield 99 + +g = g5() +g.send(None) +g.send(66) +# g.send(88) +l = list(g) +# print(r) +# print(l) +assert l == [99] +assert r == ['a', 66, None] diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 138b11c6b..9543d5ce1 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -579,6 +579,7 @@ fn comprehension_to_ast( target => expression_to_ast(vm, &comprehension.target)?, iter => expression_to_ast(vm, &comprehension.iter)?, ifs => expressions_to_ast(vm, &comprehension.ifs)?, + is_async => vm.new_bool(comprehension.is_async), })) }