Merge pull request #1215 from RustPython/syntax-fixes

Add variable annotation syntax.

compiler/src/compile.rs

@@ -368,33 +368,7 @@ impl<O: OutputStream> Compiler<O> {
                 }
                 self.set_label(end_label);
             }
-            While { test, body, orelse } => {
-                let start_label = self.new_label();
-                let else_label = self.new_label();
-                let end_label = self.new_label();
-                self.emit(Instruction::SetupLoop {
-                    start: start_label,
-                    end: end_label,
-                });
-
-                self.set_label(start_label);
-
-                self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
-
-                let was_in_loop = self.in_loop;
-                self.in_loop = true;
-                self.compile_statements(body)?;
-                self.in_loop = was_in_loop;
-                self.emit(Instruction::Jump {
-                    target: start_label,
-                });
-                self.set_label(else_label);
-                self.emit(Instruction::PopBlock);
-                if let Some(orelse) = orelse {
-                    self.compile_statements(orelse)?;
-                }
-                self.set_label(end_label);
-            }
+            While { test, body, orelse } => self.compile_while(test, body, orelse)?,
             With {
                 is_async,
                 items,
@@ -563,6 +537,11 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_op(op, true);
                 self.compile_store(target)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => self.compile_annotated_assign(target, annotation, value)?,
             Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
@@ -1011,6 +990,40 @@ impl<O: OutputStream> Compiler<O> {
         }
     }
 
+    fn compile_while(
+        &mut self,
+        test: &ast::Expression,
+        body: &[ast::Statement],
+        orelse: &Option<Vec<ast::Statement>>,
+    ) -> Result<(), CompileError> {
+        let start_label = self.new_label();
+        let else_label = self.new_label();
+        let end_label = self.new_label();
+        self.emit(Instruction::SetupLoop {
+            start: start_label,
+            end: end_label,
+        });
+
+        self.set_label(start_label);
+
+        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+
+        let was_in_loop = self.in_loop;
+        self.in_loop = true;
+        self.compile_statements(body)?;
+        self.in_loop = was_in_loop;
+        self.emit(Instruction::Jump {
+            target: start_label,
+        });
+        self.set_label(else_label);
+        self.emit(Instruction::PopBlock);
+        if let Some(orelse) = orelse {
+            self.compile_statements(orelse)?;
+        }
+        self.set_label(end_label);
+        Ok(())
+    }
+
     fn compile_for(
         &mut self,
         target: &ast::Expression,
@@ -1129,6 +1142,39 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_annotated_assign(
+        &mut self,
+        target: &ast::Expression,
+        annotation: &ast::Expression,
+        value: &Option<ast::Expression>,
+    ) -> Result<(), CompileError> {
+        if let Some(value) = value {
+            self.compile_expression(value)?;
+            self.compile_store(target)?;
+        }
+
+        // Compile annotation:
+        self.compile_expression(annotation)?;
+
+        if let ast::ExpressionType::Identifier { name } = &target.node {
+            // Store as dict entry in __annotations__ dict:
+            self.emit(Instruction::LoadName {
+                name: String::from("__annotations__"),
+                scope: bytecode::NameScope::Local,
+            });
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String {
+                    value: name.to_string(),
+                },
+            });
+            self.emit(Instruction::StoreSubscript);
+        } else {
+            // Drop annotation if not assigned to simple identifier.
+            self.emit(Instruction::Pop);
+        }
+        Ok(())
+    }
+
     fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
         match &target.node {
             ast::ExpressionType::Identifier { name } => {
@@ -1272,6 +1318,8 @@ impl<O: OutputStream> Compiler<O> {
 
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
+        self.set_source_location(&expression.location);
+
         use ast::ExpressionType::*;
         match &expression.node {
             Call {
@@ -1664,6 +1712,10 @@ impl<O: OutputStream> Compiler<O> {
 
         let mut loop_labels = vec![];
         for generator in generators {
+            if generator.is_async {
+                unimplemented!("async for comprehensions");
+            }
+
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadName {

compiler/src/symboltable.rs

@@ -335,6 +335,17 @@ impl SymbolTableBuilder {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => {
+                self.scan_expression(target)?;
+                self.scan_expression(annotation)?;
+                if let Some(value) = value {
+                    self.scan_expression(value)?;
+                }
+            }
             With { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;

examples/parse_folder.rs

@@ -14,7 +14,8 @@ extern crate log;
 use clap::{App, Arg};
 
 use rustpython_parser::{ast, parser};
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
 
 fn main() {
     env_logger::init();
@@ -32,14 +33,21 @@ fn main() {
     let folder = Path::new(matches.value_of("folder").unwrap());
     if folder.exists() && folder.is_dir() {
         println!("Parsing folder of python code: {:?}", folder);
-        let res = parse_folder(&folder).unwrap();
-        println!("Processed {:?} files", res.len());
+        let t1 = Instant::now();
+        let parsed_files = parse_folder(&folder).unwrap();
+        let t2 = Instant::now();
+        let results = ScanResult {
+            t1,
+            t2,
+            parsed_files,
+        };
+        statistics(results);
     } else {
         println!("{:?} is not a folder.", folder);
     }
 }
 
-fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
+fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> {
     let mut res = vec![];
     info!("Parsing folder of python code: {:?}", path);
     for entry in path.read_dir()? {
@@ -49,22 +57,66 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
 
         let path = entry.path();
         if metadata.is_dir() {
-            let x = parse_folder(&path)?;
-            res.extend(x);
+            res.extend(parse_folder(&path)?);
         }
 
         if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
-            match parse_python_file(&path) {
-                Ok(x) => res.push(x),
+            let result = parse_python_file(&path);
+            match &result {
+                Ok(_) => {}
                 Err(y) => error!("Erreur in file {:?} {:?}", path, y),
             }
+            res.push(ParsedFile {
+                filename: Box::new(path),
+                result,
+            });
         }
     }
     Ok(res)
 }
 
-fn parse_python_file(filename: &Path) -> Result<ast::Program, String> {
+fn parse_python_file(filename: &Path) -> ParseResult {
     info!("Parsing file {:?}", filename);
     let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
     parser::parse_program(&source).map_err(|e| e.to_string())
 }
+
+fn statistics(results: ScanResult) {
+    // println!("Processed {:?} files", res.len());
+    println!("Scanned a total of {} files", results.parsed_files.len());
+    let total = results.parsed_files.len();
+    let failed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_err())
+        .count();
+    let passed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_ok())
+        .count();
+    println!("Passed: {} Failed: {} Total: {}", passed, failed, total);
+    println!(
+        "That is {} % success rate.",
+        (passed as f64 * 100.0) / total as f64
+    );
+    let duration = results.t2 - results.t1;
+    println!("Total time spend: {:?}", duration);
+    println!(
+        "File processing rate: {} files/second",
+        (total * 1_000_000) as f64 / duration.as_micros() as f64
+    );
+}
+
+struct ScanResult {
+    t1: Instant,
+    t2: Instant,
+    parsed_files: Vec<ParsedFile>,
+}
+
+struct ParsedFile {
+    filename: Box<PathBuf>,
+    result: ParseResult,
+}
+
+type ParseResult = Result<ast::Program, String>;

parser/src/ast.rs

@@ -74,6 +74,11 @@ pub enum StatementType {
         op: Operator,
         value: Box<Expression>,
     },
+    AnnAssign {
+        target: Box<Expression>,
+        annotation: Box<Expression>,
+        value: Option<Expression>,
+    },
     Expression {
         expression: Expression,
     },
@@ -313,6 +318,7 @@ pub struct Comprehension {
     pub target: Expression,
     pub iter: Expression,
     pub ifs: Vec<Expression>,
+    pub is_async: bool,
 }
 
 #[derive(Debug, PartialEq)]

parser/src/lexer.rs

@@ -396,7 +396,7 @@ where
 
             let value = f64::from_str(&value_text).unwrap();
             // Parse trailing 'j':
-            if self.chr0 == Some('j') {
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
                 self.next_char();
                 let end_pos = self.get_pos();
                 Ok((
@@ -413,7 +413,7 @@ where
             }
         } else {
             // Parse trailing 'j':
-            if self.chr0 == Some('j') {
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
                 self.next_char();
                 let end_pos = self.get_pos();
                 let imag = f64::from_str(&value_text).unwrap();

parser/src/parser.rs

@@ -376,7 +376,7 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Expression {
-                location: ast::Location::new(1, 2),
+                location: ast::Location::new(1, 1),
                 node: ast::ExpressionType::Comprehension {
                     kind: Box::new(ast::ComprehensionKind::List {
                         element: mk_ident("x", 1, 2),
@@ -386,6 +386,7 @@ mod tests {
                         target: mk_ident("y", 1, 8),
                         iter: mk_ident("z", 1, 13),
                         ifs: vec![],
+                        is_async: false,
                     }],
                 }
             }
@@ -399,7 +400,7 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Expression {
-                location: ast::Location::new(1, 2),
+                location: ast::Location::new(1, 1),
                 node: ast::ExpressionType::Comprehension {
                     kind: Box::new(ast::ComprehensionKind::List {
                         element: mk_ident("x", 1, 2)
@@ -415,6 +416,7 @@ mod tests {
                             },
                             iter: mk_ident("z", 1, 17),
                             ifs: vec![],
+                            is_async: false,
                         },
                         ast::Comprehension {
                             location: ast::Location::new(1, 19),
@@ -436,6 +438,7 @@ mod tests {
                                     },
                                 },
                             ],
+                            is_async: false,
                         }
                     ],
                 }

parser/src/python.lalrpop

@@ -107,16 +107,26 @@ ExpressionStatement: ast::Statement = {
             }
         }
     },
-    <location:@L> <expr:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
+    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestList> => {
         ast::Statement {
             location,
             node: ast::StatementType::AugAssign {
-                target: Box::new(expr),
+                target: Box::new(target),
                 op,
                 value: Box::new(rhs)
             },
         }
     },
+    <location:@L> <target:Test> ":" <annotation:Test> <rhs:("=" Test)?> => {
+        ast::Statement {
+            location,
+            node: ast::StatementType::AnnAssign {
+                target: Box::new(target),
+                annotation: Box::new(annotation),
+                value: rhs.map(|e| e.1)
+            },
+        }
+    },
 };
 
 AssignSuffix: ast::Expression = {
@@ -877,77 +887,78 @@ Atom: ast::Expression = {
         location,
         node: ast::ExpressionType::Identifier { name }
     },
-    <location:@L> "[" <e:TestListComp?> "]" => {
+    <location:@L> "[" <e:ListLiteralValues?> "]" => {
         let elements = e.unwrap_or_default();
         ast::Expression {
             location,
             node: ast::ExpressionType::List { elements }
         }
     },
-    "[" <e:TestListComp2> "]" => e,
+    <location:@L> "[" <element:TestOrStarExpr> <generators:CompFor> "]" => {
+        ast::Expression {
+            location,
+            node: ast::ExpressionType::Comprehension {
+                kind: Box::new(ast::ComprehensionKind::List { element }),
+                generators,
+            }
+        }
+    },
     <location:@L> "(" <elements:TestOrStarExprList?> ")" => {
         elements.unwrap_or(ast::Expression {
              location,
              node: ast::ExpressionType::Tuple { elements: Vec::new() }
         })
     },
-    <location:@L> "(" <element:Test> <c:CompFor> ")" => {
+    "(" <e:YieldExpr> ")" => e,
+    <location:@L> "(" <element:Test> <generators:CompFor> ")" => {
         ast::Expression {
             location,
             node: ast::ExpressionType::Comprehension {
                 kind: Box::new(ast::ComprehensionKind::GeneratorExpression { element }),
-                generators: c,
+                generators,
             }
         }
     },
-    <location:@L> "{" <e:TestDict?> "}" => ast::Expression {
+    <location:@L> "{" <e:DictLiteralValues?> "}" => ast::Expression {
         location,
         node: ast::ExpressionType::Dict { elements: e.unwrap_or_default() }
     },
-    "{" <e:TestDictComp> "}" => e,
-    <location:@L> "{" <elements:TestSet> "}" => ast::Expression {
+    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" => {
+        ast::Expression {
+            location,
+            node: ast::ExpressionType::Comprehension {
+                kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }),
+                generators,
+            }
+        }
+    },
+    <location:@L> "{" <elements:SetLiteralValues> "}" => ast::Expression {
         location,
         node: ast::ExpressionType::Set { elements }
     },
-    "{" <e:TestSetComp> "}" => e,
+    <location:@L> "{" <element:Test> <generators:CompFor> "}" => {
+        ast::Expression {
+            location,
+            node: ast::ExpressionType::Comprehension {
+                kind: Box::new(ast::ComprehensionKind::Set { element }),
+                generators,
+            }
+        }
+    },
     <location:@L> "True" => ast::Expression { location, node: ast::ExpressionType::True },
     <location:@L> "False" => ast::Expression { location, node: ast::ExpressionType::False },
     <location:@L> "None" => ast::Expression { location, node: ast::ExpressionType::None },
     <location:@L> "..." => ast::Expression { location, node: ast::ExpressionType::Ellipsis },
 };
 
-TestListComp: Vec<ast::Expression> = {
+ListLiteralValues: Vec<ast::Expression> = {
     <e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => e,
 };
 
-TestListComp2: ast::Expression = {
-    <location:@L> <e:TestOrStarExpr> <c:CompFor> => {
-        ast::Expression {
-            location,
-            node: ast::ExpressionType::Comprehension {
-                kind: Box::new(ast::ComprehensionKind::List { element: e }),
-                generators: c,
-            }
-        }
-    },
-};
-
-TestDict: Vec<(Option<ast::Expression>, ast::Expression)> = {
+DictLiteralValues: Vec<(Option<ast::Expression>, ast::Expression)> = {
     <elements:OneOrMore<DictElement>> <_trailing_comma:","?> => elements,
 };
 
-TestDictComp: ast::Expression = {
-    <location:@L> <e1:DictEntry> <c:CompFor> => {
-        ast::Expression {
-            location,
-            node: ast::ExpressionType::Comprehension {
-                kind: Box::new(ast::ComprehensionKind::Dict { key: e1.0, value: e1.1 }),
-                generators: c,
-            }
-        }
-    }
-};
-
 DictEntry: (ast::Expression, ast::Expression) = {
     <e1: Test> ":" <e2: Test> => (e1, e2),
 };
@@ -957,22 +968,10 @@ DictElement: (Option<ast::Expression>, ast::Expression) = {
     "**" <e:Expression> => (None, e),
 };
 
-TestSet: Vec<ast::Expression> = {
+SetLiteralValues: Vec<ast::Expression> = {
     <e1:OneOrMore<TestOrStarExpr>> ","? => e1
 };
 
-TestSetComp: ast::Expression = {
-    <location:@L> <e1:Test> <c:CompFor> => {
-        ast::Expression {
-            location,
-            node: ast::ExpressionType::Comprehension {
-                kind: Box::new(ast::ComprehensionKind::Set { element: e1 }),
-                generators: c,
-            }
-        }
-    }
-};
-
 ExpressionOrStarExpression = {
     Expression,
     StarExpr
@@ -1024,8 +1023,9 @@ StarExpr: ast::Expression = {
 CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
 
 SingleForComprehension: ast::Comprehension = {
-    <location:@L> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
-        ast::Comprehension { location, target, iter, ifs }
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
+        let is_async = is_async.is_some();
+        ast::Comprehension { location, target, iter, ifs, is_async }
     }
 };
 

tests/snippets/assignment.py

@@ -51,3 +51,12 @@ def g():
 a, b = g()
 assert a == 1337
 assert b == 42
+
+# Variable annotations:
+a: bool
+b: bool = False
+
+assert a == 1337
+assert b == False
+
+assert __annotations__['a'] == bool

tests/snippets/generators.py

@@ -65,3 +65,26 @@ assert next(g) == 1
 
 with assertRaises(KeyError):
     assert g.throw(KeyError, KeyError(), None) == 2
+
+
+r = []
+def p(a, b, c):
+    # print(a, b, c)
+    r.append(a)
+    r.append(b)
+    r.append(c)
+
+
+def g5():
+    p('a', (yield 2), (yield 5))
+    yield 99
+
+g = g5()
+g.send(None)
+g.send(66)
+# g.send(88)
+l = list(g)
+# print(r)
+# print(l)
+assert l == [99]
+assert r == ['a', 66, None]

vm/src/scope.rs

@@ -72,12 +72,14 @@ impl fmt::Debug for Scope {
 }
 
 impl Scope {
-    pub fn new(locals: Option<PyDictRef>, globals: PyDictRef) -> Scope {
+    pub fn new(locals: Option<PyDictRef>, globals: PyDictRef, vm: &VirtualMachine) -> Scope {
         let locals = match locals {
             Some(dict) => RcList::new().insert(dict),
             None => RcList::new(),
         };
-        Scope { locals, globals }
+        let scope = Scope { locals, globals };
+        scope.store_name(vm, "__annotations__", vm.ctx.new_dict().into_object());
+        scope
     }
 
     pub fn with_builtins(
@@ -91,7 +93,7 @@ impl Scope {
                 .set_item("__builtins__", vm.builtins.clone(), vm)
                 .unwrap();
         }
-        Scope::new(locals, globals)
+        Scope::new(locals, globals, vm)
     }
 
     pub fn get_locals(&self) -> PyDictRef {

vm/src/stdlib/ast.rs

@@ -216,6 +216,15 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::Statement) -> PyResult
             op => vm.ctx.new_str(operator_string(op)),
             value => expression_to_ast(vm, value)?,
         }),
+        AnnAssign {
+            target,
+            annotation,
+            value,
+        } => node!(vm, AnnAssign, {
+            target => expression_to_ast(vm, target)?,
+            annotation => expression_to_ast(vm, annotation)?,
+            value => optional_expression_to_ast(vm, value)?,
+        }),
         Raise { exception, cause } => node!(vm, Raise, {
             exc => optional_expression_to_ast(vm, exception)?,
             cause => optional_expression_to_ast(vm, cause)?,
@@ -570,6 +579,7 @@ fn comprehension_to_ast(
         target => expression_to_ast(vm, &comprehension.target)?,
         iter => expression_to_ast(vm, &comprehension.iter)?,
         ifs => expressions_to_ast(vm, &comprehension.ifs)?,
+        is_async => vm.new_bool(comprehension.is_async),
     }))
 }
 
@@ -607,6 +617,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
         "alias" => py_class!(ctx, "alias", ast_base.clone(), {}),
         "arg" => py_class!(ctx, "arg", ast_base.clone(), {}),
         "arguments" => py_class!(ctx, "arguments", ast_base.clone(), {}),
+        "AnnAssign" => py_class!(ctx, "AnnAssign", ast_base.clone(), {}),
         "Assign" => py_class!(ctx, "Assign", ast_base.clone(), {}),
         "AugAssign" => py_class!(ctx, "AugAssign", ast_base.clone(), {}),
         "AsyncFor" => py_class!(ctx, "AsyncFor", ast_base.clone(), {}),