Merge pull request #904 from RustPython/async-stuff

Add async def and async for parsing.

parser/src/ast.rs

@@ -98,6 +98,12 @@ pub enum Statement {
         body: Vec<LocatedStatement>,
         orelse: Option<Vec<LocatedStatement>>,
     },
+    AsyncFor {
+        target: Expression,
+        iter: Expression,
+        body: Vec<LocatedStatement>,
+        orelse: Option<Vec<LocatedStatement>>,
+    },
     Raise {
         exception: Option<Expression>,
         cause: Option<Expression>,
@@ -122,6 +128,13 @@ pub enum Statement {
         decorator_list: Vec<Expression>,
         returns: Option<Expression>,
     },
+    AsyncFunctionDef {
+        name: String,
+        args: Parameters,
+        body: Vec<LocatedStatement>,
+        decorator_list: Vec<Expression>,
+        returns: Option<Expression>,
+    },
 }
 
 #[derive(Debug, PartialEq)]
@@ -150,6 +163,9 @@ pub enum Expression {
         op: UnaryOperator,
         a: Box<Expression>,
     },
+    Await {
+        value: Box<Expression>,
+    },
     Yield {
         value: Option<Box<Expression>>,
     },
@@ -227,6 +243,7 @@ impl Expression {
         match self {
             BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
             Subscript { .. } => "subscript",
+            Await { .. } => "await expression",
             Yield { .. } | YieldFrom { .. } => "yield expression",
             Compare { .. } => "comparison",
             Attribute { .. } => "attribute",

parser/src/lexer.rs

@@ -103,6 +103,8 @@ pub fn get_keywords() -> HashMap<String, Tok> {
     keywords.insert(String::from("and"), Tok::And);
     keywords.insert(String::from("as"), Tok::As);
     keywords.insert(String::from("assert"), Tok::Assert);
+    keywords.insert(String::from("async"), Tok::Async);
+    keywords.insert(String::from("await"), Tok::Await);
     keywords.insert(String::from("break"), Tok::Break);
     keywords.insert(String::from("class"), Tok::Class);
     keywords.insert(String::from("continue"), Tok::Continue);

parser/src/python.lalrpop

@@ -348,13 +348,14 @@ WhileStatement: ast::LocatedStatement = {
 };
 
 ForStatement: ast::LocatedStatement = {
-    <loc:@L> "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
-        let or_else = s2.map(|s| s.2);
+    <loc:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let orelse = s2.map(|s| s.2);
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::For {
-                target: e,
-                iter: t, body: s, orelse: or_else
+            node: if is_async.is_some() {
+                ast::Statement::AsyncFor { target, iter, body, orelse }
+            } else {
+                ast::Statement::For { target, iter, body, orelse }
             },
         }
     },
@@ -410,16 +411,26 @@ WithItem: ast::WithItem = {
 };
 
 FuncDef: ast::LocatedStatement = {
-    <d:Decorator*> <loc:@L> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
+    <d:Decorator*> <loc:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
         ast::LocatedStatement {
             location: loc,
-            node: ast::Statement::FunctionDef {
-                name: i,
-                args: a,
-                body: s,
-                decorator_list: d,
-                returns: r.map(|x| x.1),
-            }
+            node: if is_async.is_some() {
+                    ast::Statement::AsyncFunctionDef {
+                        name: i,
+                        args: a,
+                        body: s,
+                        decorator_list: d,
+                        returns: r.map(|x| x.1),
+                    }
+                } else {
+                    ast::Statement::FunctionDef {
+                        name: i,
+                        args: a,
+                        body: s,
+                        decorator_list: d,
+                        returns: r.map(|x| x.1),
+                    }
+                }
         }
     },
 };
@@ -736,10 +747,20 @@ Power: ast::Expression = {
 };
 
 AtomExpr: ast::Expression = {
+    <is_await:"await"?> <atom:AtomExpr2> => {
+        if is_await.is_some() {
+            ast::Expression::Await { value: Box::new(atom) }
+        } else {
+            atom
+        }
+    }
+}
+
+AtomExpr2: ast::Expression = {
     Atom,
-    <f:AtomExpr>  "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
-    <e:AtomExpr>  "[" <s:SubscriptList> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
-    <e:AtomExpr> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
+    <f:AtomExpr2> "(" <a:ArgumentList> ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 },
+    <e:AtomExpr2> "[" <s:SubscriptList> "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) },
+    <e:AtomExpr2> "." <n:Identifier> => ast::Expression::Attribute { value: Box::new(e), name: n },
 };
 
 SubscriptList: ast::Expression = {
@@ -1045,6 +1066,8 @@ extern {
         "and" => lexer::Tok::And,
         "as" => lexer::Tok::As,
         "assert" => lexer::Tok::Assert,
+        "async" => lexer::Tok::Async,
+        "await" => lexer::Tok::Await,
         "break" => lexer::Tok::Break,
         "class" => lexer::Tok::Class,
         "continue" => lexer::Tok::Continue,

parser/src/token.rs

@@ -72,6 +72,8 @@ pub enum Tok {
     And,
     As,
     Assert,
+    Async,
+    Await,
     Break,
     Class,
     Continue,

tests/snippets/test_async_stuff.py

@@ -0,0 +1,13 @@
+
+import sys
+import ast
+
+src = """
+async def x():
+    async for x in [1,2,3]:
+        await y()
+"""
+
+mod = ast.parse(src)
+# print(mod)
+# print(ast.dump(mod))

vm/src/compile.rs

@@ -339,6 +339,9 @@ impl Compiler {
                 body,
                 orelse,
             } => self.compile_for(target, iter, body, orelse)?,
+            ast::Statement::AsyncFor { .. } => {
+                unimplemented!("async for");
+            }
             ast::Statement::Raise { exception, cause } => match exception {
                 Some(value) => {
                     self.compile_expression(value)?;
@@ -369,6 +372,9 @@ impl Compiler {
                 decorator_list,
                 returns,
             } => self.compile_function_def(name, args, body, decorator_list, returns)?,
+            ast::Statement::AsyncFunctionDef { .. } => {
+                unimplemented!("async def");
+            }
             ast::Statement::ClassDef {
                 name,
                 body,
@@ -1282,6 +1288,9 @@ impl Compiler {
                 };
                 self.emit(Instruction::YieldValue);
             }
+            ast::Expression::Await { .. } => {
+                unimplemented!("await");
+            }
             ast::Expression::YieldFrom { value } => {
                 self.mark_generator();
                 self.compile_expression(value)?;

vm/src/stdlib/ast.rs

@@ -96,6 +96,26 @@ fn statement_to_ast(
                 returns => py_returns
             })
         }
+        ast::Statement::AsyncFunctionDef {
+            name,
+            args,
+            body,
+            decorator_list,
+            returns,
+        } => {
+            let py_returns = if let Some(hint) = returns {
+                expression_to_ast(vm, hint)?.into_object()
+            } else {
+                vm.ctx.none()
+            };
+            node!(vm, AsyncFunctionDef, {
+                name => vm.ctx.new_str(name.to_string()),
+                args => parameters_to_ast(vm, args)?,
+                body => statements_to_ast(vm, body)?,
+                decorator_list => expressions_to_ast(vm, decorator_list)?,
+                returns => py_returns
+            })
+        }
         ast::Statement::Continue => node!(vm, Continue),
         ast::Statement::Break => node!(vm, Break),
         ast::Statement::Pass => node!(vm, Pass),
@@ -152,6 +172,21 @@ fn statement_to_ast(
                 vm.ctx.none()
             }
         }),
+        ast::Statement::AsyncFor {
+            target,
+            iter,
+            body,
+            orelse,
+        } => node!(vm, AsyncFor, {
+            target => expression_to_ast(vm, target)?,
+            iter => expression_to_ast(vm, iter)?,
+            body => statements_to_ast(vm, body)?,
+            or_else => if let Some(orelse) = orelse {
+                statements_to_ast(vm, orelse)?.into_object()
+            } else {
+                vm.ctx.none()
+            }
+        }),
         ast::Statement::While { test, body, orelse } => node!(vm, While, {
             test => expression_to_ast(vm, test)?,
             body => statements_to_ast(vm, body)?,
@@ -350,6 +385,12 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes
                 }
             }
         }
+        ast::Expression::Await { value } => {
+            let py_value = expression_to_ast(vm, value)?;
+            node!(vm, Await, {
+                value => py_value
+            })
+        }
         ast::Expression::Yield { value } => {
             let py_value = match value {
                 Some(value) => expression_to_ast(vm, value)?.into_object(),
@@ -462,8 +503,11 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
         // TODO: There's got to be a better way!
         "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}),
         "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}),
+        "AsyncFor" => py_class!(ctx, "_ast.AsyncFor", ast_base.clone(), {}),
+        "AsyncFunctionDef" => py_class!(ctx, "_ast.AsyncFunctionDef", ast_base.clone(), {}),
         "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}),
         "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}),
+        "Await" => py_class!(ctx, "_ast.Await", ast_base.clone(), {}),
         "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}),
         "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}),
         "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}),

vm/src/symboltable.rs

@@ -223,6 +223,13 @@ impl SymbolTableBuilder {
                 args,
                 decorator_list,
                 returns,
+            }
+            | ast::Statement::AsyncFunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
             } => {
                 self.scan_expressions(decorator_list)?;
                 self.register_name(name, SymbolRole::Assigned)?;
@@ -265,6 +272,12 @@ impl SymbolTableBuilder {
                 iter,
                 body,
                 orelse,
+            }
+            | ast::Statement::AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
             } => {
                 self.scan_expression(target)?;
                 self.scan_expression(iter)?;
@@ -395,6 +408,9 @@ impl SymbolTableBuilder {
                     self.scan_expression(value)?;
                 }
             }
+            ast::Expression::Await { value } => {
+                self.scan_expression(value)?;
+            }
             ast::Expression::Yield { value } => {
                 if let Some(expression) = value {
                     self.scan_expression(expression)?;