From fd2e0aa41eecea324a5437a8cce36d232b1469b0 Mon Sep 17 00:00:00 2001 From: TheAnyKey <32773684+TheAnyKey@users.noreply.github.com> Date: Mon, 1 Jun 2020 11:22:15 +0200 Subject: [PATCH] Implement Py38 named expression (PEP 572) (#1934) * Initial implementation of named expression and import according CPython tests * added new instruction with inversed evaluation order for dict comprehension, in other cases use regular evaluation order * added further aspects to implementation, cleaned up, imported test from CPython * implemented first parts of scoping enhancement and extended checks * completion of name resolution ongoing, now more test passing, still warinings and cleanup required * further optimization of name resolution in nested scopes * Initialize the vm with imports from _io instead of io * Add the OpenBSD support that I am smart enough to * adapted grammer to full support, most test are passing now, esp. all invalids are passed. Cleaned up in symboltable * more conditional compiling, this time for errors * rustfmt was not pleased * premature push, whoops * Add expected_failure result type to jsontests * Initial implementation of named expression and import according CPython tests * added new instruction with inversed evaluation order for dict comprehension, in other cases use regular evaluation order * added further aspects to implementation, cleaned up, imported test from CPython * implemented first parts of scoping enhancement and extended checks * completion of name resolution ongoing, now more test passing, still warinings and cleanup required * further optimization of name resolution in nested scopes * adapted grammer to full support, most test are passing now, esp. all invalids are passed. Cleaned up in symboltable * Fixed nameing convention violation and removed unnecessary information from symbol resolution. Added some more comments. * Fixed nameing convention violation and removed unnecessary information from symbol resolution. Added some more comments. Co-authored-by: Noah <33094578+coolreader18@users.noreply.github.com> Co-authored-by: Reuben Staley --- Lib/test/test_named_expression.py | 577 ++++++++++++++++++++++++++++++ bytecode/src/bytecode.rs | 11 +- compiler/src/compile.rs | 11 +- compiler/src/symboltable.rs | 198 +++++++--- parser/src/ast.rs | 7 + parser/src/lexer.rs | 11 +- parser/src/python.lalrpop | 53 ++- parser/src/token.rs | 2 + vm/src/frame.rs | 8 + vm/src/stdlib/ast.rs | 3 + 10 files changed, 821 insertions(+), 60 deletions(-) create mode 100644 Lib/test/test_named_expression.py diff --git a/Lib/test/test_named_expression.py b/Lib/test/test_named_expression.py new file mode 100644 index 000000000..74500c4a0 --- /dev/null +++ b/Lib/test/test_named_expression.py @@ -0,0 +1,577 @@ +import unittest + +#This test is only applicable for Python 3.8 and later + +GLOBAL_VAR = None + +class NamedExpressionInvalidTest(unittest.TestCase): + + def test_named_expression_invalid_01(self): + code = """x := 0""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_02(self): + code = """x = y := 0""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_03(self): + code = """y := f(x)""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_04(self): + code = """y0 = y1 := f(x)""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_06(self): + code = """((a, b) := (1, 2))""" + + #with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_07(self): + code = """def spam(a = b := 42): pass""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_08(self): + code = """def spam(a: b := 42 = 5): pass""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_09(self): + code = """spam(a=b := 'c')""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_10(self): + code = """spam(x = y := f(x))""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_11(self): + code = """spam(a=1, b := 2)""" + + #with self.assertRaisesRegex(SyntaxError, + # "positional argument follows keyword argument"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_12(self): + code = """spam(a=1, (b := 2))""" + + #with self.assertRaisesRegex(SyntaxError, + # "positional argument follows keyword argument"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_13(self): + code = """spam(a=1, (b := 2))""" + + #with self.assertRaisesRegex(SyntaxError, + # "positional argument follows keyword argument"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_14(self): + code = """(x := lambda: y := 1)""" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_15(self): + code = """(lambda: x := 1)""" + + #with self.assertRaisesRegex(SyntaxError, + # "cannot use assignment expressions with lambda"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_16(self): + code = "[i + 1 for i in i := [1,2]]" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + def test_named_expression_invalid_17(self): + code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]" + + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): # TODO RustPython + exec(code, {}, {}) + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_invalid_in_class_body(self): + code = """class Foo(): + [(42, 1 + ((( j := i )))) for i in range(5)] + """ + + with self.assertRaisesRegex(SyntaxError, + "assignment expression within a comprehension cannot be used in a class body"): + exec(code, {}, {}) + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self): + cases = [ + ("Local reuse", 'i', "[i := 0 for i in range(5)]"), + ("Nested reuse", 'j', "[[(j := 0) for i in range(5)] for j in range(5)]"), + ("Reuse inner loop target", 'j', "[(j := 0) for i in range(5) for j in range(5)]"), + ("Unpacking reuse", 'i', "[i := 0 for i, j in [(0, 1)]]"), + ("Reuse in loop condition", 'i', "[i+1 for i in range(5) if (i := 0)]"), + ("Unreachable reuse", 'i', "[False or (i:=0) for i in range(5)]"), + ("Unreachable nested reuse", 'i', + "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"), + ] + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" + with self.subTest(case=case): + with self.assertRaises(SyntaxError): + exec(code, {}, {}) + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_invalid_rebinding_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"), + ("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"), + ] + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_invalid_comprehension_iterable_expression(self): + cases = [ + ("Top level", "[i for i in (i := range(5))]"), + ("Inside tuple", "[i for i in (2, 3, i := range(5))]"), + ("Inside list", "[i for i in [2, 3, i := range(5)]]"), + ("Different name", "[i for i in (j := range(5))]"), + ("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"), + ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"), + ("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"), + ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"), + ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + +class NamedExpressionAssignmentTest(unittest.TestCase): + + def test_named_expression_assignment_01(self): + (a := 10) + + self.assertEqual(a, 10) + + def test_named_expression_assignment_02(self): + a = 20 + (a := a+10) + + self.assertEqual(a, 30) + + def test_named_expression_assignment_03(self): + (total := 1 + 2) + + self.assertEqual(total, 3) + + def test_named_expression_assignment_04(self): + (info := (1, 2, 3)) + + self.assertEqual(info, (1, 2, 3)) + + def test_named_expression_assignment_05(self): + (x := 1, 2) + + self.assertEqual(x, 1) + + def test_named_expression_assignment_06(self): + (z := (y := (x := 0))) + + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_named_expression_assignment_07(self): + (loc := (1, 2)) + + self.assertEqual(loc, (1, 2)) + + def test_named_expression_assignment_08(self): + if spam := "eggs": + self.assertEqual(spam, "eggs") + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_09(self): + if True and (spam := True): + self.assertTrue(spam) + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_10(self): + if (match := 10) == 10: + pass + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_11(self): + def spam(a): + return a + input_data = [1, 2, 3] + res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) + + def test_named_expression_assignment_12(self): + def spam(a): + return a + res = [[y := spam(x), x/y] for x in range(1, 5)] + + self.assertEqual(res, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) + + def test_named_expression_assignment_13(self): + length = len(lines := [1, 2]) + + self.assertEqual(length, 2) + self.assertEqual(lines, [1,2]) + + def test_named_expression_assignment_14(self): + """ + Where all variables are positive integers, and a is at least as large + as the n'th root of x, this algorithm returns the floor of the n'th + root of x (and roughly doubling the number of accurate bits per + iteration): + """ + a = 9 + n = 2 + x = 3 + + while a > (d := x // a**(n-1)): + a = ((n-1)*a + d) // n + + self.assertEqual(a, 1) + + def test_named_expression_assignment_15(self): + while a := False: + pass # This will not run + + self.assertEqual(a, False) + + def test_named_expression_assignment_16(self): + a, b = 1, 2 + fib = {(c := a): (a := b) + (b := a + c) - b for __ in range(6)} + self.assertEqual(fib, {1: 2, 2: 3, 3: 5, 5: 8, 8: 13, 13: 21}) + + +class NamedExpressionScopeTest(unittest.TestCase): + + def test_named_expression_scope_01(self): + code = """def spam(): + (a := 5) +print(a)""" + + with self.assertRaisesRegex(NameError, "name 'a' is not defined"): + exec(code, {}, {}) + + def test_named_expression_scope_02(self): + total = 0 + partial_sums = [total := total + v for v in range(5)] + + self.assertEqual(partial_sums, [0, 1, 3, 6, 10]) + self.assertEqual(total, 10) + + def test_named_expression_scope_03(self): + containsOne = any((lastNum := num) == 1 for num in [1, 2, 3]) + + self.assertTrue(containsOne) + self.assertEqual(lastNum, 1) + + def test_named_expression_scope_04(self): + def spam(a): + return a + + y=1 + res = [[y := spam(x), x/y] for x in range(1, 5)] + + self.assertEqual(y, 4) + + def test_named_expression_scope_04a(self): + def spam(a): + return a + y=1 + res = [y := spam(x//y) for x in range(1, 5)] + + self.assertEqual(y, 4) + + def test_named_expression_scope_05(self): + def spam(a): + return a + input_data = [1, 2, 3] + res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) + self.assertEqual(y, 3) + + # TODO RustPython, + # seems to be general nesting problem, see + # test_named_expression_scop_10_rp_scope_prob as reference + @unittest.expectedFailure # TODO RustPython + def test_named_expression_scope_06(self): + res = [[spam := i for i in range(3)] for j in range(2)] + + self.assertEqual(res, [[0, 1, 2], [0, 1, 2]]) + self.assertEqual(spam, 2) + + def test_named_expression_scope_07(self): + len(lines := [1, 2]) + + self.assertEqual(lines, [1, 2]) + + def test_named_expression_scope_08(self): + def spam(a): + return a + + def eggs(b): + return b * 2 + + res = [spam(a := eggs(b := h)) for h in range(2)] + + self.assertEqual(res, [0, 2]) + self.assertEqual(a, 2) + self.assertEqual(b, 1) + + def test_named_expression_scope_09(self): + def spam(a): + return a + + def eggs(b): + return b * 2 + + res = [spam(a := eggs(a := h)) for h in range(2)] + + self.assertEqual(res, [0, 2]) + self.assertEqual(a, 2) + + # TODO RustPython, + # seems to be general nesting problem, see + # test_named_expression_scop_10_rp_scope_prob as reference + @unittest.expectedFailure + def test_named_expression_scope_10(self): + res = [b := [a := 1 for i in range(2)] for j in range(2)] + + self.assertEqual(res, [[1, 1], [1, 1]]) + self.assertEqual(b, [1, 1]) + self.assertEqual(a, 1) + + # the following test is not from CPyrgon and just as refernce for a common scoping problem of RustPython + @unittest.skip # needs skipping due to weired behaviour + def test_named_expression_scop_10_rp_scope_prob(self): + def foo(): + rr=0 + def foo0(): + nonlocal rr + def foo1(): + nonlocal rr + rr+=42 + foo1() + foo0() + return rr + + self.assertEqual(foo(), 42) + + + def test_named_expression_scope_11(self): + res = [j := i for i in range(5)] + + self.assertEqual(res, [0, 1, 2, 3, 4]) + self.assertEqual(j, 4) + + def test_named_expression_scope_17(self): + b = 0 + res = [b := i + b for i in range(5)] + + self.assertEqual(res, [0, 1, 3, 6, 10]) + self.assertEqual(b, 10) + + def test_named_expression_scope_18(self): + def spam(a): + return a + + res = spam(b := 2) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_19(self): + def spam(a): + return a + + res = spam((b := 2)) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_20(self): + def spam(a): + return a + + res = spam(a=(b := 2)) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_21(self): + def spam(a, b): + return a + b + + res = spam(c := 2, b=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_22(self): + def spam(a, b): + return a + b + + res = spam((c := 2), b=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_23(self): + def spam(a, b): + return a + b + + res = spam(b=(c := 2), a=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_24(self): + a = 10 + def spam(): + nonlocal a + (a := 20) + spam() + + self.assertEqual(a, 20) + + def test_named_expression_scope_25(self): + ns = {} + code = """a = 10 +def spam(): + global a + (a := 20) +spam()""" + + exec(code, ns, {}) + + self.assertEqual(ns["a"], 20) + + def test_named_expression_variable_reuse_in_comprehensions(self): + # The compiler is expected to raise syntax error for comprehension + # iteration variables, but should be fine with rebinding of other + # names (e.g. globals, nonlocals, other assignment expressions) + + # The cases are all defined to produce the same expected result + # Each comprehension is checked at both function scope and module scope + rebinding = "[x := i for i in range(3) if (x := i) or not x]" + filter_ref = "[x := i for i in range(3) if x or not x]" + body_ref = "[x for i in range(3) if (x := i) or not x]" + nested_ref = "[j for i in range(3) if x or not x for j in range(3) if (x := i)][:-3]" + cases = [ + ("Rebind global", f"x = 1; result = {rebinding}"), + ("Rebind nonlocal", f"result, x = (lambda x=1: ({rebinding}, x))()"), + ("Filter global", f"x = 1; result = {filter_ref}"), + ("Filter nonlocal", f"result, x = (lambda x=1: ({filter_ref}, x))()"), + ("Body global", f"x = 1; result = {body_ref}"), + ("Body nonlocal", f"result, x = (lambda x=1: ({body_ref}, x))()"), + ("Nested global", f"x = 1; result = {nested_ref}"), + ("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"), + ] + for case, code in cases: + with self.subTest(case=case): + ns = {} + exec(code, ns) + self.assertEqual(ns["x"], 2) + self.assertEqual(ns["result"], [0, 1, 2]) + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_global_scope(self): + sentinel = object() + global GLOBAL_VAR + def f(): + global GLOBAL_VAR + [GLOBAL_VAR := sentinel for _ in range(1)] + self.assertEqual(GLOBAL_VAR, sentinel) + try: + f() + self.assertEqual(GLOBAL_VAR, sentinel) + finally: + GLOBAL_VAR = None + + def test_named_expression_global_scope_no_global_keyword(self): + sentinel = object() + def f(): + GLOBAL_VAR = None + [GLOBAL_VAR := sentinel for _ in range(1)] + self.assertEqual(GLOBAL_VAR, sentinel) + f() + self.assertEqual(GLOBAL_VAR, None) + + @unittest.expectedFailure # TODO RustPython + def test_named_expression_nonlocal_scope(self): + sentinel = object() + def f(): + nonlocal_var = None + def g(): + nonlocal nonlocal_var + [nonlocal_var := sentinel for _ in range(1)] + g() + self.assertEqual(nonlocal_var, sentinel) + f() + + def test_named_expression_nonlocal_scope_no_nonlocal_keyword(self): + sentinel = object() + def f(): + nonlocal_var = None + def g(): + [nonlocal_var := sentinel for _ in range(1)] + g() + self.assertEqual(nonlocal_var, None) + f() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 907d488a8..835905c1b 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -273,6 +273,7 @@ pub enum Instruction { MapAdd { i: usize, }, + PrintExpr, LoadBuildClass, UnpackSequence { @@ -296,6 +297,13 @@ pub enum Instruction { }, GetAIter, GetANext, + + /// Reverse order evaluation in MapAdd + /// required to support named expressions of Python 3.8 in dict comprehension + /// today (including Py3.9) only required in dict comprehension. + MapAddRev { + i: usize, + }, } use self::Instruction::*; @@ -586,7 +594,7 @@ impl Instruction { BuildSlice { size } => w!(BuildSlice, size), ListAppend { i } => w!(ListAppend, i), SetAdd { i } => w!(SetAdd, i), - MapAdd { i } => w!(MapAdd, i), + MapAddRev { i } => w!(MapAddRev, i), PrintExpr => w!(PrintExpr), LoadBuildClass => w!(LoadBuildClass), UnpackSequence { size } => w!(UnpackSequence, size), @@ -597,6 +605,7 @@ impl Instruction { GetAwaitable => w!(GetAwaitable), GetAIter => w!(GetAIter), GetANext => w!(GetANext), + MapAdd { i } => w!(MapAdd, i), } } } diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 53aa36267..b2273bfaf 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1804,6 +1804,12 @@ impl Compiler { // End self.set_label(end_label); } + + NamedExpression { left, right } => { + self.compile_expression(right)?; + self.emit(Instruction::Duplicate); + self.compile_store(left)?; + } } Ok(()) } @@ -2066,10 +2072,11 @@ impl Compiler { }); } ast::ComprehensionKind::Dict { key, value } => { - self.compile_expression(value)?; + // changed evaluation order for Py38 named expression PEP 572 self.compile_expression(key)?; + self.compile_expression(value)?; - self.emit(Instruction::MapAdd { + self.emit(Instruction::MapAddRev { i: 1 + generators.len(), }); } diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 22acbe1ac..ca9594e79 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -66,6 +66,7 @@ pub enum SymbolTableType { Module, Class, Function, + Comprehension, } impl fmt::Display for SymbolTableType { @@ -74,6 +75,7 @@ impl fmt::Display for SymbolTableType { SymbolTableType::Module => write!(f, "module"), SymbolTableType::Class => write!(f, "class"), SymbolTableType::Function => write!(f, "function"), + SymbolTableType::Comprehension => write!(f, "comprehension"), } } } @@ -99,6 +101,10 @@ pub struct Symbol { pub is_assigned: bool, pub is_parameter: bool, pub is_free: bool, + + // indicates if the symbol gets a value assigned by a named expression in a comprehension + // this is required to correct the scope in the analysis. + pub is_assign_namedexpr_in_comprehension: bool, } impl Symbol { @@ -111,6 +117,7 @@ impl Symbol { is_assigned: false, is_parameter: false, is_free: false, + is_assign_namedexpr_in_comprehension: false, } } @@ -193,72 +200,138 @@ impl<'a> SymbolTableAnalyzer<'a> { for sub_table in sub_tables { self.analyze_symbol_table(sub_table)?; } - let (symbols, _) = self.tables.pop().unwrap(); + let (symbols, st_typ) = self.tables.pop().unwrap(); // Analyze symbols: for symbol in symbols.values_mut() { - self.analyze_symbol(symbol)?; + self.analyze_symbol(symbol, st_typ)?; } - Ok(()) } - fn analyze_symbol(&self, symbol: &mut Symbol) -> SymbolTableResult { - match symbol.scope { - SymbolScope::Nonlocal => { - // check if name is defined in parent table! - let parent_symbol_table = self.tables.last(); - // symbol.table.borrow().parent.clone(); - - if let Some((symbols, _)) = parent_symbol_table { - let scope_depth = self.tables.len(); - if !symbols.contains_key(&symbol.name) || scope_depth < 2 { + fn analyze_symbol( + &mut self, + symbol: &mut Symbol, + curr_st_typ: SymbolTableType, + ) -> SymbolTableResult { + if symbol.is_assign_namedexpr_in_comprehension + && curr_st_typ == SymbolTableType::Comprehension + { + self.analyze_symbol_comprehension(symbol, 0)? + } else { + match symbol.scope { + SymbolScope::Nonlocal => { + // check if name is defined in parent table! + let parent_symbol_table = self.tables.last(); + if let Some((symbols, _)) = parent_symbol_table { + let scope_depth = self.tables.len(); + if !symbols.contains_key(&symbol.name) || scope_depth < 2 { + return Err(SymbolTableError { + error: format!("no binding for nonlocal '{}' found", symbol.name), + location: Default::default(), + }); + } + } else { return Err(SymbolTableError { - error: format!("no binding for nonlocal '{}' found", symbol.name), + error: format!( + "nonlocal {} defined at place without an enclosing scope", + symbol.name + ), location: Default::default(), }); } - } else { - return Err(SymbolTableError { - error: format!( - "nonlocal {} defined at place without an enclosing scope", - symbol.name - ), - location: Default::default(), - }); + } + SymbolScope::Global => { + // TODO: add more checks for globals? + } + SymbolScope::Local => { + // all is well + } + SymbolScope::Unknown => { + // Try hard to figure out what the scope of this symbol is. + self.analyze_unknown_symbol(symbol); } } - SymbolScope::Global => { - // TODO: add more checks for globals? - } - SymbolScope::Local => { - // all is well - } - SymbolScope::Unknown => { - // Try hard to figure out what the scope of this symbol is. + } + Ok(()) + } - if symbol.is_assigned || symbol.is_parameter { - symbol.scope = SymbolScope::Local; - } else { - // Interesting stuff about the __class__ variable: - // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object - let found_in_outer_scope = symbol.name == "__class__" - || self.tables.iter().skip(1).any(|(symbols, typ)| { - *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name) - }); + fn analyze_unknown_symbol(&self, symbol: &mut Symbol) { + if symbol.is_assigned || symbol.is_parameter { + symbol.scope = SymbolScope::Local; + } else { + // Interesting stuff about the __class__ variable: + // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object + let found_in_outer_scope = symbol.name == "__class__" + || self.tables.iter().skip(1).any(|(symbols, typ)| { + *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name) + }); - if found_in_outer_scope { - // Symbol is in some outer scope. - symbol.is_free = true; - } else if self.tables.is_empty() { - // Don't make assumptions when we don't know. - symbol.scope = SymbolScope::Unknown; - } else { - // If there are scopes above we can assume global. - symbol.scope = SymbolScope::Global; + if found_in_outer_scope { + // Symbol is in some outer scope. + symbol.is_free = true; + } else if self.tables.is_empty() { + // Don't make assumptions when we don't know. + symbol.scope = SymbolScope::Unknown; + } else { + // If there are scopes above we can assume global. + symbol.scope = SymbolScope::Global; + } + } + } + + // Implements the symbol analysis and scope extension for names + // assigned by a named expression in a comprehension. See: + // https://github.com/python/cpython/blob/7b78e7f9fd77bb3280ee39fb74b86772a7d46a70/Python/symtable.c#L1435 + fn analyze_symbol_comprehension( + &mut self, + symbol: &mut Symbol, + parent_offset: usize, + ) -> SymbolTableResult { + // TODO: quite C-ish way to implement the iteration + // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol' + let offs = self.tables.len() - 1 - parent_offset; + let last = self.tables.get_mut(offs).unwrap(); + let symbols = &mut last.0; + let table_type = last.1; + + match table_type { + SymbolTableType::Module => { + symbol.scope = SymbolScope::Global; + } + SymbolTableType::Class => {} + SymbolTableType::Function => { + if let Some(parent_symbol) = symbols.get_mut(&symbol.name) { + if let SymbolScope::Unknown = parent_symbol.scope { + parent_symbol.is_assigned = true; // this information is new, as the asignment is done in inner scope + self.analyze_unknown_symbol(symbol); + } + + match symbol.scope { + SymbolScope::Global => { + symbol.scope = SymbolScope::Global; + } + _ => { + symbol.scope = SymbolScope::Nonlocal; + } } } } + SymbolTableType::Comprehension => { + // TODO check for conflicts - requires more context information about variables + match symbols.get_mut(&symbol.name) { + Some(parent_symbol) => { + parent_symbol.is_assigned = true; // more checks are required + } + None => { + let cloned_sym = symbol.clone(); + + last.0.insert(cloned_sym.name.to_owned(), cloned_sym); + } + } + + self.analyze_symbol_comprehension(symbol, parent_offset + 1)?; + } } Ok(()) } @@ -271,6 +344,7 @@ enum SymbolUsage { Used, Assigned, Parameter, + AssignedNamedExprInCompr, } #[derive(Default)] @@ -602,7 +676,7 @@ impl SymbolTableBuilder { self.enter_scope( scope_name, - SymbolTableType::Function, + SymbolTableType::Comprehension, expression.location.row(), ); @@ -679,6 +753,28 @@ impl SymbolTableBuilder { self.scan_expression(body, &ExpressionContext::Load)?; self.scan_expression(orelse, &ExpressionContext::Load)?; } + + NamedExpression { left, right } => { + self.scan_expression(right, &ExpressionContext::Load)?; + + // special handling for assigned identifier in named expressions + // that are used in comprehensions. This required to correctly + // propagate the scope of the named assigned named and not to + // propagate inner names. + if let Identifier { name } = &left.node { + let table = self.tables.last().unwrap(); + if table.typ == SymbolTableType::Comprehension { + self.register_name(name, SymbolUsage::AssignedNamedExprInCompr)?; + } else { + // omit one recursion. When the handling of an store changes for + // Identifiers this needs adapted - more forward safe would be + // calling scan_expression directly. + self.register_name(name, SymbolUsage::Assigned)?; + } + } else { + self.scan_expression(left, &ExpressionContext::Store)?; + } + } } Ok(()) } @@ -810,6 +906,10 @@ impl SymbolTableBuilder { SymbolUsage::Assigned => { symbol.is_assigned = true; } + SymbolUsage::AssignedNamedExprInCompr => { + symbol.is_assigned = true; + symbol.is_assign_namedexpr_in_comprehension = true; + } SymbolUsage::Global => { if let SymbolScope::Unknown = symbol.scope { symbol.scope = SymbolScope::Global; diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 7807c06a1..dc4acba89 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -309,6 +309,12 @@ pub enum ExpressionType { orelse: Box, }, + // A named expression + NamedExpression { + left: Box, + right: Box, + }, + /// The literal 'True'. True, @@ -364,6 +370,7 @@ impl Expression { IfExpression { .. } => "conditional expression", True | False | None => "keyword", Ellipsis => "ellipsis", + NamedExpression { .. } => "named expression", } } } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 8a1254fac..0467a01d3 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1075,7 +1075,16 @@ where self.nesting -= 1; } ':' => { - self.eat_single_char(Tok::Colon); + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + self.emit((tok_start, Tok::ColonEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + self.emit((tok_start, Tok::Colon, tok_end)); + } } ';' => { self.eat_single_char(Tok::Semi); diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index c0958a114..7e3a1dbc1 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -148,11 +148,29 @@ TestOrStarExprList: ast::Expression = { } }; +TestOrStarNamedExprList: ast::Expression = { + > => { + if elements.len() == 1 && comma.is_none() { + elements.into_iter().next().unwrap() + } else { + ast::Expression { + location, + node: ast::ExpressionType::Tuple { elements } + } + } + } +}; + TestOrStarExpr: ast::Expression = { Test, StarExpr, }; +TestOrStarNamedExpr: ast::Expression = { + NamedExpressionTest, + StarExpr, +}; + AugAssign: ast::Operator = { "+=" => ast::Operator::Add, "-=" => ast::Operator::Sub, @@ -314,7 +332,7 @@ CompoundStatement: ast::Statement = { }; IfStatement: ast::Statement = { - "if" ":" => { + "if" ":" => { // Determine last else: let mut last = s3.map(|s| s.2); @@ -335,7 +353,7 @@ IfStatement: ast::Statement = { }; WhileStatement: ast::Statement = { - "while" ":" => { + "while" ":" => { let orelse = s2.map(|s| s.2); ast::Statement { location, @@ -640,6 +658,26 @@ Test: ast::Expression = { LambdaDef, }; +NamedExpressionTest: ast::Expression = { + ?> ":" => ast::Expression { @@ -900,7 +938,7 @@ Atom: ast::Expression = { node: ast::ExpressionType::List { elements } } }, - "[" "]" => { + "[" "]" => { ast::Expression { location, node: ast::ExpressionType::Comprehension { @@ -909,7 +947,7 @@ Atom: ast::Expression = { } } }, - "(" ")" => { + "(" ")" => { elements.unwrap_or(ast::Expression { location, node: ast::ExpressionType::Tuple { elements: Vec::new() } @@ -958,7 +996,7 @@ Atom: ast::Expression = { }; ListLiteralValues: Vec = { - > ","? => e, + > ","? => e, }; DictLiteralValues: Vec<(Option, ast::Expression)> = { @@ -975,7 +1013,7 @@ DictElement: (Option, ast::Expression) = { }; SetLiteralValues: Vec = { - > ","? => e1 + > ","? => e1 }; ExpressionOrStarExpression = { @@ -1046,7 +1084,7 @@ ArgumentList: ast::ArgumentList = { }; FunctionArgument: (Option>, ast::Expression) = { - => { + => { let expr = match c { Some(c) => ast::Expression { location: e.location, @@ -1164,6 +1202,7 @@ extern { ">>=" => lexer::Tok::RightShiftEqual, "**=" => lexer::Tok::DoubleStarEqual, "//=" => lexer::Tok::DoubleSlashEqual, + ":=" => lexer::Tok::ColonEqual, "==" => lexer::Tok::EqEqual, "!=" => lexer::Tok::NotEqual, "<" => lexer::Tok::Less, diff --git a/parser/src/token.rs b/parser/src/token.rs index 09122bb83..bb11c6bac 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -61,6 +61,7 @@ pub enum Tok { RightShiftEqual, DoubleSlash, // '//' DoubleSlashEqual, + ColonEqual, At, AtEqual, Rarrow, @@ -222,6 +223,7 @@ impl fmt::Display for Tok { While => f.write_str("'while'"), With => f.write_str("'with'"), Yield => f.write_str("'yield'"), + ColonEqual => f.write_str("':='"), } } } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 0b4416fdb..f8663a652 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -426,6 +426,14 @@ impl ExecutingFrame<'_> { vm.call_method(&dict_obj, "__setitem__", vec![key, value])?; Ok(None) } + bytecode::Instruction::MapAddRev { i } => { + // change order of evalutio of key and value to support Py3.8 Named expressions in dict comprehension + let dict_obj = self.nth_value(*i + 1); + let value = self.pop_value(); + let key = self.pop_value(); + vm.call_method(&dict_obj, "__setitem__", vec![key, value])?; + Ok(None) + } bytecode::Instruction::BinaryOperation { ref op, inplace } => { self.execute_binop(vm, op, *inplace) } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 33b3ea622..e28f1e48f 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -517,6 +517,9 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyRes }), String { value } => string_to_ast(vm, value)?, Bytes { value } => node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }), + NamedExpression { left, right } => { + node!(vm, NamedExpression, { left => expression_to_ast(vm, left)?, right => expression_to_ast(vm, right)? }) + } }; let lineno = vm.ctx.new_int(expression.location.row());