Merge pull request #1964 from TheAnyKey/TheAnyKey/p38_named_expression_completion_step2

Py3.8 Named expression completion - error handling and scoping
This commit is contained in:
Noah authored and GitHub committed 2020-06-28 09:30:23 -05:00
commit 57289ff6f5
3 files changed
+217 -77

No files matched your search

+83 -67
View File
@@ -1,7 +1,5 @@
import unittest
#This test is only applicable for Python 3.8 and later
GLOBAL_VAR = None
class NamedExpressionInvalidTest(unittest.TestCase):
@@ -9,130 +7,130 @@ class NamedExpressionInvalidTest(unittest.TestCase):
def test_named_expression_invalid_01(self):
code = """x := 0"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_02(self):
code = """x = y := 0"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_03(self):
code = """y := f(x)"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_04(self):
code = """y0 = y1 := f(x)"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_06(self):
code = """((a, b) := (1, 2))"""
#with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_07(self):
code = """def spam(a = b := 42): pass"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_08(self):
code = """def spam(a: b := 42 = 5): pass"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_09(self):
code = """spam(a=b := 'c')"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_10(self):
code = """spam(x = y := f(x))"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_11(self):
code = """spam(a=1, b := 2)"""
#with self.assertRaisesRegex(SyntaxError,
# "positional argument follows keyword argument"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
# "positional argument follows keyword argument"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_12(self):
code = """spam(a=1, (b := 2))"""
#with self.assertRaisesRegex(SyntaxError,
# "positional argument follows keyword argument"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
# "positional argument follows keyword argument"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_13(self):
code = """spam(a=1, (b := 2))"""
#with self.assertRaisesRegex(SyntaxError,
# "positional argument follows keyword argument"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
# "positional argument follows keyword argument"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_14(self):
code = """(x := lambda: y := 1)"""
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_15(self):
code = """(lambda: x := 1)"""
#with self.assertRaisesRegex(SyntaxError,
# "cannot use assignment expressions with lambda"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
# "cannot use assignment expressions with lambda"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_16(self):
code = "[i + 1 for i in i := [1,2]]"
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
def test_named_expression_invalid_17(self):
code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]"
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
with self.assertRaises(SyntaxError): # TODO: RUSTPYTHON
#with self.assertRaisesRegex(SyntaxError, "invalid syntax"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_named_expression_invalid_in_class_body(self):
code = """class Foo():
[(42, 1 + ((( j := i )))) for i in range(5)]
"""
with self.assertRaisesRegex(SyntaxError,
"assignment expression within a comprehension cannot be used in a class body"):
#with self.assertRaisesRegex(SyntaxError,
# "assignment expression within a comprehension cannot be used in a class body"): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self):
cases = [
("Local reuse", 'i', "[i := 0 for i in range(5)]"),
@@ -145,28 +143,30 @@ class NamedExpressionInvalidTest(unittest.TestCase):
"[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"),
]
for case, target, code in cases:
msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'"
#msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'"
with self.subTest(case=case):
#with self.assertRaisesRegex(SyntaxError, msg): #TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {})
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_named_expression_invalid_rebinding_comprehension_inner_loop(self):
cases = [
("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"),
("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"),
]
for case, target, code in cases:
msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'"
#msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'"
with self.subTest(case=case):
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}) # Module scope
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {}) # Class scope
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(f"lambda: {code}", {}) # Function scope
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_named_expression_invalid_comprehension_iterable_expression(self):
cases = [
("Top level", "[i for i in (i := range(5))]"),
@@ -179,14 +179,17 @@ class NamedExpressionInvalidTest(unittest.TestCase):
("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"),
("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"),
]
msg = "assignment expression cannot be used in a comprehension iterable expression"
#msg = "assignment expression cannot be used in a comprehension iterable expression"
for case, code in cases:
with self.subTest(case=case):
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}) # Module scope
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(code, {}, {}) # Class scope
with self.assertRaisesRegex(SyntaxError, msg):
#with self.assertRaisesRegex(SyntaxError, msg): # TODO RustPython
with self.assertRaises(SyntaxError):
exec(f"lambda: {code}", {}) # Function scope
@@ -199,9 +202,9 @@ class NamedExpressionAssignmentTest(unittest.TestCase):
def test_named_expression_assignment_02(self):
a = 20
(a := a+10)
(a := a)
self.assertEqual(a, 30)
self.assertEqual(a, 20)
def test_named_expression_assignment_03(self):
(total := 1 + 2)
@@ -320,20 +323,10 @@ print(a)"""
def test_named_expression_scope_04(self):
def spam(a):
return a
y=1
res = [[y := spam(x), x/y] for x in range(1, 5)]
self.assertEqual(y, 4)
def test_named_expression_scope_04a(self):
def spam(a):
return a
y=1
res = [y := spam(x//y) for x in range(1, 5)]
self.assertEqual(y, 4)
def test_named_expression_scope_05(self):
def spam(a):
return a
@@ -343,14 +336,25 @@ print(a)"""
self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)])
self.assertEqual(y, 3)
# TODO: RUSTPYTHON,
@unittest.expectedFailure # TODO: RUSTPYTHON
# TODO RustPython, added test_named_expression_scope_06_rp_modified as
# minimaly modified variant of this test.
@unittest.expectedFailure
def test_named_expression_scope_06(self):
res = [[spam := i for i in range(3)] for j in range(2)]
self.assertEqual(res, [[0, 1, 2], [0, 1, 2]])
self.assertEqual(spam, 2)
# modified version of test_named_expression_scope_6, where locals
# assigned before to make them known in scop. THis is required due
# to some shortcommings in RPs name handling.
def test_named_expression_scope_06_rp_modified(self):
spam=0
res = [[spam := i for i in range(3)] for j in range(2)]
self.assertEqual(res, [[0, 1, 2], [0, 1, 2]])
self.assertEqual(spam, 2)
def test_named_expression_scope_07(self):
len(lines := [1, 2])
@@ -381,11 +385,23 @@ print(a)"""
self.assertEqual(res, [0, 2])
self.assertEqual(a, 2)
# TODO: RUSTPYTHON,
# TODO RustPython, added test_named_expression_scope_10_rp_modified
@unittest.expectedFailure
def test_named_expression_scope_10(self):
res = [b := [a := 1 for i in range(2)] for j in range(2)]
self.assertEqual(res, [[1, 1], [1, 1]])
self.assertEqual(a, 1)
self.assertEqual(b, [1, 1])
# modified version of test_named_expression_scope_10, where locals
# assigned before to make them known in scop. THis is required due
# to some shortcommings in RPs name handling.
def test_named_expression_scope_10_rp_modified(self):
a=0
b=0
res = [b := [a := 1 for i in range(2)] for j in range(2)]
self.assertEqual(res, [[1, 1], [1, 1]])
self.assertEqual(b, [1, 1])
self.assertEqual(a, 1)
@@ -395,7 +411,7 @@ print(a)"""
self.assertEqual(res, [0, 1, 2, 3, 4])
self.assertEqual(j, 4)
def test_named_expression_scope_17(self):
b = 0
res = [b := i + b for i in range(5)]
@@ -552,4 +568,4 @@ spam()"""
if __name__ == "__main__":
unittest.main()
unittest.main()
+102 -10
View File
@@ -105,6 +105,10 @@ pub struct Symbol {
// indicates if the symbol gets a value assigned by a named expression in a comprehension
// this is required to correct the scope in the analysis.
pub is_assign_namedexpr_in_comprehension: bool,
// inidicates that the symbol is used a bound iterator variable. We distinguish this case
// from normal assignment to detect unallowed re-assignment to iterator variables.
pub is_iter: bool,
}
impl Symbol {
@@ -118,6 +122,7 @@ impl Symbol {
is_parameter: false,
is_free: false,
is_assign_namedexpr_in_comprehension: false,
is_iter: false,
}
}
@@ -217,6 +222,10 @@ impl<'a> SymbolTableAnalyzer<'a> {
if symbol.is_assign_namedexpr_in_comprehension
&& curr_st_typ == SymbolTableType::Comprehension
{
// propagate symbol to next higher level that can hold it,
// i.e., function or module. Comprehension is skipped and
// Class is not allowed and detected as error.
//symbol.scope = SymbolScope::Nonlocal;
self.analyze_symbol_comprehension(symbol, 0)?
} else {
match symbol.scope {
@@ -303,16 +312,33 @@ impl<'a> SymbolTableAnalyzer<'a> {
let symbols = &mut last.0;
let table_type = last.1;
// it is not allowed to use an iterator variable as assignee in a named expression
if symbol.is_iter {
return Err(SymbolTableError {
error: format!(
"assignment expression cannot rebind comprehension iteration variable {}",
symbol.name
),
location: Default::default(),
});
}
match table_type {
SymbolTableType::Module => {
symbol.scope = SymbolScope::Global;
}
SymbolTableType::Class => {}
SymbolTableType::Class => {
// named expressions are forbidden in comprehensions on class scope
return Err(SymbolTableError {
error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
location: Default::default(),
} );
}
SymbolTableType::Function => {
if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
if let SymbolScope::Unknown = parent_symbol.scope {
parent_symbol.is_assigned = true; // this information is new, as the asignment is done in inner scope
self.analyze_unknown_symbol(symbol);
//self.analyze_unknown_symbol(symbol); // not needed, symbol is analyzed anyhow when its scope is analyzed
}
match symbol.scope {
@@ -323,17 +349,34 @@ impl<'a> SymbolTableAnalyzer<'a> {
symbol.scope = SymbolScope::Nonlocal;
}
}
} else {
let mut cloned_sym = symbol.clone();
cloned_sym.scope = SymbolScope::Local;
last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
}
}
SymbolTableType::Comprehension => {
// TODO check for conflicts - requires more context information about variables
match symbols.get_mut(&symbol.name) {
Some(parent_symbol) => {
// check if assignee is an iterator in top scope
if parent_symbol.is_iter {
return Err(SymbolTableError {
error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
location: Default::default(),
});
}
// we synthesize the assignment to the symbol from inner scope
parent_symbol.is_assigned = true; // more checks are required
}
None => {
let cloned_sym = symbol.clone();
// extend the scope of the inner symbol
// as we are in a nested comprehension, we expect that the symbol is needed
// ouside, too, and set it therefore to non-local scope. I.e., we expect to
// find a definition on a higher level
let mut cloned_sym = symbol.clone();
cloned_sym.scope = SymbolScope::Nonlocal;
last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
}
}
@@ -353,6 +396,7 @@ enum SymbolUsage {
Assigned,
Parameter,
AssignedNamedExprInCompr,
Iter,
}
#[derive(Default)]
@@ -369,6 +413,8 @@ enum ExpressionContext {
Load,
Store,
Delete,
Iter,
IterDefinitionExp,
}
impl SymbolTableBuilder {
@@ -705,11 +751,14 @@ impl SymbolTableBuilder {
let mut is_first_generator = true;
for generator in generators {
self.scan_expression(&generator.target, &ExpressionContext::Store)?;
self.scan_expression(&generator.target, &ExpressionContext::Iter)?;
if is_first_generator {
is_first_generator = false;
} else {
self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
self.scan_expression(
&generator.iter,
&ExpressionContext::IterDefinitionExp,
)?;
}
for if_expr in &generator.ifs {
@@ -721,14 +770,22 @@ impl SymbolTableBuilder {
// The first iterable is passed as an argument into the created function:
assert!(!generators.is_empty());
self.scan_expression(&generators[0].iter, &ExpressionContext::Load)?;
self.scan_expression(&generators[0].iter, &ExpressionContext::IterDefinitionExp)?;
}
Call {
function,
args,
keywords,
} => {
self.scan_expression(function, &ExpressionContext::Load)?;
match *context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(function, &ExpressionContext::IterDefinitionExp)?;
}
_ => {
self.scan_expression(function, &ExpressionContext::Load)?;
}
}
self.scan_expressions(args, &ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
@@ -743,17 +800,27 @@ impl SymbolTableBuilder {
ExpressionContext::Delete => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Load => {
ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Store => {
self.register_name(name, SymbolUsage::Assigned)?;
}
ExpressionContext::Iter => {
self.register_name(name, SymbolUsage::Iter)?;
}
}
}
Lambda { args, body } => {
self.enter_function("lambda", args, expression.location.row())?;
self.scan_expression(body, &ExpressionContext::Load)?;
match *context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, &ExpressionContext::IterDefinitionExp)?;
}
_ => {
self.scan_expression(body, &ExpressionContext::Load)?;
}
}
self.leave_scope();
}
IfExpression { test, body, orelse } => {
@@ -763,6 +830,15 @@ impl SymbolTableBuilder {
}
NamedExpression { left, right } => {
// named expressions are not allowed in the definiton of
// comprehension iterator definitions
if let ExpressionContext::IterDefinitionExp = *context {
return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
location: Default::default(),
});
}
self.scan_expression(right, &ExpressionContext::Load)?;
// special handling for assigned identifier in named expressions
@@ -933,8 +1009,24 @@ impl SymbolTableBuilder {
SymbolUsage::Used => {
symbol.is_referenced = true;
}
SymbolUsage::Iter => {
symbol.is_iter = true;
symbol.scope = SymbolScope::Local;
}
}
// and even more checking
// it is not allowed to assign to iterator variables (by named expressions)
if symbol.is_iter && symbol.is_assigned
/*&& symbol.is_assign_namedexpr_in_comprehension*/
{
return Err(SymbolTableError {
error:
"assignment expression cannot be used in a comprehension iterable expression"
.to_string(),
location,
});
}
Ok(())
}
}
+32
View File
@@ -9,11 +9,26 @@
#
# If you want to customize it or adapt the checks, just add or remove it from/ to the ACTIONS
# Preparation steps run before the first action is executed
# Typically we clear here the python cache to prevent dirty
# test results from cached python programs.
PRE_GLOBAL=("clear_pycache")
# Postprocessing steps run after the last ACTION has finished
POST_GLOBAL=()
# Preprocessing steps run before each action is executed
PRE_ACT=()
# Postprocessing steps run after each action is executed
POST_ACT=()
ACTIONS=("cargo build" "cargo build --release" "cargo fmt --all" "cargo clippy --all -- -Dwarnings" "cargo test --all" "cargo run --release -- -m test -v" "cd tests" "pytest" "cd ..")
# Usually, there should be no need to adapt the remaining file, when adding or removing actions.
# clears the python cache or RustPython
clear_pycache() { find . -name __pycache__ -type d -exec rm -r {} \; ; }
RUSTPYTHONPATH=Lib
export RUSTPYTHONPATH
@@ -21,12 +36,29 @@ export RUSTPYTHONPATH
ACT_RES=0
FAILS=()
for pre in "${PRE_GLOBAL[@]}"; do
$pre
done
for act in "${ACTIONS[@]}"; do
for pre in "${PRE_ACTION[@]}"; do
$pre
done
$act
if ! [ $? -eq 0 ]; then
ACT_RES=1
FAILS+=("${act}")
fi
for pst in "${POST_ACTION[@]}"; do
$pst
done
done
for pst in "${OST_GLOBAL[@]}"; do
$pst
done
echo