diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py new file mode 100644 index 000000000..8e83bc72f --- /dev/null +++ b/Lib/test/test_symtable.py @@ -0,0 +1,251 @@ +""" +Test the API of the symtable module. +""" +import symtable +import unittest + + + +TEST_CODE = """ +import sys + +glob = 42 +some_var = 12 + +class Mine: + instance_var = 24 + def a_method(p1, p2): + pass + +def spam(a, b, *var, **kw): + global bar + bar = 47 + some_var = 10 + x = 23 + glob + def internal(): + return x + def other_internal(): + nonlocal some_var + some_var = 3 + return some_var + return internal + +def foo(): + pass + +def namespace_test(): pass +def namespace_test(): pass +""" + + +def find_block(block, name): + for ch in block.get_children(): + if ch.get_name() == name: + return ch + + +class SymtableTest(unittest.TestCase): + + top = symtable.symtable(TEST_CODE, "?", "exec") + # These correspond to scopes in TEST_CODE + Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") + spam = find_block(top, "spam") + internal = find_block(spam, "internal") + other_internal = find_block(spam, "other_internal") + foo = find_block(top, "foo") + + def test_type(self): + self.assertEqual(self.top.get_type(), "module") + self.assertEqual(self.Mine.get_type(), "class") + self.assertEqual(self.a_method.get_type(), "function") + self.assertEqual(self.spam.get_type(), "function") + self.assertEqual(self.internal.get_type(), "function") + + @unittest.skip("TODO: RUSTPYTHON") + def test_id(self): + self.assertGreater(self.top.get_id(), 0) + self.assertGreater(self.Mine.get_id(), 0) + self.assertGreater(self.a_method.get_id(), 0) + self.assertGreater(self.spam.get_id(), 0) + self.assertGreater(self.internal.get_id(), 0) + + @unittest.skip("TODO: RUSTPYTHON") + def test_optimized(self): + self.assertFalse(self.top.is_optimized()) + + self.assertTrue(self.spam.is_optimized()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_nested(self): + self.assertFalse(self.top.is_nested()) + self.assertFalse(self.Mine.is_nested()) + self.assertFalse(self.spam.is_nested()) + self.assertTrue(self.internal.is_nested()) + + def test_children(self): + self.assertTrue(self.top.has_children()) + self.assertTrue(self.Mine.has_children()) + self.assertFalse(self.foo.has_children()) + + def test_lineno(self): + self.assertEqual(self.top.get_lineno(), 0) + self.assertEqual(self.spam.get_lineno(), 12) + + @unittest.skip("TODO: RUSTPYTHON") + def test_function_info(self): + func = self.spam + self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"]) + expected = ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x'] + self.assertEqual(sorted(func.get_locals()), expected) + self.assertEqual(sorted(func.get_globals()), ["bar", "glob"]) + self.assertEqual(self.internal.get_frees(), ("x",)) + + @unittest.skip("TODO: RUSTPYTHON") + def test_globals(self): + self.assertTrue(self.spam.lookup("glob").is_global()) + self.assertFalse(self.spam.lookup("glob").is_declared_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) + self.assertTrue(self.spam.lookup("bar").is_declared_global()) + self.assertFalse(self.internal.lookup("x").is_global()) + self.assertFalse(self.Mine.lookup("instance_var").is_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) + + @unittest.skip("TODO: RUSTPYTHON") + def test_nonlocal(self): + self.assertFalse(self.spam.lookup("some_var").is_nonlocal()) + self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal()) + expected = ("some_var",) + self.assertEqual(self.other_internal.get_nonlocals(), expected) + + def test_local(self): + self.assertTrue(self.spam.lookup("x").is_local()) + self.assertFalse(self.spam.lookup("bar").is_local()) + + def test_free(self): + self.assertTrue(self.internal.lookup("x").is_free()) + + def test_referenced(self): + self.assertTrue(self.internal.lookup("x").is_referenced()) + self.assertTrue(self.spam.lookup("internal").is_referenced()) + self.assertFalse(self.spam.lookup("x").is_referenced()) + + def test_parameters(self): + for sym in ("a", "var", "kw"): + self.assertTrue(self.spam.lookup(sym).is_parameter()) + self.assertFalse(self.spam.lookup("x").is_parameter()) + + def test_symbol_lookup(self): + self.assertEqual(len(self.top.get_identifiers()), + len(self.top.get_symbols())) + + self.assertRaises(KeyError, self.top.lookup, "not_here") + + def test_namespaces(self): + self.assertTrue(self.top.lookup("Mine").is_namespace()) + self.assertTrue(self.Mine.lookup("a_method").is_namespace()) + self.assertTrue(self.top.lookup("spam").is_namespace()) + self.assertTrue(self.spam.lookup("internal").is_namespace()) + self.assertTrue(self.top.lookup("namespace_test").is_namespace()) + self.assertFalse(self.spam.lookup("x").is_namespace()) + + # TODO(RUSTPYTHON): lookup should return same pythonref + # self.assertTrue(self.top.lookup("spam").get_namespace() is self.spam) + ns_test = self.top.lookup("namespace_test") + self.assertEqual(len(ns_test.get_namespaces()), 2) + self.assertRaises(ValueError, ns_test.get_namespace) + + def test_assigned(self): + self.assertTrue(self.spam.lookup("x").is_assigned()) + self.assertTrue(self.spam.lookup("bar").is_assigned()) + self.assertTrue(self.top.lookup("spam").is_assigned()) + self.assertTrue(self.Mine.lookup("a_method").is_assigned()) + self.assertFalse(self.internal.lookup("x").is_assigned()) + + def test_annotated(self): + st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') + st2 = st1.get_children()[0] + self.assertTrue(st2.lookup('x').is_local()) + self.assertTrue(st2.lookup('x').is_annotated()) + self.assertFalse(st2.lookup('x').is_global()) + st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') + st4 = st3.get_children()[0] + self.assertTrue(st4.lookup('x').is_local()) + self.assertFalse(st4.lookup('x').is_annotated()) + + # Test that annotations in the global scope are valid after the + # variable is declared as nonlocal. + st5 = symtable.symtable('global x\nx: int', 'test', 'exec') + self.assertTrue(st5.lookup("x").is_global()) + + # Test that annotations for nonlocals are valid after the + # variable is declared as nonlocal. + st6 = symtable.symtable('def g():\n' + ' x = 2\n' + ' def f():\n' + ' nonlocal x\n' + ' x: int', + 'test', 'exec') + + def test_imported(self): + self.assertTrue(self.top.lookup("sys").is_imported()) + + def test_name(self): + self.assertEqual(self.top.get_name(), "top") + self.assertEqual(self.spam.get_name(), "spam") + self.assertEqual(self.spam.lookup("x").get_name(), "x") + self.assertEqual(self.Mine.get_name(), "Mine") + + @unittest.skip("TODO: RUSTPYTHON") + def test_class_info(self): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + @unittest.skip("TODO: RUSTPYTHON") + def test_filename_correct(self): + ### Bug tickler: SyntaxError file name correct whether error raised + ### while parsing or building symbol table. + def checkfilename(brokencode, offset): + try: + symtable.symtable(brokencode, "spam", "exec") + except SyntaxError as e: + self.assertEqual(e.filename, "spam") + self.assertEqual(e.lineno, 1) + self.assertEqual(e.offset, offset) + else: + self.fail("no SyntaxError for %r" % (brokencode,)) + # TODO: RUSTPYTHON, now offset get 15 + checkfilename("def f(x): foo)(", 14) # parse-time + checkfilename("def f(x): global x", 11) # symtable-build-time + symtable.symtable("pass", b"spam", "exec") + with self.assertWarns(DeprecationWarning), \ + self.assertRaises(TypeError): + symtable.symtable("pass", bytearray(b"spam"), "exec") + with self.assertWarns(DeprecationWarning): + symtable.symtable("pass", memoryview(b"spam"), "exec") + with self.assertRaises(TypeError): + symtable.symtable("pass", list(b"spam"), "exec") + + def test_eval(self): + symbols = symtable.symtable("42", "?", "eval") + + @unittest.skip("TODO: RUSTPYTHON") + def test_single(self): + symbols = symtable.symtable("42", "?", "single") + + def test_exec(self): + symbols = symtable.symtable("def f(x): return x", "?", "exec") + + @unittest.skip("TODO: RUSTPYTHON") + def test_bytes(self): + top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") + self.assertIsNotNone(find_block(top, "Mine")) + + code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n' + + top = symtable.symtable(code, "?", "exec") + self.assertIsNotNone(find_block(top, "\u017d")) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 5714367bb..e8b80f09e 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -97,10 +97,13 @@ pub struct Symbol { pub name: String, // pub table: SymbolTableRef, pub scope: SymbolScope, + // TODO: Use bitflags replace pub is_referenced: bool, pub is_assigned: bool, pub is_parameter: bool, pub is_free: bool, + pub is_annotated: bool, + pub is_imported: bool, // indicates if the symbol gets a value assigned by a named expression in a comprehension // this is required to correct the scope in the analysis. @@ -121,6 +124,8 @@ impl Symbol { is_assigned: false, is_parameter: false, is_free: false, + is_annotated: false, + is_imported: false, is_assign_namedexpr_in_comprehension: false, is_iter: false, } @@ -394,7 +399,10 @@ enum SymbolUsage { Nonlocal, Used, Assigned, + Imported, + AnnotationAssigned, Parameter, + AnnotationParameter, AssignedNamedExprInCompr, Iter, } @@ -460,7 +468,12 @@ impl SymbolTableBuilder { } fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { - self.register_name(¶meter.arg, SymbolUsage::Parameter) + let usage = if parameter.annotation.is_some() { + SymbolUsage::AnnotationParameter + } else { + SymbolUsage::Parameter + }; + self.register_name(¶meter.arg, usage) } fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { @@ -564,12 +577,12 @@ impl SymbolTableBuilder { for name in names { if let Some(alias) = &name.alias { // `import mymodule as myalias` - self.register_name(alias, SymbolUsage::Assigned)?; + self.register_name(alias, SymbolUsage::Imported)?; } else { // `import module` self.register_name( name.symbol.split('.').next().unwrap(), - SymbolUsage::Assigned, + SymbolUsage::Imported, )?; } } @@ -601,7 +614,12 @@ impl SymbolTableBuilder { annotation, value, } => { - self.scan_expression(target, &ExpressionContext::Store)?; + // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233 + if let ast::ExpressionType::Identifier { ref name } = target.node { + self.register_name(name, SymbolUsage::AnnotationAssigned)?; + } else { + self.scan_expression(target, &ExpressionContext::Store)?; + } self.scan_expression(annotation, &ExpressionContext::Load)?; if let Some(value) = value { self.scan_expression(value, &ExpressionContext::Load)?; @@ -984,9 +1002,21 @@ impl SymbolTableBuilder { }); } } + SymbolUsage::Imported => { + symbol.is_assigned = true; + symbol.is_imported = true; + } SymbolUsage::Parameter => { symbol.is_parameter = true; } + SymbolUsage::AnnotationParameter => { + symbol.is_parameter = true; + symbol.is_annotated = true; + } + SymbolUsage::AnnotationAssigned => { + symbol.is_assigned = true; + symbol.is_annotated = true; + } SymbolUsage::Assigned => { symbol.is_assigned = true; } diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 0d90b0bdb..2172f974c 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -109,10 +109,17 @@ impl PySymbolTable { if let Some(symbol) = self.symtable.symbols.get(name) { Ok(PySymbol { symbol: symbol.clone(), + namespaces: self + .symtable + .sub_tables + .iter() + .filter(|table| table.name == name) + .cloned() + .collect(), } .into_ref(vm)) } else { - Err(vm.new_lookup_error(name.to_owned())) + Err(vm.new_key_error(vm.new_str(format!("lookup {} failed", name)))) } } @@ -133,7 +140,20 @@ impl PySymbolTable { .symtable .symbols .values() - .map(|s| (PySymbol { symbol: s.clone() }).into_ref(vm).into_object()) + .map(|s| { + (PySymbol { + symbol: s.clone(), + namespaces: self + .symtable + .sub_tables + .iter() + .filter(|&table| table.name == s.name) + .cloned() + .collect(), + }) + .into_ref(vm) + .into_object() + }) .collect(); Ok(vm.ctx.new_list(symbols)) } @@ -158,6 +178,7 @@ impl PySymbolTable { #[pyclass(name = "Symbol")] struct PySymbol { symbol: symboltable::Symbol, + namespaces: Vec, } impl fmt::Debug for PySymbol { @@ -189,6 +210,25 @@ impl PySymbol { self.symbol.is_local() } + #[pymethod(name = "is_imported")] + fn is_imported(&self) -> bool { + self.symbol.is_imported + } + + #[pymethod(name = "is_nested")] + fn is_nested(&self) -> bool { + // TODO + false + } + + #[pymethod(name = "is_nonlocal")] + fn is_nonlocal(&self) -> bool { + match self.symbol.scope { + symboltable::SymbolScope::Nonlocal => true, + _ => false, + } + } + #[pymethod(name = "is_referenced")] fn is_referenced(&self) -> bool { self.symbol.is_referenced @@ -211,7 +251,32 @@ impl PySymbol { #[pymethod(name = "is_namespace")] fn is_namespace(&self) -> bool { - // TODO - false + !self.namespaces.is_empty() + } + + #[pymethod(name = "is_annotated")] + fn is_annotated(&self) -> bool { + self.symbol.is_annotated + } + + #[pymethod(name = "get_namespaces")] + fn get_namespaces(&self, vm: &VirtualMachine) -> PyResult { + let namespaces = self + .namespaces + .iter() + .map(|table| to_py_symbol_table(table.clone()).into_ref(vm).into_object()) + .collect(); + Ok(vm.ctx.new_list(namespaces)) + } + + #[pymethod(name = "get_namespace")] + fn get_namespace(&self, vm: &VirtualMachine) -> PyResult { + if self.namespaces.len() != 1 { + Err(vm.new_value_error("namespace is bound to multiple namespaces".to_owned())) + } else { + Ok(to_py_symbol_table(self.namespaces.first().unwrap().clone()) + .into_ref(vm) + .into_object()) + } } }