diff --git a/Lib/symtable.py b/Lib/symtable.py index 7a30e1ac4..c7152a70f 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ from enum import StrEnum __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: @@ -184,6 +184,7 @@ class Function(SymbolTable): __frees = None __globals = None __nonlocals = None + __cells = None def __idents_matching(self, test_func): return tuple(ident for ident in self.get_identifiers() @@ -229,6 +230,14 @@ class Function(SymbolTable): self.__frees = self.__idents_matching(is_free) return self.__frees + def get_cells(self): + """Return a tuple of cell variables in the function. + """ + if self.__cells is None: + is_cell = lambda x: _get_scope(x) == CELL + self.__cells = self.__idents_matching(is_cell) + return self.__cells + class Class(SymbolTable): @@ -255,11 +264,6 @@ class Class(SymbolTable): if is_local_symbol(st.name): match st.type: case _symtable.TYPE_FUNCTION: - # generators are of type TYPE_FUNCTION with a ".0" - # parameter as a first parameter (which makes them - # distinguishable from a function named 'genexpr') - if st.name == 'genexpr' and '.0' in st.varnames: - continue d[st.name] = 1 case _symtable.TYPE_TYPE_PARAMETERS: # Get the function-def block in the annotation @@ -267,13 +271,6 @@ class Class(SymbolTable): scope_name = st.name for c in st.children: if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: - # A generic generator of type TYPE_FUNCTION - # cannot be a direct child of 'st' (but it - # can be a descendant), e.g.: - # - # class A: - # type genexpr[genexpr] = (x for x in []) - assert scope_name != 'genexpr' or '.0' not in c.varnames d[scope_name] = 1 break self.__methods = tuple(d) @@ -354,6 +351,10 @@ class Symbol: """ return bool(self.__scope == FREE) + def is_cell(self): + """Return *True* if the symbol is a cell variable.""" + return bool(self.__scope == CELL) + def is_free_class(self): """Return *True* if a class-scoped symbol is free from the perspective of a method.""" @@ -413,7 +414,7 @@ class Symbol: _flags = [('USE', USE)] _flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_')) _scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL') -_scopes_value_to_name = {globals()[n]: n for n in _scopes_names} +_scopes_value_to_name = frozendict({globals()[n]: n for n in _scopes_names}) def main(args): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 9537de375..c74824311 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -5,6 +5,7 @@ Test the API of the symtable module. import re import textwrap import symtable +import warnings import unittest from test import support @@ -254,6 +255,7 @@ class SymtableTest(unittest.TestCase): self.assertEqual(sorted(func.get_locals()), expected) self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"]) self.assertEqual(self.internal.get_frees(), ("x",)) + self.assertEqual(self.spam.get_cells(), ("some_var", "x",)) def test_globals(self): self.assertTrue(self.spam.lookup("glob").is_global()) @@ -283,6 +285,9 @@ class SymtableTest(unittest.TestCase): def test_free(self): self.assertTrue(self.internal.lookup("x").is_free()) + def test_cells(self): + self.assertTrue(self.spam.lookup("x").is_cell()) + def test_referenced(self): self.assertTrue(self.internal.lookup("x").is_referenced()) self.assertTrue(self.spam.lookup("internal").is_referenced()) @@ -527,6 +532,58 @@ class SymtableTest(unittest.TestCase): expected = f"" self.assertEqual(repr(self.top._table), expected) + def test_lambda(self): + st = symtable.symtable("lambda x: x", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_lambda(self): + st = symtable.symtable("lambda x: lambda y=x: y", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["y"]) + self.assertEqual(st.get_children(), []) + + def test_genexpr(self): + st = symtable.symtable("(x for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_genexpr(self): + st = symtable.symtable("((y for y in x) for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) + self.assertEqual(st.get_children(), []) + def test__symtable_refleak(self): # Regression test for reference leak in PyUnicode_FSDecoder. # See https://github.com/python/cpython/issues/139748. @@ -534,6 +591,30 @@ class SymtableTest(unittest.TestCase): # check error path when 'compile_type' AC conversion failed self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + symtable.symtable(source, filename, 'exec') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/crates/vm/src/stdlib/_symtable.rs b/crates/vm/src/stdlib/_symtable.rs index c2d781dbb..3ceadaa91 100644 --- a/crates/vm/src/stdlib/_symtable.rs +++ b/crates/vm/src/stdlib/_symtable.rs @@ -3,8 +3,8 @@ pub(crate) use _symtable::module_def; #[pymodule] mod _symtable { use crate::{ - AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyUtf8StrRef}, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyStrRef, PyUtf8StrRef}, compiler, function::{ArgStrOrBytesLike, FsPath}, types::Representable, @@ -96,13 +96,26 @@ mod _symtable { #[pyattr] pub(super) const TYPE_TYPE_VARIABLE: i32 = 6; - #[pyfunction] - fn symtable( + #[derive(FromArgs)] + struct SymtableArgs { + #[pyarg(positional)] source: ArgStrOrBytesLike, + #[pyarg(positional)] filename: FsPath, + #[pyarg(positional)] startstr: PyUtf8StrRef, - vm: &VirtualMachine, - ) -> PyResult> { + #[pyarg(named, optional)] + module: Option, + } + + #[pyfunction] + fn symtable(args: SymtableArgs, vm: &VirtualMachine) -> PyResult> { + let SymtableArgs { + source, + filename, + startstr, + module, + } = args; let mode = startstr .as_str() .parse::() @@ -128,10 +141,23 @@ mod _symtable { "source code string cannot contain null bytes".into(), )); } - let symtable = compiler::compile_symtable(&source, mode, &filename).map_err(|err| { - let err = vm.new_syntax_error(&err, Some(&source)); - set_syntax_error_filename(err, &filename_obj, vm) - })?; + let module = module.filter(|module| !vm.is_none(module)); + let module = module + .map(|module| { + PyStrRef::try_from_object(vm, module.clone()).map_err(|_| { + vm.new_type_error(format!( + "symtable() argument 'module' must be str or None, not {}", + module.class().name() + )) + }) + }) + .transpose()?; + let symtable = vm + .compile_symtable_with_module(&source, mode, &filename, module.as_deref()) + .map_err(|err| { + let err = err.into_pyexception(vm, Some(&source)); + set_syntax_error_filename(err, &filename_obj, vm) + })?; Ok(to_py_symbol_table(symtable, vm)) } @@ -221,7 +247,13 @@ mod _symtable { impl Py { #[pygetset] fn name(&self) -> String { - self.symtable.name.to_string() + match self.symtable.typ { + CompilerScope::Lambda => "".to_owned(), + CompilerScope::Comprehension if self.symtable.name.as_str() == "genexpr" => { + "".to_owned() + } + _ => self.symtable.name.to_string(), + } } #[pygetset(name = "type")] diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 95dbcac5b..c4733b9f0 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -586,6 +586,30 @@ impl VirtualMachine { self.compile_with_opts_and_module(source, mode, source_path, opts, None, &[]) } + pub(crate) fn compile_symtable_with_module( + &self, + source: &str, + mode: compiler::Mode, + filename: &str, + module: Option<&Py>, + ) -> Result { + #[cfg(feature = "parser")] + { + let start = match mode { + compiler::Mode::Exec | compiler::Mode::BlockExpr => CompileStart::File, + compiler::Mode::Eval => CompileStart::Eval, + compiler::Mode::Single => CompileStart::Single, + }; + // A symbol table observes parser warnings, but does not compile + // bytecode or emit code-generation and AST-preprocessing warnings. + self.emit_ast_parser_warnings(source, filename, module, start, false) + .map_err(VmCompileError::Warning)?; + } + #[cfg(not(feature = "parser"))] + let _ = module; + compiler::compile_symtable(source, mode, filename).map_err(VmCompileError::Compile) + } + pub(crate) fn compile_with_opts_and_module( &self, source: &str,