Align symbol table introspection with Python 3.15 (#9277)

Accept the module keyword when building symbol tables and emit only parser
warnings through the existing bounded warning path. Preserve source filenames
and warning-to-SyntaxError conversion without compiling bytecode.

Expose canonical lambda and generator-expression names while retaining
already-bracketed comprehension names. Synchronize symtable and its tests
from pinned CPython 3.15.0 through update_lib, including cell introspection.

Assisted-by: OpenAI Codex:model-version-unavailable

Co-authored-by: youknowdot <dot@youknowone.org>

Lib/symtable.py

@@ -17,13 +17,13 @@ from enum import StrEnum
 
 __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"]
 
-def symtable(code, filename, compile_type):
+def symtable(code, filename, compile_type, *, module=None):
     """ Return the toplevel *SymbolTable* for the source code.
 
     *filename* is the name of the file with the code
     and *compile_type* is the *compile()* mode argument.
     """
-    top = _symtable.symtable(code, filename, compile_type)
+    top = _symtable.symtable(code, filename, compile_type, module=module)
     return _newSymbolTable(top, filename)
 
 class SymbolTableFactory:
@@ -184,6 +184,7 @@ class Function(SymbolTable):
     __frees = None
     __globals = None
     __nonlocals = None
+    __cells = None
 
     def __idents_matching(self, test_func):
         return tuple(ident for ident in self.get_identifiers()
@@ -229,6 +230,14 @@ class Function(SymbolTable):
             self.__frees = self.__idents_matching(is_free)
         return self.__frees
 
+    def get_cells(self):
+        """Return a tuple of cell variables in the function.
+        """
+        if self.__cells is None:
+            is_cell = lambda x: _get_scope(x) == CELL
+            self.__cells = self.__idents_matching(is_cell)
+        return self.__cells
+
 
 class Class(SymbolTable):
 
@@ -255,11 +264,6 @@ class Class(SymbolTable):
                 if is_local_symbol(st.name):
                     match st.type:
                         case _symtable.TYPE_FUNCTION:
-                            # generators are of type TYPE_FUNCTION with a ".0"
-                            # parameter as a first parameter (which makes them
-                            # distinguishable from a function named 'genexpr')
-                            if st.name == 'genexpr' and '.0' in st.varnames:
-                                continue
                             d[st.name] = 1
                         case _symtable.TYPE_TYPE_PARAMETERS:
                             # Get the function-def block in the annotation
@@ -267,13 +271,6 @@ class Class(SymbolTable):
                             scope_name = st.name
                             for c in st.children:
                                 if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION:
-                                    # A generic generator of type TYPE_FUNCTION
-                                    # cannot be a direct child of 'st' (but it
-                                    # can be a descendant), e.g.:
-                                    #
-                                    # class A:
-                                    #   type genexpr[genexpr] = (x for x in [])
-                                    assert scope_name != 'genexpr' or '.0' not in c.varnames
                                     d[scope_name] = 1
                                     break
             self.__methods = tuple(d)
@@ -354,6 +351,10 @@ class Symbol:
         """
         return bool(self.__scope == FREE)
 
+    def is_cell(self):
+        """Return *True* if the symbol is a cell variable."""
+        return bool(self.__scope == CELL)
+
     def is_free_class(self):
         """Return *True* if a class-scoped symbol is free from
         the perspective of a method."""
@@ -413,7 +414,7 @@ class Symbol:
 _flags = [('USE', USE)]
 _flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_'))
 _scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL')
-_scopes_value_to_name = {globals()[n]: n for n in _scopes_names}
+_scopes_value_to_name = frozendict({globals()[n]: n for n in _scopes_names})
 
 
 def main(args):

Lib/test/test_symtable.py

@@ -5,6 +5,7 @@ Test the API of the symtable module.
 import re
 import textwrap
 import symtable
+import warnings
 import unittest
 
 from test import support
@@ -254,6 +255,7 @@ class SymtableTest(unittest.TestCase):
         self.assertEqual(sorted(func.get_locals()), expected)
         self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"])
         self.assertEqual(self.internal.get_frees(), ("x",))
+        self.assertEqual(self.spam.get_cells(), ("some_var", "x",))
 
     def test_globals(self):
         self.assertTrue(self.spam.lookup("glob").is_global())
@@ -283,6 +285,9 @@ class SymtableTest(unittest.TestCase):
     def test_free(self):
         self.assertTrue(self.internal.lookup("x").is_free())
 
+    def test_cells(self):
+        self.assertTrue(self.spam.lookup("x").is_cell())
+
     def test_referenced(self):
         self.assertTrue(self.internal.lookup("x").is_referenced())
         self.assertTrue(self.spam.lookup("internal").is_referenced())
@@ -527,6 +532,58 @@ class SymtableTest(unittest.TestCase):
         expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>"
         self.assertEqual(repr(self.top._table), expected)
 
+    def test_lambda(self):
+        st = symtable.symtable("lambda x: x", "?", "exec")
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<lambda>")
+        self.assertFalse(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), ["x"])
+        self.assertEqual(st.get_children(), [])
+
+    def test_nested_lambda(self):
+        st = symtable.symtable("lambda x: lambda y=x: y", "?", "exec")
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<lambda>")
+        self.assertFalse(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), ["x"])
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<lambda>")
+        self.assertTrue(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), ["y"])
+        self.assertEqual(st.get_children(), [])
+
+    def test_genexpr(self):
+        st = symtable.symtable("(x for x in a)", "?", "exec")
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<genexpr>")
+        self.assertFalse(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), [".0", "x"])
+        self.assertEqual(st.get_children(), [])
+
+    def test_nested_genexpr(self):
+        st = symtable.symtable("((y for y in x) for x in a)", "?", "exec")
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<genexpr>")
+        self.assertFalse(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), [".0", "x"])
+        self.assertEqual(len(st.get_children()), 1)
+        st = st.get_children()[0]
+        self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION)
+        self.assertEqual(st.get_name(), "<genexpr>")
+        self.assertTrue(st.is_nested())
+        self.assertEqual(sorted(st.get_identifiers()), [".0", "y"])
+        self.assertEqual(st.get_children(), [])
+
     def test__symtable_refleak(self):
         # Regression test for reference leak in PyUnicode_FSDecoder.
         # See https://github.com/python/cpython/issues/139748.
@@ -534,6 +591,30 @@ class SymtableTest(unittest.TestCase):
         # check error path when 'compile_type' AC conversion failed
         self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1)
 
+    def test_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        with open(filename, 'rb') as f:
+            source = f.read()
+        module_re = r'test\.test_import\.data\.syntax_warnings\z'
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=module_re)
+            symtable.symtable(source, filename, 'exec')
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10])
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=r'package\.module\z')
+            warnings.filterwarnings('error', module=module_re)
+            symtable.symtable(source, filename, 'exec', module='package.module')
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10])
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
 
 class ComprehensionTests(unittest.TestCase):
     def get_identifiers_recursive(self, st, res):

crates/vm/src/stdlib/_symtable.rs

@@ -3,8 +3,8 @@ pub(crate) use _symtable::module_def;
 #[pymodule]
 mod _symtable {
     use crate::{
-        AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine,
-        builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyUtf8StrRef},
+        AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
+        builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyStrRef, PyUtf8StrRef},
         compiler,
         function::{ArgStrOrBytesLike, FsPath},
         types::Representable,
@@ -96,13 +96,26 @@ mod _symtable {
     #[pyattr]
     pub(super) const TYPE_TYPE_VARIABLE: i32 = 6;
 
-    #[pyfunction]
-    fn symtable(
+    #[derive(FromArgs)]
+    struct SymtableArgs {
+        #[pyarg(positional)]
         source: ArgStrOrBytesLike,
+        #[pyarg(positional)]
         filename: FsPath,
+        #[pyarg(positional)]
         startstr: PyUtf8StrRef,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyRef<PySymbolTable>> {
+        #[pyarg(named, optional)]
+        module: Option<PyObjectRef>,
+    }
+
+    #[pyfunction]
+    fn symtable(args: SymtableArgs, vm: &VirtualMachine) -> PyResult<PyRef<PySymbolTable>> {
+        let SymtableArgs {
+            source,
+            filename,
+            startstr,
+            module,
+        } = args;
         let mode = startstr
             .as_str()
             .parse::<compiler::Mode>()
@@ -128,10 +141,23 @@ mod _symtable {
                 "source code string cannot contain null bytes".into(),
             ));
         }
-        let symtable = compiler::compile_symtable(&source, mode, &filename).map_err(|err| {
-            let err = vm.new_syntax_error(&err, Some(&source));
-            set_syntax_error_filename(err, &filename_obj, vm)
-        })?;
+        let module = module.filter(|module| !vm.is_none(module));
+        let module = module
+            .map(|module| {
+                PyStrRef::try_from_object(vm, module.clone()).map_err(|_| {
+                    vm.new_type_error(format!(
+                        "symtable() argument 'module' must be str or None, not {}",
+                        module.class().name()
+                    ))
+                })
+            })
+            .transpose()?;
+        let symtable = vm
+            .compile_symtable_with_module(&source, mode, &filename, module.as_deref())
+            .map_err(|err| {
+                let err = err.into_pyexception(vm, Some(&source));
+                set_syntax_error_filename(err, &filename_obj, vm)
+            })?;
 
         Ok(to_py_symbol_table(symtable, vm))
     }
@@ -221,7 +247,13 @@ mod _symtable {
     impl Py<PySymbolTable> {
         #[pygetset]
         fn name(&self) -> String {
-            self.symtable.name.to_string()
+            match self.symtable.typ {
+                CompilerScope::Lambda => "<lambda>".to_owned(),
+                CompilerScope::Comprehension if self.symtable.name.as_str() == "genexpr" => {
+                    "<genexpr>".to_owned()
+                }
+                _ => self.symtable.name.to_string(),
+            }
         }
 
         #[pygetset(name = "type")]

crates/vm/src/vm/compile.rs

@@ -586,6 +586,30 @@ impl VirtualMachine {
         self.compile_with_opts_and_module(source, mode, source_path, opts, None, &[])
     }
 
+    pub(crate) fn compile_symtable_with_module(
+        &self,
+        source: &str,
+        mode: compiler::Mode,
+        filename: &str,
+        module: Option<&Py<PyStr>>,
+    ) -> Result<compiler::codegen::symboltable::SymbolTable, VmCompileError> {
+        #[cfg(feature = "parser")]
+        {
+            let start = match mode {
+                compiler::Mode::Exec | compiler::Mode::BlockExpr => CompileStart::File,
+                compiler::Mode::Eval => CompileStart::Eval,
+                compiler::Mode::Single => CompileStart::Single,
+            };
+            // A symbol table observes parser warnings, but does not compile
+            // bytecode or emit code-generation and AST-preprocessing warnings.
+            self.emit_ast_parser_warnings(source, filename, module, start, false)
+                .map_err(VmCompileError::Warning)?;
+        }
+        #[cfg(not(feature = "parser"))]
+        let _ = module;
+        compiler::compile_symtable(source, mode, filename).map_err(VmCompileError::Compile)
+    }
+
     pub(crate) fn compile_with_opts_and_module(
         &self,
         source: &str,