Synchronize developer tools with Python 3.15 (#9298)

Regenerate bdb, pdb, codeop, modulefinder and the canonical modulefinder
test using scripts/update_lib at CPython 3.15.0
d220409a13b77de7a6c30abe16ea90db01528edc.

AI-assisted source selection, synchronization, dependency review and
bounded validation using OpenAI Codex.

Assisted-by: Codex:unknown

Lib/bdb.py

@@ -269,12 +269,9 @@ class Bdb:
                   is entered.
             return: A function or other code block is about to return.
             exception: An exception has occurred.
-            c_call: A C function is about to be called.
-            c_return: A C function has returned.
-            c_exception: A C function has raised an exception.
 
-        For the Python events, specialized functions (see the dispatch_*()
-        methods) are called.  For the C events, no action is taken.
+        For all the events, specialized functions (see the dispatch_*()
+        methods) are called.
 
         The arg parameter depends on the previous event.
         """
@@ -290,12 +287,6 @@ class Bdb:
                 return self.dispatch_return(frame, arg)
             if event == 'exception':
                 return self.dispatch_exception(frame, arg)
-            if event == 'c_call':
-                return self.trace_dispatch
-            if event == 'c_exception':
-                return self.trace_dispatch
-            if event == 'c_return':
-                return self.trace_dispatch
             if event == 'opcode':
                 return self.dispatch_opcode(frame, arg)
             print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))

Lib/codeop.py

@@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
             try:
                 compiler(source + "\n", filename, symbol, flags=flags)
                 return None
-            except _IncompleteInputError as e:
+            except _IncompleteInputError:
                 return None
-            except SyntaxError as e:
+            except SyntaxError:
                 pass
                 # fallthrough
 

Lib/modulefinder.py

@@ -66,7 +66,12 @@ def _find_module(name, path=None):
 
     file_path = spec.origin
 
-    if spec.loader.is_package(name):
+    # On namespace packages, spec.loader might be None, but
+    # spec.submodule_search_locations should always be set — check it instead.
+    if isinstance(spec.submodule_search_locations, importlib.machinery.NamespacePath):
+        return None, spec.submodule_search_locations, ("", "", _PKG_DIRECTORY)
+
+    if spec.loader.is_package(name):  # non-namespace package
         return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY)
 
     if isinstance(spec.loader, importlib.machinery.SourceFileLoader):
@@ -334,7 +339,7 @@ class ModuleFinder:
             self.msgout(2, "load_module ->", m)
             return m
         if type == _PY_SOURCE:
-            co = compile(fp.read(), pathname, 'exec')
+            co = compile(fp.read(), pathname, 'exec', module=fqname)
         elif type == _PY_COMPILED:
             try:
                 data = fp.read()
@@ -400,7 +405,6 @@ class ModuleFinder:
                 yield "relative_import", (level, fromlist, name)
 
     def scan_code(self, co, m):
-        code = co.co_code
         scanner = self.scan_opcodes
         for what, args in scanner(co):
             if what == "store":
@@ -454,6 +458,11 @@ class ModuleFinder:
         if newname:
             fqname = newname
         m = self.add_module(fqname)
+
+        if isinstance(pathname, importlib.machinery.NamespacePath):
+            m.__path__ = pathname
+            return m
+
         m.__file__ = pathname
         m.__path__ = [pathname]
 

Lib/pdb.py

@@ -97,12 +97,16 @@ import linecache
 import selectors
 import threading
 import _colorize
-import _pyrepl.utils
 
 from contextlib import ExitStack, closing, contextmanager
 from types import CodeType
 from warnings import deprecated
 
+try:
+    import _pyrepl.utils
+except ModuleNotFoundError:
+    _pyrepl = None
+
 
 class Restart(Exception):
     """Causes a debugger to be restarted for the debugged python program."""
@@ -130,7 +134,7 @@ def find_first_executable_line(code):
     return code.co_firstlineno
 
 def find_function(funcname, filename):
-    cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname))
+    cre = re.compile(r'(?:async\s+)?def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname))
     try:
         fp = tokenize.open(filename)
     except OSError:
@@ -314,12 +318,34 @@ class _ZipTarget(_ExecutableTarget):
 
 
 class _PdbInteractiveConsole(code.InteractiveConsole):
-    def __init__(self, ns, message):
+    def __init__(self, ns=None, message=None):
         self._message = message
         super().__init__(locals=ns, local_exit=True)
 
     def write(self, data):
-        self._message(data, end='')
+        if self._message is not None:
+            self._message(data, end='')
+        else:
+            super().write(data)
+
+    def more_lines(self, text):
+        # Generic Python multi-line completeness heuristic.
+        # Strips pyrepl's trailing auto-indent before compiling.
+        # This should be functionally identical to simple_interact._more_lines
+        src = text.rstrip(" \t")
+        n = len(src)
+        if n > 0 and text[n-1] == '\n':
+            text = src
+        try:
+            code_obj = self.compile(text, "<stdin>", "single")
+        except (OverflowError, SyntaxError, ValueError):
+            lines = text.splitlines(keepends=True)
+            if len(lines) == 1:
+                return False
+            last = lines[-1]
+            return ((last.startswith((" ", "\t")) or last.strip() != "")
+                    and not last.endswith("\n"))
+        return code_obj is None
 
 
 # Interaction prompt line will separate file and call info from code
@@ -348,6 +374,117 @@ def get_default_backend():
     return _default_backend
 
 
+def _pyrepl_available():
+    """return whether pdb should use _pyrepl for input"""
+    if os.getenv("PYTHON_BASIC_REPL"):
+        CAN_USE_PYREPL = False
+    else:
+        try:
+            from _pyrepl.main import CAN_USE_PYREPL
+        except ModuleNotFoundError:
+            CAN_USE_PYREPL = False
+    return CAN_USE_PYREPL
+
+
+class PdbPyReplInput:
+    def __init__(self, pdb_instance, stdin, stdout, prompt):
+        import _pyrepl.readline
+
+        self.pdb_instance = pdb_instance
+        self.prompt = prompt
+        self.console = _PdbInteractiveConsole()
+        if not (os.isatty(stdin.fileno())):
+            raise ValueError("stdin is not a TTY")
+        self.readline_wrapper = _pyrepl.readline._ReadlineWrapper(
+            f_in=stdin.fileno(),
+            f_out=stdout.fileno(),
+            config=_pyrepl.readline.ReadlineConfig(
+                completer_delims=frozenset(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?')
+            )
+        )
+        self.readline_wrapper.get_reader().gen_colors = self.gen_colors
+
+    def readline(self):
+
+        def more_lines(text):
+            if text.strip() == "\x1a":
+                # Ctrl + Z raises EOFError to quit pdb
+                # This is similarly handled in simple_interact.py
+                raise EOFError
+            cmd, _, line = self.pdb_instance.parseline(text)
+            if not line or not cmd:
+                return False
+            func = getattr(self.pdb_instance, 'do_' + cmd, None)
+            if func is not None:
+                return False
+            return self.console.more_lines(text)
+
+        try:
+            pyrepl_completer = self.readline_wrapper.get_completer()
+            self.readline_wrapper.set_completer(self.complete)
+            multiline = (
+                self.readline_wrapper.multiline_input(
+                    more_lines,
+                    self.prompt,
+                    '... ' + ' ' * (len(self.prompt) - 4)
+                ) + '\n'
+            )
+            return multiline
+        except EOFError:
+            return 'EOF'
+        finally:
+            self.readline_wrapper.set_completer(pyrepl_completer)
+
+    def complete(self, text, state):
+        """
+        This function is very similar to cmd.Cmd.complete.
+        However, cmd.Cmd.complete assumes that we use readline module, but
+        pyrepl does not use it.
+        """
+        if state == 0:
+            origline = self.readline_wrapper.get_line_buffer()
+            line = origline.lstrip()
+            stripped = len(origline) - len(line)
+            begidx = self.readline_wrapper.get_begidx() - stripped
+            endidx = self.readline_wrapper.get_endidx() - stripped
+            if begidx > 0:
+                cmd, args, foo = self.pdb_instance.parseline(line)
+                if not cmd:
+                    compfunc = self.pdb_instance.completedefault
+                else:
+                    try:
+                        compfunc = getattr(self.pdb_instance, 'complete_' + cmd)
+                    except AttributeError:
+                        compfunc = self.pdb_instance.completedefault
+            else:
+                compfunc = self.pdb_instance.completenames
+            self.completion_matches = compfunc(text, line, begidx, endidx)
+        try:
+            return self.completion_matches[state]
+        except IndexError:
+            return None
+
+    def gen_colors(self, buffer):
+        from _pyrepl.utils import ColorSpan, Span
+
+        if not buffer.strip():
+            return
+
+        leading_spaces = len(buffer) - len(buffer.lstrip())
+        leading_text = buffer.split()[0]
+        if hasattr(self.pdb_instance, 'do_' + leading_text):
+            yield ColorSpan(
+                Span(leading_spaces, leading_spaces + len(leading_text) - 1),
+                "soft_keyword"
+            )
+            # Redact the command text with spaces so there will be no duplicated
+            # color span generated for it later.
+            redact_length = leading_spaces + len(leading_text)
+            buffer = ' ' * redact_length + buffer[redact_length:]
+
+        yield from _pyrepl.utils.gen_colors(buffer)
+
+
 class Pdb(bdb.Bdb, cmd.Cmd):
     _previous_sigint_handler = None
 
@@ -364,8 +501,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         bdb.Bdb.__init__(self, skip=skip, backend=backend if backend else get_default_backend())
         cmd.Cmd.__init__(self, completekey, stdin, stdout)
         sys.audit("pdb.Pdb")
-        if stdout:
-            self.use_rawinput = 0
+        if stdin is not None and stdin is not sys.stdin:
+            self.use_rawinput = False
         self.prompt = '(Pdb) '
         self.aliases = {}
         self.displaying = {}
@@ -382,6 +519,12 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         except ImportError:
             pass
 
+        self.pyrepl_input = None
+        if _pyrepl_available():
+            try:
+                self.pyrepl_input = PdbPyReplInput(self, self.stdin, self.stdout, self.prompt)
+            except Exception:
+                pass
         self.allow_kbdint = False
         self.nosigint = nosigint
         # Consider these characters as part of the command so when the users type
@@ -391,17 +534,22 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         # Read ~/.pdbrc and ./.pdbrc
         self.rcLines = []
         if readrc:
+            home_rcfile = os.path.expanduser("~/.pdbrc")
+            local_rcfile = os.path.abspath(".pdbrc")
+
             try:
-                with open(os.path.expanduser('~/.pdbrc'), encoding='utf-8') as rcFile:
-                    self.rcLines.extend(rcFile)
-            except OSError:
-                pass
-            try:
-                with open(".pdbrc", encoding='utf-8') as rcFile:
-                    self.rcLines.extend(rcFile)
+                with open(home_rcfile, encoding='utf-8') as rcfile:
+                    self.rcLines.extend(rcfile)
             except OSError:
                 pass
 
+            if local_rcfile != home_rcfile:
+                try:
+                    with open(local_rcfile, encoding='utf-8') as rcfile:
+                        self.rcLines.extend(rcfile)
+                except OSError:
+                    pass
+
         self.commands = {} # associates a command list to breakpoint numbers
         self.commands_defining = False # True while in the process of defining
                                        # a command list
@@ -416,6 +564,12 @@ class Pdb(bdb.Bdb, cmd.Cmd):
 
         self._current_task = None
 
+        self.lineno = None
+        self.stack = []
+        self.curindex = 0
+        self.curframe = None
+        self._user_requested_quit = False
+
     def set_trace(self, frame=None, *, commands=None):
         Pdb._last_pdb_instance = self
         if frame is None:
@@ -492,7 +646,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         self.lineno = None
         self.stack = []
         self.curindex = 0
-        if hasattr(self, 'curframe') and self.curframe:
+        if self.curframe:
             self.curframe.f_globals.pop('__pdb_convenience_variables', None)
         self.curframe = None
         self.tb_lineno.clear()
@@ -609,6 +763,31 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         self.message('%s%s' % (prefix, self._format_exc(exc_value)))
         self.interaction(frame, exc_traceback)
 
+    @contextmanager
+    def _replace_attribute(self, attrs):
+        original_attrs = {}
+        for attr, value in attrs.items():
+            original_attrs[attr] = getattr(self, attr)
+            setattr(self, attr, value)
+        try:
+            yield
+        finally:
+            for attr, value in original_attrs.items():
+                setattr(self, attr, value)
+
+    @contextmanager
+    def _maybe_use_pyrepl_as_stdin(self):
+        if self.pyrepl_input is None:
+            yield
+            return
+
+        with self._replace_attribute({
+            'stdin': self.pyrepl_input,
+            'use_rawinput': False,
+            'prompt': '',
+        }):
+            yield
+
     # General interaction function
     def _cmdloop(self):
         while True:
@@ -616,7 +795,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 # keyboard interrupts allow for an easy way to cancel
                 # the current command, so allow them during interactive input
                 self.allow_kbdint = True
-                self.cmdloop()
+                with self._maybe_use_pyrepl_as_stdin():
+                    self.cmdloop()
                 self.allow_kbdint = False
                 break
             except KeyboardInterrupt:
@@ -666,7 +846,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
 
     def _get_tb_and_exceptions(self, tb_or_exc):
         """
-        Given a tracecack or an exception, return a tuple of chained exceptions
+        Given a traceback or an exception, return a tuple of chained exceptions
         and current traceback to inspect.
 
         This will deal with selecting the right ``__cause__`` or ``__context__``
@@ -1086,7 +1266,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         return False
 
     def _colorize_code(self, code):
-        if self.colorize:
+        if self.colorize and _pyrepl:
             colors = list(_pyrepl.utils.gen_colors(code))
             chars, _ = _pyrepl.utils.disp_str(code, colors=colors, force_color=True)
             code = "".join(chars)
@@ -1309,7 +1489,14 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         reached.
         """
         if not arg:
-            bnum = len(bdb.Breakpoint.bpbynumber) - 1
+            for bp in reversed(bdb.Breakpoint.bpbynumber):
+                if bp is None:
+                    continue
+                bnum = bp.number
+                break
+            else:
+                self.error('cannot set commands: no existing breakpoint')
+                return
         else:
             try:
                 bnum = int(arg)
@@ -1513,7 +1700,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         """
         # this method should be callable before starting debugging, so default
         # to "no globals" if there is no current frame
-        frame = getattr(self, 'curframe', None)
+        frame = self.curframe
         if module_globals is None:
             module_globals = frame.f_globals if frame else None
         line = linecache.getline(filename, lineno, module_globals)
@@ -2248,9 +2435,13 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 s += '->'
             elif lineno == exc_lineno:
                 s += '>>'
+            # Strip the trailing newline before colorizing: the colorizer
+            # renders control characters (like '\n') in caret notation, so a
+            # later rstrip() could not remove the resulting '^J'.
+            line = line.rstrip()
             if self.colorize:
                 line = self._colorize_code(line)
-            self.message(s + '\t' + line.rstrip())
+            self.message(s + '\t' + line)
 
     def do_whatis(self, arg):
         """whatis expression
@@ -2342,10 +2533,21 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         contains all the (global and local) names found in the current scope.
         """
         ns = {**self.curframe.f_globals, **self.curframe.f_locals}
-        with self._enable_rlcompleter(ns):
-            console = _PdbInteractiveConsole(ns, message=self.message)
-            console.interact(banner="*pdb interact start*",
-                             exitmsg="*exit from pdb interact command*")
+        console = _PdbInteractiveConsole(ns, message=self.message)
+        banner = "*pdb interact start*"
+        exitmsg = "*exit from pdb interact command*"
+        if self.pyrepl_input is not None:
+            from _pyrepl.simple_interact import run_multiline_interactive_console
+            self.message(banner)
+            try:
+                run_multiline_interactive_console(console)
+            except SystemExit:
+                pass
+            self.message(exitmsg)
+        else:
+            with self._enable_rlcompleter(ns):
+                console.interact(banner=banner,
+                                 exitmsg=exitmsg)
 
     def do_alias(self, arg):
         """alias [name [command]]
@@ -2443,7 +2645,9 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         except KeyboardInterrupt:
             pass
 
-    def print_stack_entry(self, frame_lineno, prompt_prefix=line_prefix):
+    def print_stack_entry(self, frame_lineno, prompt_prefix=None):
+        if prompt_prefix is None:
+            prompt_prefix = line_prefix
         frame, lineno = frame_lineno
         if frame is self.curframe:
             prefix = '> '
@@ -2958,6 +3162,15 @@ class _PdbServer(Pdb):
         if self.quitting:
             self.detach()
 
+    @contextmanager
+    def _maybe_use_pyrepl_as_stdin(self):
+        # The server reads every command from the client over the socket, never
+        # from a local pyrepl. The base implementation swaps in pyrepl as stdin
+        # and blanks `self.prompt` to '' (pyrepl would draw the prompt itself),
+        # which here would transmit an empty prompt to the client whenever the
+        # target process happens to be pyrepl-capable. Keep the real prompt.
+        yield
+
     def detach(self):
         # Detach the debugger and close the socket without raising BdbQuit
         self.quitting = False
@@ -3428,8 +3641,7 @@ def _connect(
             f"\nLocal pdb module's protocol version: {attach_ver}"
         )
     else:
-        remote_pdb.rcLines.extend(commands.splitlines())
-        remote_pdb.set_trace(frame=frame)
+        remote_pdb.set_trace(frame=frame, commands=commands.splitlines())
 
 
 def attach(pid, commands=()):
@@ -3535,18 +3747,18 @@ def help():
     pydoc.pager(__doc__)
 
 _usage = """\
-Debug the Python program given by pyfile. Alternatively,
+Debug the Python program given by `pyfile`. Alternatively,
 an executable module or package to debug can be specified using
-the -m switch. You can also attach to a running Python process
-using the -p option with its PID.
+the `-m` switch. You can also attach to a running Python process
+using the `-p` option with its PID.
 
-Initial commands are read from .pdbrc files in your home directory
+Initial commands are read from `.pdbrc` files in your home directory
 and in the current directory, if they exist.  Commands supplied with
--c are executed after commands from .pdbrc files.
+`-c` are executed after commands from `.pdbrc` files.
 
-To let the script run until an exception occurs, use "-c continue".
+To let the script run until an exception occurs, use `-c continue`.
 To let the script run up to a given line X in the debugged file, use
-"-c 'until X'"."""
+`-c 'until X'`."""
 
 
 def exit_with_permission_help_text():
@@ -3579,13 +3791,12 @@ def parse_args():
         description=_usage,
         formatter_class=argparse.RawDescriptionHelpFormatter,
         allow_abbrev=False,
-        color=True,
     )
 
     # Get all the commands out first. For backwards compatibility, we allow
     # -c commands to be after the target.
     parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands',
-                        help='pdb commands to execute as if given in a .pdbrc file')
+                        help='pdb commands to execute as if given in a `.pdbrc` file')
 
     opts, args = parser.parse_known_args()
 
@@ -3630,7 +3841,13 @@ def main():
     if getattr(opts, 'pid', None) is not None:
         try:
             attach(opts.pid, opts.commands)
-        except PermissionError as e:
+        except RuntimeError:
+            print(
+                f"Cannot attach to pid {opts.pid}, please make sure that the process exists "
+                "and is using the same Python version."
+            )
+            sys.exit(1)
+        except PermissionError:
             exit_with_permission_help_text()
         return
     elif getattr(opts, 'module', None) is not None:

Lib/test/test_modulefinder.py

@@ -76,6 +76,18 @@ a/c.py
                                 from sys import version_info
 """]
 
+namespace_package_test = [
+    "module",
+    ["a", "module"],
+    ["a.c", "blahblah"], [],
+    """\
+module.py
+                                import a
+                                import a.c
+                                import blahblah
+a/b.py
+"""]
+
 absolute_import_test = [
     "a.module",
     ["a", "a.module",
@@ -353,6 +365,9 @@ class ModuleFinderTest(unittest.TestCase):
     def test_package(self):
         self._do_test(package_test)
 
+    def test_namespace_package(self):
+        self._do_test(namespace_package_test)
+
     def test_maybe(self):
         self._do_test(maybe_test)