From ccab27c12ab833b7bc451d2d233a754fa3fb0327 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Thu, 22 Oct 2026 02:36:51 -0700 Subject: [PATCH] Synchronize developer tools with Python 3.15 (#9298) Regenerate bdb, pdb, codeop, modulefinder and the canonical modulefinder test using scripts/update_lib at CPython 3.15.0 d220409a13b77de7a6c30abe16ea90db01528edc. AI-assisted source selection, synchronization, dependency review and bounded validation using OpenAI Codex. Assisted-by: Codex:unknown --- Lib/bdb.py | 13 +- Lib/codeop.py | 4 +- Lib/modulefinder.py | 15 +- Lib/pdb.py | 291 +++++++++++++++++++++++++++++----- Lib/test/test_modulefinder.py | 15 ++ 5 files changed, 285 insertions(+), 53 deletions(-) diff --git a/Lib/bdb.py b/Lib/bdb.py index 79da4bab9..50cf2b3f5 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -269,12 +269,9 @@ class Bdb: is entered. return: A function or other code block is about to return. exception: An exception has occurred. - c_call: A C function is about to be called. - c_return: A C function has returned. - c_exception: A C function has raised an exception. - For the Python events, specialized functions (see the dispatch_*() - methods) are called. For the C events, no action is taken. + For all the events, specialized functions (see the dispatch_*() + methods) are called. The arg parameter depends on the previous event. """ @@ -290,12 +287,6 @@ class Bdb: return self.dispatch_return(frame, arg) if event == 'exception': return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': - return self.trace_dispatch if event == 'opcode': return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) diff --git a/Lib/codeop.py b/Lib/codeop.py index 8cac00442..40e884231 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags): try: compiler(source + "\n", filename, symbol, flags=flags) return None - except _IncompleteInputError as e: + except _IncompleteInputError: return None - except SyntaxError as e: + except SyntaxError: pass # fallthrough diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index ac478ee7f..b8dccc2dd 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -66,7 +66,12 @@ def _find_module(name, path=None): file_path = spec.origin - if spec.loader.is_package(name): + # On namespace packages, spec.loader might be None, but + # spec.submodule_search_locations should always be set — check it instead. + if isinstance(spec.submodule_search_locations, importlib.machinery.NamespacePath): + return None, spec.submodule_search_locations, ("", "", _PKG_DIRECTORY) + + if spec.loader.is_package(name): # non-namespace package return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY) if isinstance(spec.loader, importlib.machinery.SourceFileLoader): @@ -334,7 +339,7 @@ class ModuleFinder: self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read(), pathname, 'exec') + co = compile(fp.read(), pathname, 'exec', module=fqname) elif type == _PY_COMPILED: try: data = fp.read() @@ -400,7 +405,6 @@ class ModuleFinder: yield "relative_import", (level, fromlist, name) def scan_code(self, co, m): - code = co.co_code scanner = self.scan_opcodes for what, args in scanner(co): if what == "store": @@ -454,6 +458,11 @@ class ModuleFinder: if newname: fqname = newname m = self.add_module(fqname) + + if isinstance(pathname, importlib.machinery.NamespacePath): + m.__path__ = pathname + return m + m.__file__ = pathname m.__path__ = [pathname] diff --git a/Lib/pdb.py b/Lib/pdb.py index 7dcd0cb1b..e0c9b4d2c 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -97,12 +97,16 @@ import linecache import selectors import threading import _colorize -import _pyrepl.utils from contextlib import ExitStack, closing, contextmanager from types import CodeType from warnings import deprecated +try: + import _pyrepl.utils +except ModuleNotFoundError: + _pyrepl = None + class Restart(Exception): """Causes a debugger to be restarted for the debugged python program.""" @@ -130,7 +134,7 @@ def find_first_executable_line(code): return code.co_firstlineno def find_function(funcname, filename): - cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) + cre = re.compile(r'(?:async\s+)?def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: @@ -314,12 +318,34 @@ class _ZipTarget(_ExecutableTarget): class _PdbInteractiveConsole(code.InteractiveConsole): - def __init__(self, ns, message): + def __init__(self, ns=None, message=None): self._message = message super().__init__(locals=ns, local_exit=True) def write(self, data): - self._message(data, end='') + if self._message is not None: + self._message(data, end='') + else: + super().write(data) + + def more_lines(self, text): + # Generic Python multi-line completeness heuristic. + # Strips pyrepl's trailing auto-indent before compiling. + # This should be functionally identical to simple_interact._more_lines + src = text.rstrip(" \t") + n = len(src) + if n > 0 and text[n-1] == '\n': + text = src + try: + code_obj = self.compile(text, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = text.splitlines(keepends=True) + if len(lines) == 1: + return False + last = lines[-1] + return ((last.startswith((" ", "\t")) or last.strip() != "") + and not last.endswith("\n")) + return code_obj is None # Interaction prompt line will separate file and call info from code @@ -348,6 +374,117 @@ def get_default_backend(): return _default_backend +def _pyrepl_available(): + """return whether pdb should use _pyrepl for input""" + if os.getenv("PYTHON_BASIC_REPL"): + CAN_USE_PYREPL = False + else: + try: + from _pyrepl.main import CAN_USE_PYREPL + except ModuleNotFoundError: + CAN_USE_PYREPL = False + return CAN_USE_PYREPL + + +class PdbPyReplInput: + def __init__(self, pdb_instance, stdin, stdout, prompt): + import _pyrepl.readline + + self.pdb_instance = pdb_instance + self.prompt = prompt + self.console = _PdbInteractiveConsole() + if not (os.isatty(stdin.fileno())): + raise ValueError("stdin is not a TTY") + self.readline_wrapper = _pyrepl.readline._ReadlineWrapper( + f_in=stdin.fileno(), + f_out=stdout.fileno(), + config=_pyrepl.readline.ReadlineConfig( + completer_delims=frozenset(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') + ) + ) + self.readline_wrapper.get_reader().gen_colors = self.gen_colors + + def readline(self): + + def more_lines(text): + if text.strip() == "\x1a": + # Ctrl + Z raises EOFError to quit pdb + # This is similarly handled in simple_interact.py + raise EOFError + cmd, _, line = self.pdb_instance.parseline(text) + if not line or not cmd: + return False + func = getattr(self.pdb_instance, 'do_' + cmd, None) + if func is not None: + return False + return self.console.more_lines(text) + + try: + pyrepl_completer = self.readline_wrapper.get_completer() + self.readline_wrapper.set_completer(self.complete) + multiline = ( + self.readline_wrapper.multiline_input( + more_lines, + self.prompt, + '... ' + ' ' * (len(self.prompt) - 4) + ) + '\n' + ) + return multiline + except EOFError: + return 'EOF' + finally: + self.readline_wrapper.set_completer(pyrepl_completer) + + def complete(self, text, state): + """ + This function is very similar to cmd.Cmd.complete. + However, cmd.Cmd.complete assumes that we use readline module, but + pyrepl does not use it. + """ + if state == 0: + origline = self.readline_wrapper.get_line_buffer() + line = origline.lstrip() + stripped = len(origline) - len(line) + begidx = self.readline_wrapper.get_begidx() - stripped + endidx = self.readline_wrapper.get_endidx() - stripped + if begidx > 0: + cmd, args, foo = self.pdb_instance.parseline(line) + if not cmd: + compfunc = self.pdb_instance.completedefault + else: + try: + compfunc = getattr(self.pdb_instance, 'complete_' + cmd) + except AttributeError: + compfunc = self.pdb_instance.completedefault + else: + compfunc = self.pdb_instance.completenames + self.completion_matches = compfunc(text, line, begidx, endidx) + try: + return self.completion_matches[state] + except IndexError: + return None + + def gen_colors(self, buffer): + from _pyrepl.utils import ColorSpan, Span + + if not buffer.strip(): + return + + leading_spaces = len(buffer) - len(buffer.lstrip()) + leading_text = buffer.split()[0] + if hasattr(self.pdb_instance, 'do_' + leading_text): + yield ColorSpan( + Span(leading_spaces, leading_spaces + len(leading_text) - 1), + "soft_keyword" + ) + # Redact the command text with spaces so there will be no duplicated + # color span generated for it later. + redact_length = leading_spaces + len(leading_text) + buffer = ' ' * redact_length + buffer[redact_length:] + + yield from _pyrepl.utils.gen_colors(buffer) + + class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None @@ -364,8 +501,8 @@ class Pdb(bdb.Bdb, cmd.Cmd): bdb.Bdb.__init__(self, skip=skip, backend=backend if backend else get_default_backend()) cmd.Cmd.__init__(self, completekey, stdin, stdout) sys.audit("pdb.Pdb") - if stdout: - self.use_rawinput = 0 + if stdin is not None and stdin is not sys.stdin: + self.use_rawinput = False self.prompt = '(Pdb) ' self.aliases = {} self.displaying = {} @@ -382,6 +519,12 @@ class Pdb(bdb.Bdb, cmd.Cmd): except ImportError: pass + self.pyrepl_input = None + if _pyrepl_available(): + try: + self.pyrepl_input = PdbPyReplInput(self, self.stdin, self.stdout, self.prompt) + except Exception: + pass self.allow_kbdint = False self.nosigint = nosigint # Consider these characters as part of the command so when the users type @@ -391,17 +534,22 @@ class Pdb(bdb.Bdb, cmd.Cmd): # Read ~/.pdbrc and ./.pdbrc self.rcLines = [] if readrc: + home_rcfile = os.path.expanduser("~/.pdbrc") + local_rcfile = os.path.abspath(".pdbrc") + try: - with open(os.path.expanduser('~/.pdbrc'), encoding='utf-8') as rcFile: - self.rcLines.extend(rcFile) - except OSError: - pass - try: - with open(".pdbrc", encoding='utf-8') as rcFile: - self.rcLines.extend(rcFile) + with open(home_rcfile, encoding='utf-8') as rcfile: + self.rcLines.extend(rcfile) except OSError: pass + if local_rcfile != home_rcfile: + try: + with open(local_rcfile, encoding='utf-8') as rcfile: + self.rcLines.extend(rcfile) + except OSError: + pass + self.commands = {} # associates a command list to breakpoint numbers self.commands_defining = False # True while in the process of defining # a command list @@ -416,6 +564,12 @@ class Pdb(bdb.Bdb, cmd.Cmd): self._current_task = None + self.lineno = None + self.stack = [] + self.curindex = 0 + self.curframe = None + self._user_requested_quit = False + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: @@ -492,7 +646,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): self.lineno = None self.stack = [] self.curindex = 0 - if hasattr(self, 'curframe') and self.curframe: + if self.curframe: self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -609,6 +763,31 @@ class Pdb(bdb.Bdb, cmd.Cmd): self.message('%s%s' % (prefix, self._format_exc(exc_value))) self.interaction(frame, exc_traceback) + @contextmanager + def _replace_attribute(self, attrs): + original_attrs = {} + for attr, value in attrs.items(): + original_attrs[attr] = getattr(self, attr) + setattr(self, attr, value) + try: + yield + finally: + for attr, value in original_attrs.items(): + setattr(self, attr, value) + + @contextmanager + def _maybe_use_pyrepl_as_stdin(self): + if self.pyrepl_input is None: + yield + return + + with self._replace_attribute({ + 'stdin': self.pyrepl_input, + 'use_rawinput': False, + 'prompt': '', + }): + yield + # General interaction function def _cmdloop(self): while True: @@ -616,7 +795,8 @@ class Pdb(bdb.Bdb, cmd.Cmd): # keyboard interrupts allow for an easy way to cancel # the current command, so allow them during interactive input self.allow_kbdint = True - self.cmdloop() + with self._maybe_use_pyrepl_as_stdin(): + self.cmdloop() self.allow_kbdint = False break except KeyboardInterrupt: @@ -666,7 +846,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): def _get_tb_and_exceptions(self, tb_or_exc): """ - Given a tracecack or an exception, return a tuple of chained exceptions + Given a traceback or an exception, return a tuple of chained exceptions and current traceback to inspect. This will deal with selecting the right ``__cause__`` or ``__context__`` @@ -1086,7 +1266,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): return False def _colorize_code(self, code): - if self.colorize: + if self.colorize and _pyrepl: colors = list(_pyrepl.utils.gen_colors(code)) chars, _ = _pyrepl.utils.disp_str(code, colors=colors, force_color=True) code = "".join(chars) @@ -1309,7 +1489,14 @@ class Pdb(bdb.Bdb, cmd.Cmd): reached. """ if not arg: - bnum = len(bdb.Breakpoint.bpbynumber) - 1 + for bp in reversed(bdb.Breakpoint.bpbynumber): + if bp is None: + continue + bnum = bp.number + break + else: + self.error('cannot set commands: no existing breakpoint') + return else: try: bnum = int(arg) @@ -1513,7 +1700,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame - frame = getattr(self, 'curframe', None) + frame = self.curframe if module_globals is None: module_globals = frame.f_globals if frame else None line = linecache.getline(filename, lineno, module_globals) @@ -2248,9 +2435,13 @@ class Pdb(bdb.Bdb, cmd.Cmd): s += '->' elif lineno == exc_lineno: s += '>>' + # Strip the trailing newline before colorizing: the colorizer + # renders control characters (like '\n') in caret notation, so a + # later rstrip() could not remove the resulting '^J'. + line = line.rstrip() if self.colorize: line = self._colorize_code(line) - self.message(s + '\t' + line.rstrip()) + self.message(s + '\t' + line) def do_whatis(self, arg): """whatis expression @@ -2342,10 +2533,21 @@ class Pdb(bdb.Bdb, cmd.Cmd): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe.f_locals} - with self._enable_rlcompleter(ns): - console = _PdbInteractiveConsole(ns, message=self.message) - console.interact(banner="*pdb interact start*", - exitmsg="*exit from pdb interact command*") + console = _PdbInteractiveConsole(ns, message=self.message) + banner = "*pdb interact start*" + exitmsg = "*exit from pdb interact command*" + if self.pyrepl_input is not None: + from _pyrepl.simple_interact import run_multiline_interactive_console + self.message(banner) + try: + run_multiline_interactive_console(console) + except SystemExit: + pass + self.message(exitmsg) + else: + with self._enable_rlcompleter(ns): + console.interact(banner=banner, + exitmsg=exitmsg) def do_alias(self, arg): """alias [name [command]] @@ -2443,7 +2645,9 @@ class Pdb(bdb.Bdb, cmd.Cmd): except KeyboardInterrupt: pass - def print_stack_entry(self, frame_lineno, prompt_prefix=line_prefix): + def print_stack_entry(self, frame_lineno, prompt_prefix=None): + if prompt_prefix is None: + prompt_prefix = line_prefix frame, lineno = frame_lineno if frame is self.curframe: prefix = '> ' @@ -2958,6 +3162,15 @@ class _PdbServer(Pdb): if self.quitting: self.detach() + @contextmanager + def _maybe_use_pyrepl_as_stdin(self): + # The server reads every command from the client over the socket, never + # from a local pyrepl. The base implementation swaps in pyrepl as stdin + # and blanks `self.prompt` to '' (pyrepl would draw the prompt itself), + # which here would transmit an empty prompt to the client whenever the + # target process happens to be pyrepl-capable. Keep the real prompt. + yield + def detach(self): # Detach the debugger and close the socket without raising BdbQuit self.quitting = False @@ -3428,8 +3641,7 @@ def _connect( f"\nLocal pdb module's protocol version: {attach_ver}" ) else: - remote_pdb.rcLines.extend(commands.splitlines()) - remote_pdb.set_trace(frame=frame) + remote_pdb.set_trace(frame=frame, commands=commands.splitlines()) def attach(pid, commands=()): @@ -3535,18 +3747,18 @@ def help(): pydoc.pager(__doc__) _usage = """\ -Debug the Python program given by pyfile. Alternatively, +Debug the Python program given by `pyfile`. Alternatively, an executable module or package to debug can be specified using -the -m switch. You can also attach to a running Python process -using the -p option with its PID. +the `-m` switch. You can also attach to a running Python process +using the `-p` option with its PID. -Initial commands are read from .pdbrc files in your home directory +Initial commands are read from `.pdbrc` files in your home directory and in the current directory, if they exist. Commands supplied with --c are executed after commands from .pdbrc files. +`-c` are executed after commands from `.pdbrc` files. -To let the script run until an exception occurs, use "-c continue". +To let the script run until an exception occurs, use `-c continue`. To let the script run up to a given line X in the debugged file, use -"-c 'until X'".""" +`-c 'until X'`.""" def exit_with_permission_help_text(): @@ -3579,13 +3791,12 @@ def parse_args(): description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False, - color=True, ) # Get all the commands out first. For backwards compatibility, we allow # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', - help='pdb commands to execute as if given in a .pdbrc file') + help='pdb commands to execute as if given in a `.pdbrc` file') opts, args = parser.parse_known_args() @@ -3630,7 +3841,13 @@ def main(): if getattr(opts, 'pid', None) is not None: try: attach(opts.pid, opts.commands) - except PermissionError as e: + except RuntimeError: + print( + f"Cannot attach to pid {opts.pid}, please make sure that the process exists " + "and is using the same Python version." + ) + sys.exit(1) + except PermissionError: exit_with_permission_help_text() return elif getattr(opts, 'module', None) is not None: diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index b64e684f8..703bfbf94 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -76,6 +76,18 @@ a/c.py from sys import version_info """] +namespace_package_test = [ + "module", + ["a", "module"], + ["a.c", "blahblah"], [], + """\ +module.py + import a + import a.c + import blahblah +a/b.py +"""] + absolute_import_test = [ "a.module", ["a", "a.module", @@ -353,6 +365,9 @@ class ModuleFinderTest(unittest.TestCase): def test_package(self): self._do_test(package_test) + def test_namespace_package(self): + self._do_test(namespace_package_test) + def test_maybe(self): self._do_test(maybe_test)