Synchronize regression harness helpers with Python 3.15 (#9297)

Regenerate nine canonical modules using scripts/update_lib at CPython
3.15.0 d220409a13b77de7a6c30abe16ea90db01528edc. Keep the discovery
package split out until its missing test family is available.

AI-assisted source selection, synchronization, review and bounded
validation using OpenAI Codex. No native capability claims are added.

Assisted-by: Codex:unknown

Lib/test/bisect_cmd.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""
+Command line tool to bisect failing CPython tests.
+
+Find the test_os test method which alters the environment:
+
+    ./python -m test.bisect_cmd --fail-env-changed test_os
+
+Find a reference leak in "test_os", write the list of failing tests into the
+"bisect" file:
+
+    ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
+
+Load an existing list of tests from a file using -i option:
+
+    ./python -m test --list-cases -m FileTests test_os > tests
+    ./python -m test.bisect_cmd -i tests test_os
+"""
+
+import argparse
+import datetime
+import os.path
+import math
+import random
+import subprocess
+import sys
+import tempfile
+import time
+
+
+def write_tests(filename, tests):
+    with open(filename, "w") as fp:
+        for name in tests:
+            print(name, file=fp)
+        fp.flush()
+
+
+def write_output(filename, tests):
+    if not filename:
+        return
+    print("Writing %s tests into %s" % (len(tests), filename))
+    write_tests(filename, tests)
+    return filename
+
+
+def format_shell_args(args):
+    return ' '.join(args)
+
+
+def python_cmd():
+    cmd = [sys.executable]
+    cmd.extend(subprocess._args_from_interpreter_flags())
+    cmd.extend(subprocess._optim_args_from_interpreter_flags())
+    cmd.extend(('-X', 'faulthandler'))
+    return cmd
+
+
+def list_cases(args):
+    cmd = python_cmd()
+    cmd.extend(['-m', 'test', '--list-cases'])
+    cmd.extend(args.test_args)
+    proc = subprocess.run(cmd,
+                          stdout=subprocess.PIPE,
+                          universal_newlines=True)
+    exitcode = proc.returncode
+    if exitcode:
+        cmd = format_shell_args(cmd)
+        print("Failed to list tests: %s failed with exit code %s"
+              % (cmd, exitcode))
+        sys.exit(exitcode)
+    tests = proc.stdout.splitlines()
+    return tests
+
+
+def run_tests(args, tests, huntrleaks=None):
+    tmp = tempfile.mktemp()
+    try:
+        write_tests(tmp, tests)
+
+        cmd = python_cmd()
+        cmd.extend(['-u', '-m', 'test', '--matchfile', tmp])
+        cmd.extend(args.test_args)
+        print("+ %s" % format_shell_args(cmd))
+
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        proc = subprocess.run(cmd)
+        return proc.returncode
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-i', '--input',
+                        help='Test names produced by --list-tests written '
+                             'into a file. If not set, run --list-tests')
+    parser.add_argument('-o', '--output',
+                        help='Result of the bisection')
+    parser.add_argument('-n', '--max-tests', type=int, default=1,
+                        help='Maximum number of tests to stop the bisection '
+                             '(default: 1)')
+    parser.add_argument('-N', '--max-iter', type=int, default=100,
+                        help='Maximum number of bisection iterations '
+                             '(default: 100)')
+    # FIXME: document that following arguments are test arguments
+
+    args, test_args = parser.parse_known_args()
+    args.test_args = test_args
+    return args
+
+
+def main():
+    args = parse_args()
+    for opt in ('-w', '--rerun', '--verbose2'):
+        if opt in args.test_args:
+            print(f"WARNING: {opt} option should not be used to bisect!")
+            print()
+
+    if args.input:
+        with open(args.input) as fp:
+            tests = [line.strip() for line in fp]
+    else:
+        tests = list_cases(args)
+
+    print("Start bisection with %s tests" % len(tests))
+    print("Test arguments: %s" % format_shell_args(args.test_args))
+    print("Bisection will stop when getting %s or less tests "
+          "(-n/--max-tests option), or after %s iterations "
+          "(-N/--max-iter option)"
+          % (args.max_tests, args.max_iter))
+    output = write_output(args.output, tests)
+    print()
+
+    start_time = time.monotonic()
+    iteration = 1
+    try:
+        while len(tests) > args.max_tests and iteration <= args.max_iter:
+            ntest = len(tests)
+            ntest = max(ntest // 2, 1)
+            subtests = random.sample(tests, ntest)
+
+            print(f"[+] Iteration {iteration}/{args.max_iter}: "
+                  f"run {len(subtests)} tests/{len(tests)}")
+            print()
+
+            exitcode = run_tests(args, subtests)
+
+            print("ran %s tests/%s" % (ntest, len(tests)))
+            print("exit", exitcode)
+            if exitcode:
+                print("Tests failed: continuing with this subtest")
+                tests = subtests
+                output = write_output(args.output, tests)
+            else:
+                print("Tests succeeded: skipping this subtest, trying a new subset")
+            print()
+            iteration += 1
+    except KeyboardInterrupt:
+        print()
+        print("Bisection interrupted!")
+        print()
+
+    print("Tests (%s):" % len(tests))
+    for test in tests:
+        print("* %s" % test)
+    print()
+
+    if output:
+        print("Output written into %s" % output)
+
+    dt = math.ceil(time.monotonic() - start_time)
+    if len(tests) <= args.max_tests:
+        print("Bisection completed in %s iterations and %s"
+              % (iteration, datetime.timedelta(seconds=dt)))
+    else:
+        print("Bisection failed after %s iterations and %s"
+              % (iteration, datetime.timedelta(seconds=dt)))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

Lib/test/cov.py

@@ -0,0 +1,48 @@
+"""A minimal hook for gathering line coverage of the standard library.
+
+Designed to be used with -Xpresite=test.cov:enable which means:
+
+* it can't import anything besides `sys` to avoid tainting gathered coverage
+* filenames are not normalized
+
+To get gathered coverage back, look for 'test.cov' in `sys.modules`
+instead of importing directly. That way you can determine if the module
+was already in use.
+
+If you need to disable the hook, call the `disable()` function.
+"""
+
+import sys
+
+mon = sys.monitoring
+
+FileName = str
+LineNo = int
+Location = tuple[FileName, LineNo]
+
+coverage: set[Location] = set()
+
+
+# `types` and `typing` aren't imported to avoid invalid coverage
+def add_line(
+    code: "types.CodeType",
+    lineno: int,
+) -> "typing.Literal[sys.monitoring.DISABLE]":
+    coverage.add((code.co_filename, lineno))
+    return mon.DISABLE
+
+
+def enable():
+    mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage")
+    mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line)
+    mon.set_events(mon.COVERAGE_ID, mon.events.LINE)
+
+
+def disable():
+    mon.set_events(mon.COVERAGE_ID, 0)
+    mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, None)
+    mon.free_tool_id(mon.COVERAGE_ID)
+
+
+if __name__ == "__main__":
+    enable()

Lib/test/libregrtest/refleak.py

@@ -93,6 +93,13 @@ def runtest_refleak(test_name, test_func,
         for obj in abc.__subclasses__() + [abc]:
             abcs[obj] = _get_dump(obj)[0]
 
+    # `ByteString` is not included in `collections.abc.__all__`
+    with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
+        ByteString = collections.abc.ByteString
+    # Mypy doesn't even think `ByteString` is a class, hence the `type: ignore`
+    for obj in ByteString.__subclasses__() + [ByteString]:  # type: ignore[attr-defined]
+        abcs[obj] = _get_dump(obj)[0]
+
     # bpo-31217: Integer pool to get a single integer object for the same
     # value. The pool is used to prevent false alarm when checking for memory
     # block leaks. Fill the pool with values in -1000..1000 which are the most
@@ -254,6 +261,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data):
 
     # Clear ABC registries, restoring previously saved ABC registries.
     abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
+    with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
+        abs_classes.append(collections.abc.ByteString)
     abs_classes = filter(isabstract, abs_classes)
     for abc in abs_classes:
         for obj in abc.__subclasses__() + [abc]:

Lib/test/libregrtest/run_workers.py

@@ -147,9 +147,12 @@ class WorkerThread(threading.Thread):
 
         use_killpg = USE_PROCESS_GROUP
         if use_killpg:
-            parent_sid = os.getsid(0)
-            sid = os.getsid(popen.pid)
-            use_killpg = (sid != parent_sid)
+            try:
+                use_killpg = (os.getsid(popen.pid) != os.getsid(0))
+            except PermissionError:
+                # On OpenBSD getsid() is only allowed for a process in the
+                # same session, so the failure means that it is not.
+                use_killpg = True
 
         if use_killpg:
             what = f"{self} process group"

Lib/test/libregrtest/runtests.py

@@ -159,7 +159,7 @@ class RunTests:
         if '-u' not in python_opts:
             cmd.append('-u')  # Unbuffered stdout and stderr
         if self.coverage:
-            cmd.append("-Xpresite=test.cov")
+            cmd.append("-Xpresite=test.cov:enable")
         return cmd
 
     def bisect_cmd_args(self) -> list[str]:

Lib/test/libregrtest/setup.py

@@ -50,6 +50,14 @@ def setup_process() -> None:
         for signum in signals:
             faulthandler.register(signum, chain=True, file=stderr_fd)
 
+    # Restore the default SIGINT handler if there is no Python-level
+    # handler.  Python inherits the SIG_IGN disposition when the test
+    # suite runs as a shell background job; then asyncio.Runner does not
+    # install its own handler and _thread.interrupt_main() is a no-op,
+    # which makes some tests in test_asyncio and test_threading hang.
+    if signal.getsignal(signal.SIGINT) in (signal.SIG_IGN, signal.SIG_DFL):
+        signal.signal(signal.SIGINT, signal.default_int_handler)
+
     adjust_rlimit_nofile()
 
     support.record_original_stdout(sys.stdout)

Lib/test/libregrtest/tsan.py

@@ -3,13 +3,12 @@
 
 TSAN_TESTS = [
     'test_asyncio',
-    # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909).
-    'test_capi.test_mem',
-    'test_capi.test_pyatomic',
+    'test_capi',
     'test_code',
     'test_ctypes',
-    # 'test_concurrent_futures',  # gh-130605: too many data races
+    'test_concurrent_futures',
     'test_enum',
+    'test_enumerate',
     'test_functools',
     'test_httpservers',
     'test_imaplib',
@@ -31,6 +30,7 @@ TSAN_TESTS = [
     'test_threadsignals',
     'test_weakref',
     'test_free_threading',
+    'test_embed',
 ]
 
 # Tests that should be run with `--parallel-threads=N` under TSAN. These tests

Lib/test/subprocessdata/fd_status.py

@@ -2,7 +2,7 @@
 file descriptors on stdout.
 
 Usage:
-fd_stats.py: check all file descriptors
+fd_status.py: check all file descriptors (up to 255)
 fd_status.py fd1 fd2 ...: check only specified file descriptors
 """
 
@@ -18,7 +18,7 @@ if __name__ == "__main__":
             _MAXFD = os.sysconf("SC_OPEN_MAX")
         except:
             _MAXFD = 256
-        test_fds = range(0, _MAXFD)
+        test_fds = range(0, min(_MAXFD, 256))
     else:
         test_fds = map(int, sys.argv[1:])
     for fd in test_fds:

Lib/test/support/strace_helper.py

@@ -74,7 +74,7 @@ class StraceResult:
 def _filter_memory_call(call):
     # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory.
     # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias.
-    if call.syscall == "mmap" and "MAP_ANON" in call.args[3]:
+    if call.syscall in ("mmap", "mmap2") and "MAP_ANON" in call.args[3]:
         return True
 
     if call.syscall in ("munmap", "mprotect"):