From 557995b3e565ce9379316ffe01624e25178e52a7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 25 Oct 2026 18:12:14 -0700 Subject: [PATCH] Sync canonical gated test sources and Lib metadata (#9369) Assisted-by: OpenAI Codex: model version unavailable Co-authored-by: Codex --- Lib/asyncio/staggered.py | 1 - Lib/site-packages/README.txt | 2 + Lib/test/_crossinterp_definitions.py | 120 + Lib/test/support/channels.py | 4 +- Lib/test/test_android.py | 19 +- Lib/test/test_ctypes/test_dlerror.py | 5 +- Lib/test/test_interpreters/__init__.py | 9 + Lib/test/test_interpreters/__main__.py | 4 + Lib/test/test_interpreters/test_api.py | 2489 ++++++++++++++++++ Lib/test/test_interpreters/test_channels.py | 610 +++++ Lib/test/test_interpreters/test_lifecycle.py | 194 ++ Lib/test/test_interpreters/test_queues.py | 667 +++++ Lib/test/test_interpreters/test_stress.py | 94 + Lib/test/test_interpreters/utils.py | 687 +++++ Lib/test/test_opcache.py | 463 +++- Lib/test/test_picklebuffer.py | 3 +- Lib/test/test_pickletools.py | 3 +- Lib/test/test_remote_pdb.py | 1754 ++++++++++++ Lib/test/test_type_cache.py | 24 +- Lib/test/test_urllib2net.py | 21 - Lib/test/test_uuid.py | 0 Lib/urllib/error.py | 2 +- Lib/wsgiref/handlers.py | 3 +- 23 files changed, 7090 insertions(+), 88 deletions(-) create mode 100644 Lib/site-packages/README.txt create mode 100644 Lib/test/_crossinterp_definitions.py create mode 100644 Lib/test/test_interpreters/__init__.py create mode 100644 Lib/test/test_interpreters/__main__.py create mode 100644 Lib/test/test_interpreters/test_api.py create mode 100644 Lib/test/test_interpreters/test_channels.py create mode 100644 Lib/test/test_interpreters/test_lifecycle.py create mode 100644 Lib/test/test_interpreters/test_queues.py create mode 100644 Lib/test/test_interpreters/test_stress.py create mode 100644 Lib/test/test_interpreters/utils.py create mode 100644 Lib/test/test_remote_pdb.py mode change 100644 => 100755 Lib/test/test_uuid.py diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 2ad65d864..845aed4c6 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -62,7 +62,6 @@ async def staggered_race(coro_fns, delay, *, loop=None): coroutine's entry is ``None``. """ - # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) diff --git a/Lib/site-packages/README.txt b/Lib/site-packages/README.txt new file mode 100644 index 000000000..273f6251a --- /dev/null +++ b/Lib/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/Lib/test/_crossinterp_definitions.py b/Lib/test/_crossinterp_definitions.py new file mode 100644 index 000000000..5c1344278 --- /dev/null +++ b/Lib/test/_crossinterp_definitions.py @@ -0,0 +1,120 @@ +# This may be loaded as a module, in the current __main__ module, +# or in another __main__ module. + + +####################################### +# functions and generators + +from test._code_definitions import * + + +####################################### +# classes + +class Spam: + # minimal + pass + + +class SpamOkay: + def okay(self): + return True + + +class SpamFull: + + a: object + b: object + c: object + + @staticmethod + def staticmeth(cls): + return True + + @classmethod + def classmeth(cls): + return True + + def __new__(cls, *args, **kwargs): + return super().__new__(cls) + + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + # __repr__ + # __str__ + # ... + + def __eq__(self, other): + if not isinstance(other, SpamFull): + return NotImplemented + return (self.a == other.a and + self.b == other.b and + self.c == other.c) + + @property + def prop(self): + return True + + +class SubSpamFull(SpamFull): + ... + + +class SubTuple(tuple): + ... + + +def class_eggs_inner(): + class EggsNested: + ... + return EggsNested +EggsNested = class_eggs_inner() + + +TOP_CLASSES = { + Spam: (), + SpamOkay: (), + SpamFull: (1, 2, 3), + SubSpamFull: (1, 2, 3), + SubTuple: ([1, 2, 3],), +} +CLASSES_WITHOUT_EQUALITY = [ + Spam, + SpamOkay, +] +BUILTIN_SUBCLASSES = [ + SubTuple, +] +NESTED_CLASSES = { + EggsNested: (), +} +CLASSES = { + **TOP_CLASSES, + **NESTED_CLASSES, +} + + +####################################### +# exceptions + +class MimimalError(Exception): + pass + + +class RichError(Exception): + def __init__(self, msg, value=None): + super().__init__(msg, value) + self.msg = msg + self.value = value + + def __eq__(self, other): + if not isinstance(other, RichError): + return NotImplemented + if self.msg != other.msg: + return False + if self.value != other.value: + return False + return True diff --git a/Lib/test/support/channels.py b/Lib/test/support/channels.py index 5352f7d4d..fab179765 100644 --- a/Lib/test/support/channels.py +++ b/Lib/test/support/channels.py @@ -6,8 +6,8 @@ from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( - ChannelError, ChannelNotFoundError, ChannelClosedError, - ChannelEmptyError, ChannelNotEmptyError, + ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 + ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 31daafbc3..edab0fe71 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -1,4 +1,5 @@ import io +import platform import queue import re import subprocess @@ -16,6 +17,11 @@ from unittest.mock import patch if sys.platform != "android": raise unittest.SkipTest("Android-specific") +# Some API levels filter out consecutive identical lines and replace them with a +# "chatty" marker. +api_level = platform.android_ver().api_level +chatty_issue = (26 <= api_level <= 30) + # (name, level, fileno) STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] @@ -174,15 +180,13 @@ class TestAndroidOutput(unittest.TestCase): write("\u0000b", [r"\xc0\x80b"]) write("a\u0000b", [r"a\xc0\x80b"]) - # Multi-line messages. Avoid identical consecutive lines, as - # they may activate "chatty" filtering and break the tests. - # - # Additional spaces will appear in the output where necessary to - # protect leading newlines. + # Multi-line messages. Additional spaces will appear in the output where + # necessary to protect leading newlines. write("\nx", [" "]) write("\na\n", ["x", "a"]) write("\n", [" "]) - write("\n\n", [" ", " "]) + if not chatty_issue: + write("\n\n", [" ", " "]) write("b\n", ["b"]) write("c\n\n", ["c", " "]) write("d\ne", ["d"]) @@ -201,7 +205,8 @@ class TestAndroidOutput(unittest.TestCase): write("\nx", [" ", "x"]) write("\na\n", [" ", "a"]) write("\n", [" "]) - write("\n\n", [" ", " "]) + if not chatty_issue: + write("\n\n", [" ", " "]) write("b\n", ["b"]) write("c\n\n", ["c", " "]) write("d\ne", ["d", "e"]) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 5658234f9..bb0aec0b3 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -30,8 +30,9 @@ void *foo(void) """ -@unittest.skipUnless(sys.platform.startswith('linux'), - 'test requires GNU IFUNC support') +@unittest.skipIf(not sys.platform.startswith('linux') + or platform.machine().startswith('parisc'), + 'test requires GNU IFUNC support') @unittest.skipIf(test.support.linked_to_musl(), "Requires glibc") class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py new file mode 100644 index 000000000..e3d189c4e --- /dev/null +++ b/Lib/test/test_interpreters/__init__.py @@ -0,0 +1,9 @@ +import os +from test.support import load_package_tests, Py_GIL_DISABLED +import unittest + +if Py_GIL_DISABLED: + raise unittest.SkipTest("GIL disabled") + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py new file mode 100644 index 000000000..40a23a297 --- /dev/null +++ b/Lib/test/test_interpreters/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py new file mode 100644 index 000000000..13d23af5a --- /dev/null +++ b/Lib/test/test_interpreters/test_api.py @@ -0,0 +1,2489 @@ +import contextlib +import os +import pickle +import signal +import sys +from textwrap import dedent +import threading +import types +import unittest + +from test import support +from test.support import os_helper +from test.support import script_helper +from test.support import import_helper +from test.support.script_helper import assert_python_ok, spawn_python +# Raise SkipTest if subinterpreters not supported. +_interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters +from test.support import Py_GIL_DISABLED +from test.support import force_not_colorized +import test._crossinterp_definitions as defs +from concurrent.interpreters import ( + InterpreterError, InterpreterNotFoundError, ExecutionFailed, +) +from .utils import ( + _captured_script, _run_output, _running, TestBase, + requires_test_modules, _testinternalcapi, +) + + +WHENCE_STR_UNKNOWN = 'unknown' +WHENCE_STR_RUNTIME = 'runtime init' +WHENCE_STR_LEGACY_CAPI = 'legacy C-API' +WHENCE_STR_CAPI = 'C-API' +WHENCE_STR_XI = 'cross-interpreter C-API' +WHENCE_STR_STDLIB = '_interpreters module' + + +def is_pickleable(obj): + try: + pickle.dumps(obj) + except Exception: + return False + return True + + +@contextlib.contextmanager +def defined_in___main__(name, script, *, remove=False): + import __main__ as mainmod + mainns = vars(mainmod) + assert name not in mainns + exec(script, mainns, mainns) + if remove: + yield mainns.pop(name) + else: + try: + yield mainns[name] + finally: + mainns.pop(name, None) + + +def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None): + if isinstance(exctype, type): + assert issubclass(exctype, BaseException), exctype + exctype = types.SimpleNamespace( + __name__=exctype.__name__, + __qualname__=exctype.__qualname__, + __module__=exctype.__module__, + ) + elif isinstance(exctype, str): + module, _, name = exctype.rpartition(exctype) + if not module and name in __builtins__: + module = 'builtins' + exctype = types.SimpleNamespace( + __name__=name, + __qualname__=exctype, + __module__=module or None, + ) + else: + assert isinstance(exctype, types.SimpleNamespace) + assert msg is None or isinstance(msg, str), msg + assert formatted is None or isinstance(formatted, str), formatted + assert errdisplay is None or isinstance(errdisplay, str), errdisplay + return types.SimpleNamespace( + type=exctype, + msg=msg, + formatted=formatted, + errdisplay=errdisplay, + ) + + +class ModuleTests(TestBase): + + def test_queue_aliases(self): + first = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + second = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + self.assertEqual(second, first) + + +class CreateTests(TestBase): + + def test_in_main(self): + interp = interpreters.create() + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertIn(interp, interpreters.list_all()) + + # GH-126221: Passing an invalid Unicode character used to cause a SystemError + self.assertRaises(UnicodeEncodeError, _interpreters.create, '\udc80') + + def test_in_thread(self): + lock = threading.Lock() + interp = None + def f(): + nonlocal interp + interp = interpreters.create() + lock.acquire() + lock.release() + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(interp, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from concurrent import interpreters + interp = interpreters.create() + print(interp.id) + """)) + interp2 = interpreters.Interpreter(int(out)) + self.assertEqual(interpreters.list_all(), [main, interp, interp2]) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.close() + interp2.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) + + +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertEqual(current, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from concurrent import interpreters + cur = interpreters.get_current() + print(cur.id) + """)) + current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) + self.assertNotEqual(current, main) + + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from concurrent import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from concurrent import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = id(interp) + self.assertNotEqual(id1, id2) + + @requires_test_modules + def test_created_with_capi(self): + expected = _testinternalcapi.next_interpreter_id() + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print((interp.id, interp.whence)) + """) + interpid, whence = eval(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, WHENCE_STR_CAPI) + + +class ListAllTests(TestBase): + + def test_initial(self): + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) + + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.close() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + def test_created_with_capi(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected = [ + (mainid, WHENCE_STR_RUNTIME), + (interpid1, WHENCE_STR_STDLIB), + (interpid2, WHENCE_STR_STDLIB), + (interpid3, WHENCE_STR_STDLIB), + (interpid4, WHENCE_STR_CAPI), + (interpid5, WHENCE_STR_STDLIB), + ] + expected2 = expected[:-2] + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.create() + print( + [(i.id, i.whence) for i in interpreters.list_all()]) + """) + res = eval(text) + res2 = [(i.id, i.whence) for i in interpreters.list_all()] + self.assertEqual(res, expected) + self.assertEqual(res2, expected2) + + +class InterpreterObjectTests(TestBase): + + def test_init_int(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_init_interpreter_id(self): + interpid = interpreters.get_current()._id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp._id, interpid) + + def test_init_unsupported(self): + actualid = interpreters.get_current().id + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(main.id) + self.assertIs(interp, main) + + def test_init_does_not_exist(self): + with self.assertRaises(InterpreterNotFoundError): + interpreters.Interpreter(1_000_000) + + def test_init_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + + def test_id_type(self): + main = interpreters.get_main() + current = interpreters.get_current() + interp = interpreters.create() + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) + + def test_id_readonly(self): + interp = interpreters.create() + with self.assertRaises(AttributeError): + interp.id = 1_000_000 + + def test_whence(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertEqual(main.whence, WHENCE_STR_RUNTIME) + + with self.subTest('from _interpreters'): + self.assertEqual(interp.whence, WHENCE_STR_STDLIB) + + with self.subTest('from C-API'): + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(repr(interp.whence)) + """) + whence = eval(text) + self.assertEqual(whence, WHENCE_STR_CAPI) + + with self.subTest('readonly'): + for value in [ + None, + WHENCE_STR_UNKNOWN, + WHENCE_STR_RUNTIME, + WHENCE_STR_STDLIB, + WHENCE_STR_CAPI, + ]: + with self.assertRaises(AttributeError): + interp.whence = value + with self.assertRaises(AttributeError): + main.whence = value + + def test_hashable(self): + interp = interpreters.create() + expected = hash(interp.id) + actual = hash(interp) + self.assertEqual(actual, expected) + + def test_equality(self): + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(interp1, interp1) + self.assertNotEqual(interp1, interp2) + + def test_pickle(self): + interp = interpreters.create() + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(interp, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + + @support.requires_subprocess() + @force_not_colorized + def test_cleanup_in_repl(self): + # GH-135729: Using a subinterpreter in the REPL would lead to an unraisable + # exception during finalization + repl = script_helper.spawn_python("-i") + script = b"""if True: + from concurrent import interpreters + interpreters.create() + exit()""" + stdout, stderr = repl.communicate(script) + self.assertIsNone(stderr) + self.assertIn(b"Interpreter.close()", stdout) + self.assertNotIn(b"Traceback", stdout) + + @support.requires_subprocess() + @unittest.skipIf(os.name == 'nt', "signals don't work well on windows") + def test_keyboard_interrupt_in_thread_running_interp(self): + import subprocess + source = f"""if True: + from concurrent import interpreters + from threading import Thread + + def test(): + import time + print('a', flush=True, end='') + time.sleep(10) + + interp = interpreters.create() + interp.call_in_thread(test) + """ + + with spawn_python("-c", source, stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(1), b'a') + proc.send_signal(signal.SIGINT) + proc.stderr.flush() + error = proc.stderr.read() + self.assertIn(b"KeyboardInterrupt", error) + retcode = proc.wait() + # Sometimes we send the SIGINT after the subthread yields the GIL to + # the main thread, which results in the main thread getting the + # KeyboardInterrupt before finalization is reached. There's not + # any great way to protect against that, so we just allow a -2 + # return code as well. + self.assertIn(retcode, (0, -signal.SIGINT)) + + +class TestInterpreterIsRunning(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(main.is_running()) + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_finished(self): + r, w = self.pipe() + interp = interpreters.create() + interp.exec(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.is_running() + + def test_with_only_background_threads(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.exec('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.is_running()) + """) + def parse_results(text): + self.assertNotEqual(text, "") + try: + return eval(text) + except Exception: + raise Exception(repr(text)) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + text = self.run_from_capi(interpid, script, main=True) + running = parse_results(text) + self.assertTrue(running) + + with self.subTest('running, but not __main__ (from self)'): + text = self.run_temp_from_capi(script) + running = parse_results(text) + self.assertFalse(running) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=True): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertTrue(during) + self.assertFalse(after) + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=False): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertFalse(during) + self.assertFalse(after) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, _): + running = interp.is_running() + self.assertFalse(running) + + +class TestInterpreterClose(TestBase): + + def test_basic(self): + main = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2, interp3}) + interp2.close() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp3}) + + def test_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(set(interpreters.list_all()), before | interps) + for interp in interps: + interp.close() + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(InterpreterError): + main.close() + + def f(): + with self.assertRaises(InterpreterError): + main.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.close() + + def test_from_current(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + from concurrent import interpreters + interp = interpreters.Interpreter({interp.id}) + try: + interp.close() + except interpreters.InterpreterError: + print('failed') + """)) + self.assertEqual(out.strip(), 'failed') + self.assertEqual(set(interpreters.list_all()), {main, interp}) + + def test_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2}) + interp1.exec(dedent(f""" + from concurrent import interpreters + interp2 = interpreters.Interpreter({interp2.id}) + interp2.close() + interp3 = interpreters.create() + interp3.close() + """)) + self.assertEqual(set(interpreters.list_all()), {main, interp1}) + + def test_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(InterpreterError): + interp.close() + self.assertTrue(interp.is_running()) + + def test_subthreads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + interp.close() + """) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_from_capi(interpid, script, main=True) + + with self.subTest('running, but not __main__ (from self)'): + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_temp_from_capi(script) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=True): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + # The rest would be skipped until we deal with running threads when + # interp.close() is called. However, the "whence" restrictions + # trigger first. + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=False): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + self.assertTrue( + self.interp_exists(interpid)) + + + def test_remaining_threads(self): + r_interp, w_interp = self.pipe() + + FINISHED = b'F' + + # It's unlikely, but technically speaking, it's possible + # that the thread could've finished before interp.close() is + # reached, so this test might not properly exercise the case. + # However, it's quite unlikely and probably not worth bothering about. + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + def task(): + time.sleep(1) + os.write({w_interp}, {FINISHED!r}) + + threads = (threading.Thread(target=task) for _ in range(3)) + for t in threads: + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_remaining_daemon_threads(self): + # Daemon threads leak reference by nature, because they hang threads + # without allowing them to do cleanup (i.e., release refs). + # To prevent that from messing up the refleak hunter and whatnot, we + # run this in a subprocess. + code = '''if True: + import _interpreters + import types + interp = _interpreters.create( + types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=True, + gil='own', + ) + ) + _interpreters.exec(interp, f"""if True: + import threading + import time + + def task(): + time.sleep(3) + + threads = (threading.Thread(target=task, daemon=True) for _ in range(3)) + for t in threads: + t.start() + """) + _interpreters.destroy(interp) + ''' + assert_python_ok('-c', code) + + +class TestInterpreterPrepareMain(TestBase): + + def test_empty(self): + interp = interpreters.create() + with self.assertRaises(ValueError): + interp.prepare_main() + + def test_dict(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_tuple(self): + values = {'spam': 42, 'eggs': 'ham'} + values = tuple(values.items()) + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(**values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_dict_and_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values, foo='bar') + out = _run_output(interp, dedent(""" + print(spam, eggs, foo) + """)) + self.assertEqual(out.strip(), '42 ham bar') + + def test_not_shareable(self): + interp = interpreters.create() + with self.assertRaises(interpreters.NotShareableError): + interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) + + # Make sure neither was actually bound. + with self.assertRaises(ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(ExecutionFailed): + interp.exec('print(spam)') + + def test_running(self): + interp = interpreters.create() + interp.prepare_main({'spam': True}) + with self.running(interp): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.prepare_main({'spam': False}) + interp.exec('assert spam is True') + + @requires_test_modules + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.prepare_main({'spam': True}) + with self.assertRaisesRegex(ExecutionFailed, 'NameError'): + self.run_from_capi(interpid, 'spam') + + +class TestInterpreterExec(TestBase): + + def test_success(self): + interp = interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + interp.exec(script) + results = results.final() + results.raise_if_failed() + out = results.stdout + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.exec('raise Exception') + + @force_not_colorized + def test_display_preserved_exception(self): + tempdir = self.temp_dir() + modfile = self.make_module('spam', tempdir, text=""" + def ham(): + raise RuntimeError('uh-oh!') + + def eggs(): + ham() + """) + scriptfile = self.make_script('script.py', tempdir, text=""" + from concurrent import interpreters + + def script(): + import spam + spam.eggs() + + interp = interpreters.create() + try: + interp.exec(script) + finally: + interp.close() + """) + + stdout, stderr = self.assert_python_failure(scriptfile) + self.maxDiff = None + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec + self.assertEqual(stderr, dedent(f"""\ + Traceback (most recent call last): + File "{scriptfile}", line 10, in + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ + {interpmod_line.strip()} + raise ExecutionFailed(excinfo) + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! + + Uncaught in the interpreter: + + Traceback (most recent call last): + File "{scriptfile}", line 6, in script + spam.eggs() + ~~~~~~~~~^^ + File "{modfile}", line 6, in eggs + ham() + ~~~^^ + File "{modfile}", line 3, in ham + raise RuntimeError('uh-oh!') + RuntimeError: uh-oh! + """)) + self.assertEqual(stdout, '') + + def test_in_thread(self): + interp = interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + def f(): + interp.exec(script) + + t = threading.Thread(target=f) + t.start() + t.join() + results = results.final() + results.raise_if_failed() + out = results.stdout + + self.assertEqual(out, 'it worked!') + + @support.requires_fork() + def test_fork(self): + interp = interpreters.create() + import tempfile + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w', encoding='utf-8') as out: + out.write('{expected}') + """) + interp.exec(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + # XXX Is this still true? + @unittest.skip('Fails on FreeBSD') + def test_already_running(self): + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.exec('print("spam")') + + def test_bad_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec(10) + + def test_bytes_for_script(self): + r, w = self.pipe() + RAN = b'R' + DONE = b'D' + interp = interpreters.create() + interp.exec(f"""if True: + import os + os.write({w}, {RAN!r}) + """) + os.write(w, DONE) + self.assertEqual(os.read(r, 1), RAN) + + def test_with_background_threads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.exec(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.exec('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, _): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.exec('raise Exception("it worked!")') + + def test_list_comprehension(self): + # gh-135450: List comprehensions caused an assertion failure + # in _PyCode_CheckNoExternalState() + import string + r_interp, w_interp = self.pipe() + + interp = interpreters.create() + interp.exec(f"""if True: + import os + comp = [str(i) for i in range(10)] + os.write({w_interp}, ''.join(comp).encode()) + """) + self.assertEqual(os.read(r_interp, 10).decode(), string.digits) + interp.close() + + + # test__interpreters covers the remaining + # Interpreter.exec() behavior. + + +call_func_noop = defs.spam_minimal +call_func_ident = defs.spam_returns_arg +call_func_failure = defs.spam_raises + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_stateless_func(): + return (lambda x: x) + + +def call_func_return_pickleable(): + return [1, 2, 3] + + +def call_func_return_unpickleable(): + x = 42 + return (lambda: x) + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +def call_func_exec_wrapper(script, ns): + res = exec(script, ns, ns) + return res, ns, id(ns) + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + @contextlib.contextmanager + def assert_fails(self, expected): + with self.assertRaises(ExecutionFailed) as cm: + yield cm + uncaught = cm.exception.excinfo + self.assertEqual(uncaught.type.__name__, expected.__name__) + + def assert_fails_not_shareable(self): + return self.assert_fails(interpreters.NotShareableError) + + def assert_code_equal(self, code1, code2): + if code1 == code2: + return + self.assertEqual(code1.co_name, code2.co_name) + self.assertEqual(code1.co_flags, code2.co_flags) + self.assertEqual(code1.co_consts, code2.co_consts) + self.assertEqual(code1.co_varnames, code2.co_varnames) + self.assertEqual(code1.co_cellvars, code2.co_cellvars) + self.assertEqual(code1.co_freevars, code2.co_freevars) + self.assertEqual(code1.co_names, code2.co_names) + self.assertEqual( + _testinternalcapi.get_code_var_counts(code1), + _testinternalcapi.get_code_var_counts(code2), + ) + self.assertEqual(code1.co_code, code2.co_code) + + def assert_funcs_equal(self, func1, func2): + if func1 == func2: + return + self.assertIs(type(func1), type(func2)) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + self.assertEqual(func1.__closure__, func2.__closure__) + self.assert_code_equal(func1.__code__, func2.__code__) + self.assertEqual( + _testinternalcapi.get_code_var_counts(func1), + _testinternalcapi.get_code_var_counts(func2), + ) + + def assert_exceptions_equal(self, exc1, exc2): + assert isinstance(exc1, Exception) + assert isinstance(exc2, Exception) + if exc1 == exc2: + return + self.assertIs(type(exc1), type(exc2)) + self.assertEqual(exc1.args, exc2.args) + + def test_stateless_funcs(self): + interp = interpreters.create() + + func = call_func_noop + with self.subTest('no args, no return'): + res = interp.call(func) + self.assertIsNone(res) + + func = call_func_return_shareable + with self.subTest('no args, returns shareable'): + res = interp.call(func) + self.assertEqual(res, (1, None)) + + func = call_func_return_stateless_func + expected = (lambda x: x) + with self.subTest('no args, returns stateless func'): + res = interp.call(func) + self.assert_funcs_equal(res, expected) + + func = call_func_return_pickleable + with self.subTest('no args, returns pickleable'): + res = interp.call(func) + self.assertEqual(res, [1, 2, 3]) + + func = call_func_return_unpickleable + with self.subTest('no args, returns unpickleable'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + def test_stateless_func_returns_arg(self): + interp = interpreters.create() + + for arg in [ + None, + 10, + 'spam!', + b'spam!', + (1, 2, 'spam!'), + memoryview(b'spam!'), + ]: + with self.subTest(f'shareable {arg!r}'): + assert _interpreters.is_shareable(arg) + res = interp.call(defs.spam_returns_arg, arg) + self.assertEqual(res, arg) + + for arg in defs.STATELESS_FUNCTIONS: + with self.subTest(f'stateless func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + + for arg in defs.TOP_FUNCTIONS: + if arg in defs.STATELESS_FUNCTIONS: + continue + with self.subTest(f'stateful func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + assert is_pickleable(arg) + + for arg in [ + Ellipsis, + NotImplemented, + object(), + 2**1000, + [1, 2, 3], + {'a': 1, 'b': 2}, + types.SimpleNamespace(x=42), + # builtin types + object, + type, + Exception, + ModuleNotFoundError, + # builtin exceptions + Exception('uh-oh!'), + ModuleNotFoundError('mymodule'), + # builtin fnctions + len, + sys.exit, + # user classes + *defs.TOP_CLASSES, + *(c(*a) for c, a in defs.TOP_CLASSES.items() + if c not in defs.CLASSES_WITHOUT_EQUALITY), + ]: + with self.subTest(f'pickleable {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + if type(arg) is object: + self.assertIs(type(res), object) + elif isinstance(arg, BaseException): + self.assert_exceptions_equal(res, arg) + else: + self.assertEqual(res, arg) + assert is_pickleable(arg) + + for arg in [ + types.MappingProxyType({}), + *(f for f in defs.NESTED_FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]: + with self.subTest(f'unpickleable {arg!r}'): + assert not _interpreters.is_shareable(arg) + assert not is_pickleable(arg) + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_full_args(self): + interp = interpreters.create() + expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8}) + func = defs.spam_full_args + res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8) + self.assertEqual(res, expected) + + def test_full_defaults(self): + # pickleable, but not stateless + interp = interpreters.create() + expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9}) + res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9) + self.assertEqual(res, expected) + + def test_modified_arg(self): + interp = interpreters.create() + script = dedent(""" + a = 7 + b = 2 + c = a ** b + """) + ns = {} + expected = {'a': 7, 'b': 2, 'c': 49} + res = interp.call(call_func_exec_wrapper, script, ns) + obj, resns, resid = res + del resns['__builtins__'] + self.assertIsNone(obj) + self.assertEqual(ns, {}) + self.assertEqual(resns, expected) + self.assertNotEqual(resid, id(ns)) + self.assertNotEqual(resid, id(resns)) + + def test_func_in___main___valid(self): + # pickleable, already there' + + with os_helper.temp_dir() as tempdir: + def new_mod(name, text): + script_helper.make_script(tempdir, name, dedent(text)) + + def run(text): + name = 'myscript' + text = dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + + """) + dedent(text) + filename = script_helper.make_script(tempdir, name, text) + res = script_helper.assert_python_ok(filename) + return res.out.decode('utf-8').strip() + + # no module indirection + with self.subTest('no indirection'): + text = run(f""" + from concurrent import interpreters + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = interp.call(spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, direct interp + new_mod('mymod', f""" + def run(interp, func): + return interp.call(func) + """) + with self.subTest('indirect as func, direct interp'): + text = run(f""" + from concurrent import interpreters + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = mymod.run(interp, spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, indirect interp + new_mod('mymod', f""" + from concurrent import interpreters + def run(func): + interp = interpreters.create() + return interp.call(func) + """) + with self.subTest('indirect as func, indirect interp'): + text = run(f""" + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + res = mymod.run(spam) + print(res) + """) + self.assertEqual(text, '') + + def test_func_in___main___invalid(self): + interp = interpreters.create() + + funcname = f'{__name__.replace(".", "_")}_spam_okay' + script = dedent(f""" + def {funcname}(): + # This a global var... + return __name__ + """) + + with self.subTest('pickleable, added dynamically'): + with defined_in___main__(funcname, script) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + with self.subTest('lying about __main__'): + with defined_in___main__(funcname, script, remove=True) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_func_in___main___hidden(self): + # When a top-level function that uses global variables is called + # through Interpreter.call(), it will be pickled, sent over, + # and unpickled. That requires that it be found in the other + # interpreter's __main__ module. However, the original script + # that defined the function is only run in the main interpreter, + # so pickle.loads() would normally fail. + # + # We work around this by running the script in the other + # interpreter. However, this is a one-off solution for the sake + # of unpickling, so we avoid modifying that interpreter's + # __main__ module by running the script in a hidden module. + # + # In this test we verify that the function runs with the hidden + # module as its __globals__ when called in the other interpreter, + # and that the interpreter's __main__ module is unaffected. + text = dedent(""" + eggs = True + + def spam(*, explicit=False): + if explicit: + import __main__ + ns = __main__.__dict__ + else: + # For now we have to have a LOAD_GLOBAL in the + # function in order for globals() to actually return + # spam.__globals__. Maybe it doesn't go through pickle? + # XXX We will fix this later. + spam + ns = globals() + + func = ns.get('spam') + return [ + id(ns), + ns.get('__name__'), + ns.get('__file__'), + id(func), + None if func is None else repr(func), + ns.get('eggs'), + ns.get('ham'), + ] + + if __name__ == "__main__": + from concurrent import interpreters + interp = interpreters.create() + + ham = True + print([ + [ + spam(explicit=True), + spam(), + ], + [ + interp.call(spam, explicit=True), + interp.call(spam), + ], + ]) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + local, remote = eval(stdout) + + # In the main interpreter. + main, unpickled = local + nsid, _, _, funcid, func, _, _ = main + self.assertEqual(main, [ + nsid, + '__main__', + filename, + funcid, + func, + True, + True, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^$') + self.assertEqual(unpickled, main) + + # In the subinterpreter. + main, unpickled = remote + nsid1, _, _, funcid1, _, _, _ = main + self.assertEqual(main, [ + nsid1, + '__main__', + None, + funcid1, + None, + None, + None, + ]) + nsid2, _, _, funcid2, func, _, _ = unpickled + self.assertEqual(unpickled, [ + nsid2, + '', + filename, + funcid2, + func, + True, + None, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^$') + self.assertNotEqual(nsid2, nsid1) + self.assertNotEqual(funcid2, funcid1) + + def test_func_in___main___uses_globals(self): + # See the note in test_func_in___main___hidden about pickle + # and the __main__ module. + # + # Additionally, the solution to that problem must provide + # for global variables on which a pickled function might rely. + # + # To check that, we run a script that has two global functions + # and a global variable in the __main__ module. One of the + # functions sets the global variable and the other returns + # the value. + # + # The script calls those functions multiple times in another + # interpreter, to verify the following: + # + # * the global variable is properly initialized + # * the global variable retains state between calls + # * the setter modifies that persistent variable + # * the getter uses the variable + # * the calls in the other interpreter do not modify + # the main interpreter + # * those calls don't modify the interpreter's __main__ module + # * the functions and variable do not actually show up in the + # other interpreter's __main__ module + text = dedent(""" + count = 0 + + def inc(x=1): + global count + count += x + + def get_count(): + return count + + if __name__ == "__main__": + counts = [] + results = [count, counts] + + from concurrent import interpreters + interp = interpreters.create() + + val = interp.call(get_count) + counts.append(val) + + interp.call(inc) + val = interp.call(get_count) + counts.append(val) + + interp.call(inc, 3) + val = interp.call(get_count) + counts.append(val) + + results.append(count) + + modified = {name: interp.call(eval, f'{name!r} in vars()') + for name in ('count', 'inc', 'get_count')} + results.append(modified) + + print(results) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + before, counts, after, modified = eval(stdout) + self.assertEqual(modified, { + 'count': False, + 'inc': False, + 'get_count': False, + }) + self.assertEqual(before, 0) + self.assertEqual(after, 0) + self.assertEqual(counts, [0, 1, 4]) + + def test_raises(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + with self.assert_fails(ValueError): + interp.call(call_func_complex, '???', exc=ValueError('spam')) + + def test_call_valid(self): + interp = interpreters.create() + + for i, (callable, args, kwargs, expected) in enumerate([ + (call_func_noop, (), {}, None), + (call_func_ident, ('spamspamspam',), {}, 'spamspamspam'), + (call_func_return_shareable, (), {}, (1, None)), + (call_func_return_pickleable, (), {}, [1, 2, 3]), + (Spam.noop, (), {}, None), + (Spam.from_values, (), {}, Spam(())), + (Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))), + (Spam, ('???',), {}, Spam('???')), + (Spam(101), (), {}, (101, (), {})), + (Spam(10101).run, (), {}, (10101, (), {})), + (call_func_complex, ('ident', 'spam'), {}, 'spam'), + (call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}, + ('spam', ('ham',), {'eggs': '!!!'})), + (call_func_complex, ('globals',), {}, __name__), + (call_func_complex, ('interpid',), {}, interp.id), + (call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable, *args, **kwargs) + self.assertEqual(res, expected) + + def test_call_invalid(self): + interp = interpreters.create() + + func = get_call_func_closure + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, 42) + + func = get_call_func_closure(42) + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + func = call_func_complex + op = 'closure' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, value='~~~') + + op = 'custom-inner' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, 'eggs!') + + def test_callable_requires_frame(self): + # There are various functions that require a current frame. + interp = interpreters.create() + for call, expected in [ + ((eval, '[1, 2, 3]'), + [1, 2, 3]), + ((eval, 'sum([1, 2, 3])'), + 6), + ((exec, '...'), + None), + ]: + with self.subTest(str(call)): + res = interp.call(*call) + self.assertEqual(res, expected) + + result_not_pickleable = [ + globals, + locals, + vars, + ] + for func, expectedtype in { + globals: dict, + locals: dict, + vars: dict, + dir: list, + }.items(): + with self.subTest(str(func)): + if func in result_not_pickleable: + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + else: + res = interp.call(func) + self.assertIsInstance(res, expectedtype) + self.assertIn('__builtins__', res) + + def test_globals_from_builtins(self): + # The builtins exec(), eval(), globals(), locals(), vars(), + # and dir() each runs relative to the target interpreter's + # __main__ module, when called directly. However, + # globals(), locals(), and vars() don't work when called + # directly so we don't check them. + from _frozen_importlib import BuiltinImporter + interp = interpreters.create() + + names = interp.call(dir) + self.assertEqual(names, [ + '__builtins__', + '__doc__', + '__loader__', + '__name__', + '__package__', + '__spec__', + ]) + + values = {name: interp.call(eval, name) + for name in names if name != '__builtins__'} + self.assertEqual(values, { + '__name__': '__main__', + '__doc__': None, + '__spec__': None, # It wasn't imported, so no module spec? + '__package__': None, + '__loader__': BuiltinImporter, + }) + with self.assertRaises(ExecutionFailed): + interp.call(eval, 'spam'), + + interp.call(exec, f'assert dir() == {names}') + + # Update the interpreter's __main__. + interp.prepare_main(spam=42) + expected = names + ['spam'] + + names = interp.call(dir) + self.assertEqual(names, expected) + + value = interp.call(eval, 'spam') + self.assertEqual(value, 42) + + interp.call(exec, f'assert dir() == {expected}, dir()') + + def test_globals_from_stateless_func(self): + # A stateless func, which doesn't depend on any globals, + # doesn't go through pickle, so it runs in __main__. + def set_global(name, value): + globals()[name] = value + + def get_global(name): + return globals().get(name) + + interp = interpreters.create() + + modname = interp.call(get_global, '__name__') + self.assertEqual(modname, '__main__') + + res = interp.call(get_global, 'spam') + self.assertIsNone(res) + + interp.exec('spam = True') + res = interp.call(get_global, 'spam') + self.assertTrue(res) + + interp.call(set_global, 'spam', 42) + res = interp.call(get_global, 'spam') + self.assertEqual(res, 42) + + interp.exec('assert spam == 42, repr(spam)') + + def test_call_in_thread(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_pickleable, (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (Spam.noop, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('custom', 'spam!'), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable, *args, **kwargs) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable, *args, **kwargs) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) + t.join() + self.assertIsNotNone(ctx.caught) + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + True, + False, + 100.0, + (), + (1, ('spam', 'eggs'), True), + ] + for obj in shareables: + with self.subTest(obj): + shareable = interpreters.is_shareable(obj) + self.assertTrue(shareable) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_main(self): + interpid, whence = _interpreters.get_main() + self.assertEqual(interpid, 0) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_RUNTIME) + + def test_get_current(self): + with self.subTest('main'): + main, *_ = _interpreters.get_main() + interpid, whence = _interpreters.get_current() + self.assertEqual(interpid, main) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + script = f""" + import _interpreters + interpid, whence = _interpreters.get_current() + print((interpid, whence)) + """ + def parse_stdout(text): + interpid, whence = eval(text) + return interpid, whence + + with self.subTest('from _interpreters'): + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, orig) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + with self.subTest('from C-API'): + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = last + 1 + text = self.run_temp_from_capi(script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + def test_list_all(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + expected = [ + (mainid, _interpreters.WHENCE_RUNTIME), + (interpid1, _interpreters.WHENCE_STDLIB), + (interpid2, _interpreters.WHENCE_STDLIB), + (interpid3, _interpreters.WHENCE_STDLIB), + ] + + with self.subTest('main'): + res = _interpreters.list_all() + self.assertEqual(res, expected) + + with self.subTest('via interp from _interpreters'): + text = self.run_and_capture(interpid2, f""" + import _interpreters + print( + _interpreters.list_all()) + """) + + res = eval(text) + self.assertEqual(res, expected) + + with self.subTest('via interp from C-API'): + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected2 = expected + [ + (interpid4, _interpreters.WHENCE_CAPI), + (interpid5, _interpreters.WHENCE_STDLIB), + ] + expected3 = expected + [ + (interpid5, _interpreters.WHENCE_STDLIB), + ] + text = self.run_temp_from_capi(f""" + import _interpreters + _interpreters.create() + print( + _interpreters.list_all()) + """) + res2 = eval(text) + res3 = _interpreters.list_all() + self.assertEqual(res2, expected2) + self.assertEqual(res3, expected3) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no args'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: \'empty\''): + with self.assertRaises(InterpreterError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.check_multi_interp_extensions = bool(Py_GIL_DISABLED) + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('whence'): + interpid = _interpreters.create() + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_STDLIB) + + @requires_test_modules + def test_destroy(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before = [id for id, *_ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, *_ in _interpreters.list_all()] + + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + self.assertFalse( + self.interp_exists(interpid)) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + with self.assertRaises(InterpreterError): + # It is the current interpreter. + _interpreters.destroy(interpid) + + with self.subTest('from C-API'): + interpid = _testinternalcapi.create_interpreter() + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.destroy(interpid, restrict=True) + self.assertTrue( + self.interp_exists(interpid)) + _interpreters.destroy(interpid) + self.assertFalse( + self.interp_exists(interpid)) + + with self.subTest('basic C-API'): + interpid = _testinternalcapi.create_interpreter() + self.assertTrue( + self.interp_exists(interpid)) + _testinternalcapi.destroy_interpreter(interpid, basic=True) + self.assertFalse( + self.interp_exists(interpid)) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('from C-API'): + orig = _interpreters.new_config('isolated') + with self.interpreter_from_capi(orig) as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.get_config(interpid, restrict=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_test_modules + def test_whence(self): + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + with self.subTest('stdlib'): + interpid = _interpreters.create() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + for orig, name in { + _interpreters.WHENCE_UNKNOWN: 'not ready', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + }.items(): + with self.subTest(f'from C-API ({orig}: {name})'): + with self.interpreter_from_capi(whence=orig) as interpid: + whence = _interpreters.whence(interpid) + self.assertEqual(whence, orig) + + with self.subTest('from C-API, running'): + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=True) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + with self.subTest('from legacy C-API, running'): + ... + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=False) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + + def test_contextvars_missing(self): + script = f""" + import contextvars + print(getattr(contextvars.Token, "MISSING", "'doesn't exist'")) + """ + + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + self.assertEqual(text.strip(), "") + + def test_is_running(self): + def check(interpid, expected): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.is_running(interpid, restrict=True) + running = _interpreters.is_running(interpid) + self.assertIs(running, expected) + + with self.subTest('from _interpreters (running)'): + interpid = _interpreters.create() + with self.running(interpid): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('from _interpreters (not running)'): + interpid = _interpreters.create() + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + check(interpid, True) + + with self.subTest('from C-API (running __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=True): + check(interpid, True) + + with self.subTest('from C-API (running, but not __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=False): + check(interpid, False) + + with self.subTest('from C-API (not running)'): + with self.interpreter_from_capi() as interpid: + check(interpid, False) + + def test_exec(self): + with self.subTest('run script'): + interpid = _interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + exc = _interpreters.exec(interpid, script) + results = results.final() + results.raise_if_failed() + out = results.stdout + self.assertEqual(out, 'it worked!') + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + script, results = _captured_script(""" + raise Exception('uh-oh!') + print("it worked!", end="") + """) + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + expected = build_excinfo( + Exception, 'uh-oh!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + ) + self.assertEqual(out, '') + self.assert_ns_equal(exc, expected) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.exec(interpid, 'raise Exception("it worked!")', + restrict=True) + exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') + self.assertIsNot(exc, None) + self.assertEqual(exc.msg, 'it worked!') + + def test_call(self): + interpid = _interpreters.create() + + # Here we focus on basic args and return values. + # See TestInterpreterCall for full operational coverage, + # including supported callables. + + with self.subTest('no args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('empty args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func, (), {}) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('no args, return non-None'): + func = defs.script_with_return + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIs(res, True) + + with self.subTest('full args, return non-None'): + expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0}) + func = defs.spam_full_args + args = (1, 2, 3, 4, 7, 8) + kwargs = dict(e=5, f=6, g=9, h=0) + res, exc = _interpreters.call(interpid, func, args, kwargs) + self.assertIsNone(exc) + self.assertEqual(res, expected) + + with self.subTest('uncaught exception'): + func = defs.spam_raises + res, exc = _interpreters.call(interpid, func) + expected = build_excinfo( + Exception, 'spam!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + ) + self.assertIsNone(res) + self.assertEqual(exc, expected) + + @requires_test_modules + def test_set___main___attrs(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + + with self.assertRaises(ValueError): + # GH-127165: Embedded NULL characters broke the lookup + _interpreters.set___main___attrs(interpid, {"\x00": 1}) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.set___main___attrs(interpid, {'spam': True}, + restrict=True) + _interpreters.set___main___attrs(interpid, {'spam': True}) + rc = _testinternalcapi.exec_interpreter( + interpid, + 'assert spam is True', + ) + self.assertEqual(rc, 0) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py new file mode 100644 index 000000000..5437792b5 --- /dev/null +++ b/Lib/test/test_interpreters/test_channels.py @@ -0,0 +1,610 @@ +import importlib +import pickle +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_channels = import_helper.import_module('_interpchannels') +from concurrent import interpreters +from test.support import channels +from .utils import _run_output, TestBase + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + # Additional tests are found in Lib/test/test__interpchannels.py. + # XXX Those should be either moved to LowLevelTests or eliminated + # in favor of high-level tests in this file. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(channels) + + +class TestChannels(TestBase): + + def test_create(self): + r, s = channels.create() + self.assertIsInstance(r, channels.RecvChannel) + self.assertIsInstance(s, channels.SendChannel) + + def test_list_all(self): + self.assertEqual(channels.list_all(), []) + created = set() + for _ in range(3): + ch = channels.create() + created.add(ch) + after = set(channels.list_all()) + self.assertEqual(after, created) + + def test_list_all_closed(self): + created = [channels.create() for _ in range(3)] + rch, sch = created.pop(1) + rch.close() + self.assertEqual(set(channels.list_all()), set(created)) + + def test_shareable(self): + interp = interpreters.create() + rch, sch = channels.create() + + self.assertTrue( + interpreters.is_shareable(rch)) + self.assertTrue( + interpreters.is_shareable(sch)) + + sch.send_nowait(rch) + sch.send_nowait(sch) + rch2 = rch.recv() + sch2 = rch.recv() + + interp.prepare_main(rch=rch, sch=sch) + sch.send_nowait(rch) + sch.send_nowait(sch) + interp.exec(dedent(""" + rch2 = rch.recv() + sch2 = rch.recv() + assert rch2 == rch + assert sch2 == sch + + sch.send_nowait(rch2) + sch.send_nowait(sch2) + """)) + rch3 = rch.recv() + sch3 = rch.recv() + + self.assertEqual(rch2, rch) + self.assertEqual(sch2, sch) + self.assertEqual(rch3, rch) + self.assertEqual(sch3, sch) + + def test_is_closed(self): + rch, sch = channels.create() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + + +class TestRecvChannelAttrs(TestBase): + + def test_id_type(self): + rch, _ = channels.create() + self.assertIsInstance(rch.id, _channels.ChannelID) + + def test_custom_id(self): + rch = channels.RecvChannel(1) + self.assertEqual(rch.id, 1) + + with self.assertRaises(TypeError): + channels.RecvChannel('1') + + def test_id_readonly(self): + rch = channels.RecvChannel(1) + with self.assertRaises(AttributeError): + rch.id = 2 + + def test_equality(self): + ch1, _ = channels.create() + ch2, _ = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + def test_pickle(self): + ch, _ = channels.create() + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + + +class TestSendChannelAttrs(TestBase): + + def test_id_type(self): + _, sch = channels.create() + self.assertIsInstance(sch.id, _channels.ChannelID) + + def test_custom_id(self): + sch = channels.SendChannel(1) + self.assertEqual(sch.id, 1) + + with self.assertRaises(TypeError): + channels.SendChannel('1') + + def test_id_readonly(self): + sch = channels.SendChannel(1) + with self.assertRaises(AttributeError): + sch.id = 2 + + def test_equality(self): + _, ch1 = channels.create() + _, ch2 = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + def test_pickle(self): + _, ch = channels.create() + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + + +class TestSendRecv(TestBase): + + def test_send_recv_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from test.support import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_send_recv_different_threads(self): + r, s = channels.create() + + def f(): + while True: + try: + obj = r.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + s.send(orig) + obj = r.recv() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main_with_default(self): + r, _ = channels.create() + obj = r.recv_nowait(None) + + self.assertIsNone(obj) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from test.support import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + assert obj == orig, 'expected: obj == orig' + # When going back to the same interpreter we get the same object. + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_nowait_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv_nowait() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv_nowait() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_recv_timeout(self): + r, _ = channels.create() + with self.assertRaises(TimeoutError): + r.recv(timeout=1) + + def test_recv_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv() + + def test_send_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send(b'spam') + + def test_recv_nowait_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv_nowait() + + def test_send_nowait_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send_nowait(b'spam') + + def test_recv_nowait_empty(self): + ch, _ = channels.create() + with self.assertRaises(channels.ChannelEmptyError): + ch.recv_nowait() + + def test_recv_nowait_default(self): + default = object() + rch, sch = channels.create() + obj1 = rch.recv_nowait(default) + sch.send_nowait(None) + sch.send_nowait(1) + sch.send_nowait(b'spam') + sch.send_nowait(b'eggs') + obj2 = rch.recv_nowait(default) + obj3 = rch.recv_nowait(default) + obj4 = rch.recv_nowait() + obj5 = rch.recv_nowait(default) + obj6 = rch.recv_nowait(default) + + self.assertIs(obj1, default) + self.assertIs(obj2, None) + self.assertEqual(obj3, 1) + self.assertEqual(obj4, b'spam') + self.assertEqual(obj5, b'eggs') + self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = channels.create() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = channels.create() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_cleared_with_subinterpreter(self): + def common(rch, sch, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is channels.UNBOUND: + extraargs = ', unbounditems=channels.UNBOUND' + elif unbound is channels.UNBOUND_ERROR: + extraargs = ', unbounditems=channels.UNBOUND_ERROR' + elif unbound is channels.UNBOUND_REMOVE: + extraargs = ', unbounditems=channels.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" + from test.support import channels + sch = channels.SendChannel({sch.id}) + obj1 = b'spam' + obj2 = b'eggs' + sch.send_nowait(obj1{extraargs}) + sch.send_nowait(obj2{extraargs}) + """)) + self.assertEqual( + _channels.get_count(rch.id), + presize + 2, + ) + + if presize == 0: + obj1 = rch.recv() + self.assertEqual(obj1, b'spam') + self.assertEqual( + _channels.get_count(rch.id), + presize + 1, + ) + + return interp + + with self.subTest('default'): # UNBOUND + rch, sch = channels.create() + interp = common(rch, sch) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_ERROR'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND_ERROR) + + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_REMOVE'): + rch, sch = channels.create() + + interp = common(rch, sch, channels.UNBOUND_REMOVE) + del interp + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + sch.send_nowait(b'ham', unbounditems=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 1) + interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) + self.assertEqual(_channels.get_count(rch.id), 3) + sch.send_nowait(42, unbounditems=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 4) + del interp + self.assertEqual(_channels.get_count(rch.id), 2) + obj1 = rch.recv() + obj2 = rch.recv() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + def test_send_cleared_with_subinterpreter_mixed(self): + rch, sch = channels.create() + interp = interpreters.create() + + # If we don't associate the main interpreter with the channel + # then the channel will be automatically closed when interp + # is destroyed. + sch.send_nowait(None) + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + + _run_output(interp, dedent(f""" + from test.support import channels + sch = channels.SendChannel({sch.id}) + sch.send_nowait(1, unbounditems=channels.UNBOUND) + sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) + sch.send_nowait(3) + sch.send_nowait(4, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbounditems=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 5) + + del interp + self.assertEqual(_channels.get_count(rch.id), 4) + + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 3) + + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + obj2 = rch.recv() + self.assertIs(obj2, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + obj3 = rch.recv() + self.assertIs(obj3, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + + def test_send_cleared_with_subinterpreter_multiple(self): + rch, sch = channels.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + sch.send_nowait(1) + _run_output(interp1, dedent(f""" + from test.support import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj1 = rch.recv() + sch.send_nowait(2, unbounditems=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj2 = rch.recv() + obj1 = rch.recv() + """)) + self.assertEqual(_channels.get_count(rch.id), 0) + sch.send_nowait(3) + _run_output(interp1, dedent(""" + sch.send_nowait(4, unbounditems=channels.UNBOUND) + # interp closed here + sch.send_nowait(5, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbounditems=channels.UNBOUND) + """)) + _run_output(interp2, dedent(""" + sch.send_nowait(7, unbounditems=channels.UNBOUND_ERROR) + # interp closed here + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbounditems=channels.UNBOUND) + """)) + _run_output(interp1, dedent(""" + sch.send_nowait(9, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbounditems=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 10) + + obj3 = rch.recv() + self.assertEqual(obj3, 3) + self.assertEqual(_channels.get_count(rch.id), 9) + + obj4 = rch.recv() + self.assertEqual(obj4, 4) + self.assertEqual(_channels.get_count(rch.id), 8) + + del interp1 + self.assertEqual(_channels.get_count(rch.id), 6) + + # obj5 was removed + + obj6 = rch.recv() + self.assertIs(obj6, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 5) + + obj7 = rch.recv() + self.assertEqual(obj7, 7) + self.assertEqual(_channels.get_count(rch.id), 4) + + del interp2 + self.assertEqual(_channels.get_count(rch.id), 3) + + # obj1 + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + # obj2 was removed + + obj8 = rch.recv() + self.assertIs(obj8, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + # obj9 was removed + + obj10 = rch.recv() + self.assertIs(obj10, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py new file mode 100644 index 000000000..39c1441b6 --- /dev/null +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -0,0 +1,194 @@ +import contextlib +import json +import os +import os.path +import sys +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_interpreters') +from .utils import TestBase + + +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from concurrent import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.exec(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + interp.close() + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + +class FinalizationTests(TestBase): + + @support.requires_subprocess() + def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. + import subprocess + argv = [sys.executable, '-c', '''if True: + from concurrent import interpreters + interp = interpreters.create() + try: + raise Exception + finally: + interp.close() + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py new file mode 100644 index 000000000..77334aea3 --- /dev/null +++ b/Lib/test/test_interpreters/test_queues.py @@ -0,0 +1,667 @@ +import importlib +import pickle +import threading +from textwrap import dedent +import unittest + +from test.support import import_helper, Py_DEBUG +# Raise SkipTest if subinterpreters not supported. +_queues = import_helper.import_module('_interpqueues') +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp +from .utils import _run_output, TestBase as _TestBase + +HUGE_TIMEOUT = 3600 +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] + + +def get_num_queues(): + return len(_queues.list_all()) + + +class TestBase(_TestBase): + def tearDown(self): + for qid, _, _ in _queues.list_all(): + try: + _queues.destroy(qid) + except Exception: + pass + + +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(queues) + + def test_create_destroy(self): + qid = _queues.create(2, REPLACE, -1) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + with self.assertRaises(queues.QueueNotFoundError): + _queues.get(qid) + with self.assertRaises(queues.QueueNotFoundError): + _queues.destroy(qid) + + def test_not_destroyed(self): + # It should have cleaned up any remaining queues. + stdout, stderr = self.assert_python_ok( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, {REPLACE}, -1) + """), + ) + self.assertEqual(stdout, '') + if Py_DEBUG: + self.assertNotEqual(stderr, '') + else: + self.assertEqual(stderr, '') + + def test_bind_release(self): + with self.subTest('typical'): + qid = _queues.create(2, REPLACE, -1) + _queues.bind(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('bind too much'): + qid = _queues.create(2, REPLACE, -1) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('nested'): + qid = _queues.create(2, REPLACE, -1) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('release without binding'): + qid = _queues.create(2, REPLACE, -1) + with self.assertRaises(queues.QueueError): + _queues.release(qid) + + +class QueueTests(TestBase): + + def test_create(self): + with self.subTest('vanilla'): + queue = queues.create() + self.assertEqual(queue.maxsize, 0) + + with self.subTest('small maxsize'): + queue = queues.create(3) + self.assertEqual(queue.maxsize, 3) + + with self.subTest('big maxsize'): + queue = queues.create(100) + self.assertEqual(queue.maxsize, 100) + + with self.subTest('no maxsize'): + queue = queues.create(0) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('negative maxsize'): + queue = queues.create(-10) + self.assertEqual(queue.maxsize, -10) + + with self.subTest('bad maxsize'): + with self.assertRaises(TypeError): + queues.create('1') + + def test_shareable(self): + queue1 = queues.create() + + interp = interpreters.create() + interp.exec(dedent(f""" + from concurrent.interpreters import _queues as queues + queue1 = queues.Queue({queue1.id}) + """)); + + with self.subTest('same interpreter'): + queue2 = queues.create() + queue1.put(queue2) + queue3 = queue1.get() + self.assertIs(queue3, queue2) + + with self.subTest('from current interpreter'): + queue4 = queues.create() + queue1.put(queue4) + out = _run_output(interp, dedent(""" + queue4 = queue1.get() + print(queue4.id) + """)) + qid = int(out) + self.assertEqual(qid, queue4.id) + + with self.subTest('from subinterpreter'): + out = _run_output(interp, dedent(""" + queue5 = queues.create() + queue1.put(queue5) + print(queue5.id) + """)) + qid = int(out) + queue5 = queue1.get() + self.assertEqual(queue5.id, qid) + + def test_id_type(self): + queue = queues.create() + self.assertIsInstance(queue.id, int) + + def test_custom_id(self): + with self.assertRaises(queues.QueueNotFoundError): + queues.Queue(1_000_000) + + def test_id_readonly(self): + queue = queues.create() + with self.assertRaises(AttributeError): + queue.id = 1_000_000 + + def test_maxsize_readonly(self): + queue = queues.create(10) + with self.assertRaises(AttributeError): + queue.maxsize = 1_000_000 + + def test_hashable(self): + queue = queues.create() + expected = hash(queue.id) + actual = hash(queue) + self.assertEqual(actual, expected) + + def test_equality(self): + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(queue1, queue1) + self.assertNotEqual(queue1, queue2) + + def test_pickle(self): + queue = queues.create() + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(queue, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) + + +class TestQueueOps(TestBase): + + def test_empty(self): + queue = queues.create() + before = queue.empty() + queue.put(None) + during = queue.empty() + queue.get() + after = queue.empty() + + self.assertIs(before, True) + self.assertIs(during, False) + self.assertIs(after, True) + + def test_full(self): + for maxsize in [1, 3, 11]: + with self.subTest(f'maxsize={maxsize}'): + num_to_add = maxsize + expected = [False] * (num_to_add * 2 + 3) + expected[maxsize] = True + expected[maxsize + 1] = True + + queue = queues.create(maxsize) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) + + # no max size + for args in [(), (0,), (-1,), (-10,)]: + with self.subTest(f'maxsize={args[0]}' if args else ''): + num_to_add = 13 + expected = [False] * (num_to_add * 2 + 3) + + queue = queues.create(*args) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) + + def test_qsize(self): + expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] + actual = [] + queue = queues.create() + for _ in range(3): + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + for _ in range(3): + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + + self.assertEqual(actual, expected) + + def test_put_get_main(self): + expected = list(range(20)) + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + def test_put_timeout(self): + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + with self.assertRaises(queues.QueueFull): + queue.put(None, HUGE_TIMEOUT, 0.1) + queue.get() + queue.put(None) + + def test_put_nowait(self): + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, False) + with self.assertRaises(queues.QueueFull): + queue.put(None, False, timeout=HUGE_TIMEOUT) + queue.get() + queue.put_nowait(None) + + def test_put_full_fallback(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) + + def test_get_timeout(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0.1) + with self.assertRaises(queues.QueueEmpty): + queue.get(HUGE_TIMEOUT, 0.1) + + def test_get_nowait(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + with self.assertRaises(queues.QueueEmpty): + queue.get(False) + with self.assertRaises(queues.QueueEmpty): + queue.get(False, timeout=HUGE_TIMEOUT) + + def test_put_get_full_fallback(self): + expected = list(range(20)) + queue = queues.create() + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + + def test_put_get_same_interpreter(self): + interp = interpreters.create() + interp.exec(dedent(""" + from concurrent.interpreters import _queues as queues + queue = queues.create() + """)) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + interp.exec(dedent(f""" + orig = b'spam' + queue.put(orig) + obj = queue.{methname}() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + def test_put_get_different_interpreters(self): + interp = interpreters.create() + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(len(queues.list_all()), 2) + + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + obj1 = b'spam' + queue1.put(obj1) + + out = _run_output( + interp, + dedent(f""" + from concurrent.interpreters import _queues as queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' + obj = queue1.{methname}() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' + queue2.put(obj2) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + """)) + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + + get = getattr(queue2, methname) + obj2 = get() + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_put_cleared_with_subinterpreter(self): + def common(queue, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is queues.UNBOUND: + extraargs = ', unbounditems=queues.UNBOUND' + elif unbound is queues.UNBOUND_ERROR: + extraargs = ', unbounditems=queues.UNBOUND_ERROR' + elif unbound is queues.UNBOUND_REMOVE: + extraargs = ', unbounditems=queues.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" + from concurrent.interpreters import _queues as queues + queue = queues.Queue({queue.id}) + obj1 = b'spam' + obj2 = b'eggs' + queue.put(obj1{extraargs}) + queue.put(obj2{extraargs}) + """)) + self.assertEqual(queue.qsize(), presize + 2) + + if presize == 0: + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), presize + 1) + + return interp + + with self.subTest('default'): # UNBOUND + queue = queues.create() + interp = common(queue) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND'): + queue = queues.create() + interp = common(queue, queues.UNBOUND) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_ERROR'): + queue = queues.create() + interp = common(queue, queues.UNBOUND_ERROR) + + del interp + self.assertEqual(queue.qsize(), 1) + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_REMOVE'): + queue = queues.create() + + interp = common(queue, queues.UNBOUND_REMOVE) + del interp + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + queue.put(b'ham', unbounditems=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 1) + interp = common(queue, queues.UNBOUND_REMOVE, 1) + self.assertEqual(queue.qsize(), 3) + queue.put(42, unbounditems=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 4) + del interp + self.assertEqual(queue.qsize(), 2) + obj1 = queue.get() + obj2 = queue.get() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_cleared_with_subinterpreter_mixed(self): + queue = queues.create() + interp = interpreters.create() + _run_output(interp, dedent(f""" + from concurrent.interpreters import _queues as queues + queue = queues.Queue({queue.id}) + queue.put(1, unbounditems=queues.UNBOUND) + queue.put(2, unbounditems=queues.UNBOUND_ERROR) + queue.put(3) + queue.put(4, unbounditems=queues.UNBOUND_REMOVE) + queue.put(5, unbounditems=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 5) + + del interp + self.assertEqual(queue.qsize(), 4) + + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 3) + + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + obj2 = queue.get() + self.assertIs(obj2, queues.UNBOUND) + self.assertEqual(queue.qsize(), 1) + + obj3 = queue.get() + self.assertIs(obj3, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + + def test_put_cleared_with_subinterpreter_multiple(self): + queue = queues.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + queue.put(1) + _run_output(interp1, dedent(f""" + from concurrent.interpreters import _queues as queues + queue = queues.Queue({queue.id}) + obj1 = queue.get() + queue.put(2, unbounditems=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from concurrent.interpreters import _queues as queues + queue = queues.Queue({queue.id}) + obj2 = queue.get() + obj1 = queue.get() + """)) + self.assertEqual(queue.qsize(), 0) + queue.put(3) + _run_output(interp1, dedent(""" + queue.put(4, unbounditems=queues.UNBOUND) + # interp closed here + queue.put(5, unbounditems=queues.UNBOUND_REMOVE) + queue.put(6, unbounditems=queues.UNBOUND) + """)) + _run_output(interp2, dedent(""" + queue.put(7, unbounditems=queues.UNBOUND_ERROR) + # interp closed here + queue.put(obj1, unbounditems=queues.UNBOUND_ERROR) + queue.put(obj2, unbounditems=queues.UNBOUND_REMOVE) + queue.put(8, unbounditems=queues.UNBOUND) + """)) + _run_output(interp1, dedent(""" + queue.put(9, unbounditems=queues.UNBOUND_REMOVE) + queue.put(10, unbounditems=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 10) + + obj3 = queue.get() + self.assertEqual(obj3, 3) + self.assertEqual(queue.qsize(), 9) + + obj4 = queue.get() + self.assertEqual(obj4, 4) + self.assertEqual(queue.qsize(), 8) + + del interp1 + self.assertEqual(queue.qsize(), 6) + + # obj5 was removed + + obj6 = queue.get() + self.assertIs(obj6, queues.UNBOUND) + self.assertEqual(queue.qsize(), 5) + + obj7 = queue.get() + self.assertEqual(obj7, 7) + self.assertEqual(queue.qsize(), 4) + + del interp2 + self.assertEqual(queue.qsize(), 3) + + # obj1 + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + # obj2 was removed + + obj8 = queue.get() + self.assertIs(obj8, queues.UNBOUND) + self.assertEqual(queue.qsize(), 1) + + # obj9 was removed + + obj10 = queue.get() + self.assertIs(obj10, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + + def test_put_get_different_threads(self): + queue1 = queues.create() + queue2 = queues.create() + + def f(): + while True: + try: + obj = queue1.get(timeout=0.1) + break + except queues.QueueEmpty: + continue + queue2.put(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + queue1.put(orig) + obj = queue2.get() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py new file mode 100644 index 000000000..50d2444a4 --- /dev/null +++ b/Lib/test/test_interpreters/test_stress.py @@ -0,0 +1,94 @@ +import threading +import unittest + +from test import support +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +_interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters +from concurrent.interpreters import InterpreterError +from .utils import TestBase + + +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + @support.requires_resource('cpu') + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + del alive + support.gc_collect() + + @threading_helper.requires_working_threading() + @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False) + def test_create_many_threaded(self, size): + alive = [] + start = threading.Event() + def task(): + # try to create all interpreters simultaneously + if not start.wait(support.SHORT_TIMEOUT): + raise TimeoutError + interp = interpreters.create() + alive.append(interp) + threads = [threading.Thread(target=task) for _ in range(size)] + with threading_helper.start_threads(threads): + start.set() + del alive + support.gc_collect() + + @threading_helper.requires_working_threading() + @support.bigmemtest(size=200, memuse=34*2**20, dry_run=False) + def test_many_threads_running_interp_in_other_interp(self, size): + start = threading.Event() + interp = interpreters.create() + + script = f"""if True: + import _interpreters + _interpreters.run_string({interp.id}, '1') + """ + + def run(): + interp = interpreters.create() + alreadyrunning = (f'{interpreters.InterpreterError}: ' + 'interpreter already running') + # try to run all interpreters simultaneously + if not start.wait(support.SHORT_TIMEOUT): + raise TimeoutError + success = False + while not success: + try: + interp.exec(script) + except interpreters.ExecutionFailed as exc: + if exc.excinfo.msg != 'interpreter already running': + raise # re-raise + assert exc.excinfo.type.__name__ == 'InterpreterError' + else: + success = True + + threads = [threading.Thread(target=run) for _ in range(size)] + with threading_helper.start_threads(threads): + start.set() + support.gc_collect() + + @support.nomemtest + def test_create_interpreter_no_memory(self): + import _testcapi + + assertion = self.assertRaises(InterpreterError) + try: + _testcapi.set_nomemory(0, 1) + with assertion: + _interpreters.create() + finally: + _testcapi.remove_mem_hooks() + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py new file mode 100644 index 000000000..bb6da5272 --- /dev/null +++ b/Lib/test/test_interpreters/utils.py @@ -0,0 +1,687 @@ +from collections import namedtuple +import contextlib +import errno +import json +import logging +import os +import os.path +#import select +import subprocess +import sys +import tempfile +from textwrap import dedent +import threading +import types +import unittest + +from test import support + +# We would use test.support.import_helper.import_module(), +# but the indirect import of test.support.os_helper causes refleaks. +try: + import _interpreters +except ImportError as exc: + raise unittest.SkipTest(str(exc)) +from concurrent import interpreters + + +try: + import _testinternalcapi + import _testcapi +except ImportError: + _testinternalcapi = None + _testcapi = None + +def requires_test_modules(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + +def _dump_script(text): + lines = text.splitlines() + print() + print('-' * 20) + for i, line in enumerate(lines, 1): + print(f' {i:>{len(str(len(lines)))}} {line}') + print('-' * 20) + + +def _close_file(file): + try: + if hasattr(file, 'close'): + file.close() + else: + os.close(file) + except OSError as exc: + if exc.errno != errno.EBADF: + raise # re-raise + # It was closed already. + + +def pack_exception(exc=None): + captured = _interpreters.capture_exception(exc) + data = dict(captured.__dict__) + data['type'] = dict(captured.type.__dict__) + return json.dumps(data) + + +def unpack_exception(packed): + try: + data = json.loads(packed) + except json.decoder.JSONDecodeError as e: + logging.getLogger(__name__).warning('incomplete exception data', exc_info=e) + print(packed if isinstance(packed, str) else packed.decode('utf-8')) + return None + exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return exc; + + +class CapturingResults: + + STDIO = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: + _captured_std{stream} = io.StringIO() + with contextlib.redirect_std{stream}(_captured_std{stream}): + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + text = _captured_std{stream}.getvalue() + _spipe_{stream}.write(text.encode('utf-8')) + """)[:-1] + EXC = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: + try: + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + except Exception as exc: + text = _interp_utils.pack_exception(exc) + _spipe_exc.write(text.encode('utf-8')) + """)[:-1] + + @classmethod + def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): + script = dedent(script).strip(os.linesep) + imports = [ + f'import {__name__} as _interp_utils', + ] + wrapped = script + + # Handle exc. + if exc: + exc = os.pipe() + r_exc, w_exc = exc + indented = wrapped.replace('\n', '\n ') + wrapped = cls.EXC.format( + w_pipe=w_exc, + indented=indented, + ) + else: + exc = None + + # Handle stdout. + if stdout: + imports.extend([ + 'import contextlib, io', + ]) + stdout = os.pipe() + r_out, w_out = stdout + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_out, + indented=indented, + stream='out', + ) + else: + stdout = None + + # Handle stderr. + if stderr == 'stdout': + stderr = None + elif stderr: + if not stdout: + imports.extend([ + 'import contextlib, io', + ]) + stderr = os.pipe() + r_err, w_err = stderr + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_err, + indented=indented, + stream='err', + ) + else: + stderr = None + + if wrapped == script: + raise NotImplementedError + else: + for line in imports: + wrapped = f'{line}{os.linesep}{wrapped}' + + results = cls(stdout, stderr, exc) + return wrapped, results + + def __init__(self, out, err, exc): + self._rf_out = None + self._rf_err = None + self._rf_exc = None + self._w_out = None + self._w_err = None + self._w_exc = None + + if out is not None: + r_out, w_out = out + self._rf_out = open(r_out, 'rb', buffering=0) + self._w_out = w_out + + if err is not None: + r_err, w_err = err + self._rf_err = open(r_err, 'rb', buffering=0) + self._w_err = w_err + + if exc is not None: + r_exc, w_exc = exc + self._rf_exc = open(r_exc, 'rb', buffering=0) + self._w_exc = w_exc + + self._buf_out = b'' + self._buf_err = b'' + self._buf_exc = b'' + self._exc = None + + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def closed(self): + return self._closed + + def close(self): + if self._closed: + return + self._closed = True + + if self._w_out is not None: + _close_file(self._w_out) + self._w_out = None + if self._w_err is not None: + _close_file(self._w_err) + self._w_err = None + if self._w_exc is not None: + _close_file(self._w_exc) + self._w_exc = None + + self._capture() + + if self._rf_out is not None: + _close_file(self._rf_out) + self._rf_out = None + if self._rf_err is not None: + _close_file(self._rf_err) + self._rf_err = None + if self._rf_exc is not None: + _close_file(self._rf_exc) + self._rf_exc = None + + def _capture(self): + # Ideally this is called only after the script finishes + # (and thus has closed the write end of the pipe. + if self._rf_out is not None: + chunk = self._rf_out.read(100) + while chunk: + self._buf_out += chunk + chunk = self._rf_out.read(100) + if self._rf_err is not None: + chunk = self._rf_err.read(100) + while chunk: + self._buf_err += chunk + chunk = self._rf_err.read(100) + if self._rf_exc is not None: + chunk = self._rf_exc.read(100) + while chunk: + self._buf_exc += chunk + chunk = self._rf_exc.read(100) + + def _unpack_stdout(self): + return self._buf_out.decode('utf-8') + + def _unpack_stderr(self): + return self._buf_err.decode('utf-8') + + def _unpack_exc(self): + if self._exc is not None: + return self._exc + if not self._buf_exc: + return None + self._exc = unpack_exception(self._buf_exc) + return self._exc + + def stdout(self): + if self.closed: + return self.final().stdout + self._capture() + return self._unpack_stdout() + + def stderr(self): + if self.closed: + return self.final().stderr + self._capture() + return self._unpack_stderr() + + def exc(self): + if self.closed: + return self.final().exc + self._capture() + return self._unpack_exc() + + def final(self, *, force=False): + try: + return self._final + except AttributeError: + if not self._closed: + if not force: + raise Exception('no final results available yet') + else: + return CapturedResults.Proxy(self) + self._final = CapturedResults( + self._unpack_stdout(), + self._unpack_stderr(), + self._unpack_exc(), + ) + return self._final + + +class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')): + + class Proxy: + def __init__(self, capturing): + self._capturing = capturing + def _finish(self): + if self._capturing is None: + return + self._final = self._capturing.final() + self._capturing = None + def __iter__(self): + self._finish() + yield from self._final + def __len__(self): + self._finish() + return len(self._final) + def __getattr__(self, name): + self._finish() + if name.startswith('_'): + raise AttributeError(name) + return getattr(self._final, name) + + def raise_if_failed(self): + if self.exc is not None: + raise interpreters.ExecutionFailed(self.exc) + + +def _captured_script(script, *, stdout=True, stderr=False, exc=False): + return CapturingResults.wrap_script( + script, + stdout=stdout, + stderr=stderr, + exc=exc, + ) + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue + try: + interp.close() + except _interpreters.InterpreterError: + pass # already destroyed + + +def _run_output(interp, request, init=None): + script, results = _captured_script(request) + with results: + if init: + interp.prepare_main(init) + interp.exec(script) + return results.stdout() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +class TestBase(unittest.TestCase): + + def tearDown(self): + clean_up_interpreters() + + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + + def temp_dir(self): + tempdir = tempfile.mkdtemp() + tempdir = os.path.realpath(tempdir) + from test.support import os_helper + self.addCleanup(lambda: os_helper.rmtree(tempdir)) + return tempdir + + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + + def make_script(self, filename, dirname=None, text=None): + if text: + text = dedent(text) + if dirname is None: + dirname = self.temp_dir() + filename = os.path.join(dirname, filename) + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + def make_module(self, name, pathentry=None, text=None): + if text: + text = dedent(text) + if pathentry is None: + pathentry = self.temp_dir() + else: + os.makedirs(pathentry, exist_ok=True) + *subnames, basename = name.split('.') + + dirname = pathentry + for subname in subnames: + dirname = os.path.join(dirname, subname) + if os.path.isdir(dirname): + pass + elif os.path.exists(dirname): + raise Exception(dirname) + else: + os.mkdir(dirname) + initfile = os.path.join(dirname, '__init__.py') + if not os.path.exists(initfile): + with open(initfile, 'w'): + pass + filename = os.path.join(dirname, basename + '.py') + + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + @support.requires_subprocess() + def run_python(self, *argv): + proc = subprocess.run( + [sys.executable, *argv], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout, proc.stderr + + def assert_python_ok(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 1) + return stdout, stderr + + def assert_python_failure(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 0) + return stdout, stderr + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def _run_string(self, interp, script): + wrapped, results = _captured_script(script, exc=False) + #_dump_script(wrapped) + with results: + if isinstance(interp, interpreters.Interpreter): + interp.exec(script) + else: + err = _interpreters.run_string(interp, wrapped) + if err is not None: + return None, err + return results.stdout(), None + + def run_and_capture(self, interp, script): + text, err = self._run_string(interp, script) + if err is not None: + raise interpreters.ExecutionFailed(err) + else: + return text + + def interp_exists(self, interpid): + try: + _interpreters.whence(interpid) + except _interpreters.InterpreterNotFoundError: + return False + else: + return True + + @requires_test_modules + @contextlib.contextmanager + def interpreter_from_capi(self, config=None, whence=None): + if config is False: + if whence is None: + whence = _interpreters.WHENCE_LEGACY_CAPI + else: + assert whence in (_interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN), repr(whence) + config = None + elif config is True: + config = _interpreters.new_config('default') + elif config is None: + if whence not in ( + _interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN, + ): + config = _interpreters.new_config('legacy') + elif isinstance(config, str): + config = _interpreters.new_config(config) + + if whence is None: + whence = _interpreters.WHENCE_XI + + interpid = _testinternalcapi.create_interpreter(config, whence=whence) + try: + yield interpid + finally: + try: + _testinternalcapi.destroy_interpreter(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + @contextlib.contextmanager + def interpreter_obj_from_capi(self, config='legacy'): + with self.interpreter_from_capi(config) as interpid: + interp = interpreters.Interpreter( + interpid, + _whence=_interpreters.WHENCE_CAPI, + _ownsref=False, + ) + yield interp, interpid + + @contextlib.contextmanager + def capturing(self, script): + wrapped, capturing = _captured_script(script, stdout=True, exc=True) + #_dump_script(wrapped) + with capturing: + yield wrapped, capturing.final(force=True) + + @requires_test_modules + def run_from_capi(self, interpid, script, *, main=False): + with self.capturing(script) as (wrapped, results): + rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout + + @contextlib.contextmanager + def _running(self, run_interp, exec_interp): + token = b'\0' + r_in, w_in = self.pipe() + r_out, w_out = self.pipe() + + def close(): + _close_file(r_in) + _close_file(w_in) + _close_file(r_out) + _close_file(w_out) + + # Start running (and wait). + script = dedent(f""" + import os + try: + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + except BrokenPipeError: + pass + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + """) + failed = None + def run(): + nonlocal failed + try: + run_interp(script) + except Exception as exc: + failed = exc + close() + t = threading.Thread(target=run) + t.start() + + # handshake + try: + os.write(w_in, token) + token2 = os.read(r_out, 1) + assert token2 == token, (token2, token) + except OSError: + t.join() + if failed is not None: + raise failed + + # CM __exit__() + try: + try: + yield + finally: + # Send "done". + os.write(w_in, b'\0') + finally: + close() + t.join() + if failed is not None: + raise failed + + @contextlib.contextmanager + def running(self, interp): + if isinstance(interp, int): + interpid = interp + def exec_interp(script): + exc = _interpreters.exec(interpid, script) + assert exc is None, exc + run_interp = exec_interp + else: + def run_interp(script): + text = self.run_and_capture(interp, script) + assert text == '', repr(text) + def exec_interp(script): + interp.exec(script) + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + @contextlib.contextmanager + def running_from_capi(self, interpid, *, main=False): + def run_interp(script): + text = self.run_from_capi(interpid, script, main=main) + assert text == '', repr(text) + def exec_interp(script): + rc = _testinternalcapi.exec_interpreter(interpid, script) + assert rc == 0, rc + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + def run_temp_from_capi(self, script, config='legacy'): + if config is False: + # Force using Py_NewInterpreter(). + run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s)) + config = None + else: + run_in_interp = _testinternalcapi.run_in_subinterp_with_config + if config is True: + config = 'default' + if isinstance(config, str): + config = _interpreters.new_config(config) + with self.capturing(script) as (wrapped, results): + rc = run_in_interp(wrapped, config) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 2e50c1c62..7946550ec 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1,3 +1,4 @@ +import collections import copy import pickle import dis @@ -5,7 +6,7 @@ import threading import types import unittest from test.support import (threading_helper, check_impl_detail, - requires_specialization, requires_specialization_ft, + requires_specialization, cpython_only, requires_jit_disabled, reset_code) from test.support.import_helper import import_module @@ -524,7 +525,7 @@ class TestCallCache(TestBase): f() @requires_jit_disabled - @requires_specialization_ft + @requires_specialization def test_assign_init_code(self): class MyClass: def __init__(self): @@ -547,7 +548,7 @@ class TestCallCache(TestBase): instantiate() @requires_jit_disabled - @requires_specialization_ft + @requires_specialization def test_push_init_frame_fails(self): def instantiate(): return InitTakesArg() @@ -575,6 +576,39 @@ class TestCallCache(TestBase): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() + @requires_jit_disabled + @requires_specialization + def test_specialize_call_function_ex_py(self): + def ex_py(*args, **kwargs): + return 1 + + def instantiate(): + args = (1, 2, 3) + kwargs = {} + return ex_py(*args, **kwargs) + + # Trigger specialization + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + instantiate() + self.assert_specialized(instantiate, "CALL_EX_PY") + + @requires_jit_disabled + @requires_specialization + def test_specialize_call_function_ex_py_fail(self): + def ex_py(*args, **kwargs): + return 1 + + def instantiate(): + args = (1, 2, 3) + kwargs = {} + return ex_py(*args, **kwargs) + + _testinternalcapi.set_vectorcall_nop(ex_py) + # Trigger specialization + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + instantiate() + self.assert_no_opcode(instantiate, "CALL_EX_PY") + self.assert_specialized(instantiate, "CALL_EX_NON_PY_GENERAL") def make_deferred_ref_count_obj(): """Create an object that uses deferred reference counting. @@ -590,7 +624,7 @@ def make_deferred_ref_count_obj(): class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: - ITEMS = 1000 + ITEMS = 1400 LOOPS = 4 WRITERS = 2 @@ -627,7 +661,7 @@ class TestRacesDoNotCrash(TestBase): for writer in writers: writer.join() - @requires_specialization_ft + @requires_specialization def test_binary_subscr_getitem(self): def get_items(): class C: @@ -657,7 +691,7 @@ class TestRacesDoNotCrash(TestBase): opname = "BINARY_OP_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_binary_subscr_list_int(self): def get_items(): items = [] @@ -735,7 +769,7 @@ class TestRacesDoNotCrash(TestBase): opname = "FOR_ITER_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_class(self): def get_items(): class C: @@ -765,7 +799,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_CLASS" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_class_with_metaclass_check(self): def get_items(): class Meta(type): @@ -798,7 +832,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_getattribute_overridden(self): def get_items(): class C: @@ -828,7 +862,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_instance_value(self): def get_items(): class C: @@ -852,7 +886,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_method_lazy_dict(self): def get_items(): class C(Exception): @@ -882,7 +916,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_METHOD_LAZY_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_method_no_dict(self): def get_items(): class C: @@ -913,7 +947,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_METHOD_NO_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_method_with_values(self): def get_items(): class C: @@ -943,7 +977,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_METHOD_WITH_VALUES" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_module(self): def get_items(): items = [] @@ -968,7 +1002,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_MODULE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_property(self): def get_items(): class C: @@ -998,7 +1032,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_PROPERTY" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_slot(self): def get_items(): class C: @@ -1025,7 +1059,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_SLOT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_attr_with_hint(self): def get_items(): class C: @@ -1052,7 +1086,7 @@ class TestRacesDoNotCrash(TestBase): opname = "LOAD_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_load_global_module(self): if not have_dict_key_versions(): raise unittest.SkipTest("Low on dict key versions") @@ -1125,7 +1159,7 @@ class TestRacesDoNotCrash(TestBase): opname = "STORE_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_store_subscr_list_int(self): def get_items(): items = [] @@ -1149,7 +1183,7 @@ class TestRacesDoNotCrash(TestBase): opname = "STORE_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization_ft + @requires_specialization def test_unpack_sequence_list(self): def get_items(): items = [] @@ -1329,7 +1363,7 @@ class TestInstanceDict(unittest.TestCase): class TestSpecializer(TestBase): @cpython_only - @requires_specialization_ft + @requires_specialization def test_binary_op(self): def binary_op_add_int(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): @@ -1341,6 +1375,21 @@ class TestSpecializer(TestBase): self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") self.assert_no_opcode(binary_op_add_int, "BINARY_OP") + def binary_op_int_non_compact(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 10000000000, 1 + c = a + b + self.assertEqual(c, 10000000001) + c = a - b + self.assertEqual(c, 9999999999) + c = a * b + self.assertEqual(c, 10000000000) + + binary_op_int_non_compact() + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_SUBTRACT_INT") + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_MULTIPLY_INT") + def binary_op_add_unicode(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = "foo", "bar" @@ -1370,11 +1419,50 @@ class TestSpecializer(TestBase): self.assertEqual(c, 2.0) c = b / a self.assertEqual(c, 0.5) + c = a + c += b + self.assertEqual(c, 9.0) + c = b + c += a + self.assertEqual(c, 9.0) + c = a + c -= b + self.assertEqual(c, 3.0) + c = b + c -= a + self.assertEqual(c, -3.0) + c = a + c *= b + self.assertEqual(c, 18.0) + c = b + c *= a + self.assertEqual(c, 18.0) + c = a + c /= b + self.assertEqual(c, 2.0) + c = b + c /= a + self.assertEqual(c, 0.5) binary_op_add_extend() self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + def binary_op_add_extend_sequences(): + l1 = [1, 2] + l2 = [None] + t1 = (1, 2) + t2 = (None,) + for _ in range(100): + list_sum = l1 + l2 + self.assertEqual(list_sum, [1, 2, None]) + tuple_sum = t1 + t2 + self.assertEqual(tuple_sum, (1, 2, None)) + + binary_op_add_extend_sequences() + self.assert_specialized(binary_op_add_extend_sequences, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_add_extend_sequences, "BINARY_OP") + def binary_op_zero_division(): def compactlong_lhs(arg): 42 / arg @@ -1449,7 +1537,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(binary_op_bitwise_extend, "BINARY_OP") @cpython_only - @requires_specialization_ft + @requires_specialization def test_load_super_attr(self): """Ensure that LOAD_SUPER_ATTR is specialized as expected.""" @@ -1488,7 +1576,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_METHOD") @cpython_only - @requires_specialization_ft + @requires_specialization def test_contain_op(self): def contains_op_dict(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): @@ -1500,6 +1588,27 @@ class TestSpecializer(TestBase): self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") + def contains_op_frozen_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, frozendict({1: 2, 2: 5}) + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_frozen_dict() + self.assert_specialized(contains_op_frozen_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_frozen_dict, "CONTAINS_OP") + + def contains_op_frozen_dict_subclass(): + class MyFrozenDict(frozendict): + pass + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, MyFrozenDict({1: 2, 2: 5}) + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_frozen_dict_subclass() + self.assert_no_opcode(contains_op_frozen_dict_subclass, "CONTAINS_OP_DICT") + def contains_op_set(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, {1, 2} @@ -1511,7 +1620,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(contains_op_set, "CONTAINS_OP") @cpython_only - @requires_specialization_ft + @requires_specialization def test_send_with(self): def run_async(coro): while True: @@ -1538,7 +1647,7 @@ class TestSpecializer(TestBase): self.assert_specialized(send_with, "SEND_GEN") @cpython_only - @requires_specialization_ft + @requires_specialization def test_send_yield_from(self): def g(): yield None @@ -1553,7 +1662,38 @@ class TestSpecializer(TestBase): self.assert_no_opcode(send_yield_from, "SEND") @cpython_only - @requires_specialization_ft + @requires_specialization + def test_send_yield_from_iter(self): + L = list(range(100)) + def send_yield_from(): + yield from L + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_VIRTUAL") + self.assert_no_opcode(send_yield_from, "SEND") + + @cpython_only + @requires_specialization + def test_send_async_for(self): + async def g(): + yield None + + async def send_for(): + async for _ in g(): + break + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + try: + send_for().send(None) + except StopIteration: + pass + self.assert_specialized(send_for, "SEND_ASYNC_GEN") + self.assert_no_opcode(send_for, "SEND") + + @cpython_only + @requires_specialization def test_store_attr_slot(self): class C: __slots__ = ['x'] @@ -1574,7 +1714,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") @cpython_only - @requires_specialization_ft + @requires_specialization def test_store_attr_instance_value(self): class C: pass @@ -1596,7 +1736,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") @cpython_only - @requires_specialization_ft + @requires_specialization def test_store_attr_with_hint(self): class C: pass @@ -1621,7 +1761,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") @cpython_only - @requires_specialization_ft + @requires_specialization def test_to_bool(self): def to_bool_bool(): true_cnt, false_cnt = 0, 0 @@ -1689,7 +1829,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(to_bool_str, "TO_BOOL") @cpython_only - @requires_specialization_ft + @requires_specialization def test_unpack_sequence(self): def unpack_sequence_two_tuple(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): @@ -1726,7 +1866,7 @@ class TestSpecializer(TestBase): self.assert_no_opcode(unpack_sequence_list, "UNPACK_SEQUENCE") @cpython_only - @requires_specialization_ft + @requires_specialization def test_binary_subscr(self): def binary_subscr_list_int(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): @@ -1760,6 +1900,63 @@ class TestSpecializer(TestBase): self.assert_specialized(binary_subscr_dict, "BINARY_OP_SUBSCR_DICT") self.assert_no_opcode(binary_subscr_dict, "BINARY_OP") + def binary_subscr_frozen_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = frozendict({1: 2, 2: 3}) + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_frozen_dict() + self.assert_specialized(binary_subscr_frozen_dict, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_frozen_dict, "BINARY_OP") + + def binary_subscr_frozen_dict_subclass(): + class MyFrozenDict(frozendict): + pass + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = MyFrozenDict({1: 2, 2: 3}) + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_frozen_dict_subclass() + self.assert_specialized(binary_subscr_frozen_dict_subclass, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_frozen_dict_subclass, "BINARY_OP") + + def binary_subscr_defaultdict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = collections.defaultdict(lambda: 42, {1: 2, 2: 3}) + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + self.assertEqual(a[7], 42) + + binary_subscr_defaultdict() + self.assert_specialized(binary_subscr_defaultdict, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_defaultdict, "BINARY_OP") + + def binary_subscr_counter(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = collections.Counter('abcdeabcdabcaba') + self.assertEqual(a['a'], 5) + self.assertEqual(a['b'], 4) + self.assertEqual(a['m'], 0) + + binary_subscr_counter() + self.assert_specialized(binary_subscr_counter, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_counter, "BINARY_OP") + + def binary_subscr_dict_subclass_override(): + class MyDict(dict): + def __getitem__(self, key): + return 42 + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = MyDict() + self.assertEqual(a['a'], 42) + self.assertEqual(a['b'], 42) + + binary_subscr_dict_subclass_override() + self.assert_no_opcode(binary_subscr_dict_subclass_override, "BINARY_OP_SUBSCR_DICT") + def binary_subscr_str_int(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = "foobar" @@ -1770,6 +1967,16 @@ class TestSpecializer(TestBase): self.assert_specialized(binary_subscr_str_int, "BINARY_OP_SUBSCR_STR_INT") self.assert_no_opcode(binary_subscr_str_int, "BINARY_OP") + def binary_subscr_str_int_non_compact(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = "바이트코드_특수화" + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_str_int_non_compact() + self.assert_specialized(binary_subscr_str_int_non_compact, "BINARY_OP_SUBSCR_USTR_INT") + self.assert_no_opcode(binary_subscr_str_int_non_compact, "BINARY_OP_SUBSCR_STR_INT") + def binary_subscr_getitems(): class C: def __init__(self, val): @@ -1786,7 +1993,54 @@ class TestSpecializer(TestBase): self.assert_no_opcode(binary_subscr_getitems, "BINARY_OP") @cpython_only - @requires_specialization_ft + @requires_specialization + def test_store_subscr(self): + def store_subscr_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = {1: 2, 2: 3} + a[1] = 4 + self.assertEqual(a[1], 4) + + store_subscr_dict() + self.assert_specialized(store_subscr_dict, "STORE_SUBSCR_DICT") + self.assert_no_opcode(store_subscr_dict, "STORE_SUBSCR") + + def store_subscr_frozen_dict(): + dicts = [{1: 2}, frozendict({1: 2})] + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + d = dicts[i == _testinternalcapi.SPECIALIZATION_THRESHOLD] + d[1] = 3 + + with self.assertRaises(TypeError): + store_subscr_frozen_dict() + self.assert_specialized(store_subscr_frozen_dict, "STORE_SUBSCR_DICT") + self.assert_no_opcode(store_subscr_frozen_dict, "STORE_SUBSCR") + + def store_subscr_defaultdict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = collections.defaultdict(int) + a[1] = 4 + self.assertEqual(a[1], 4) + + store_subscr_defaultdict() + self.assert_specialized(store_subscr_defaultdict, "STORE_SUBSCR_DICT") + self.assert_no_opcode(store_subscr_defaultdict, "STORE_SUBSCR") + + def store_subscr_dict_subclass_override(): + class MyDict(dict): + def __setitem__(self, key, value): + super().__setitem__(key, value * 2) + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = MyDict() + a['x'] = 5 + self.assertEqual(a['x'], 10) + + store_subscr_dict_subclass_override() + self.assert_no_opcode(store_subscr_dict_subclass_override, "STORE_SUBSCR_DICT") + + @cpython_only + @requires_specialization def test_compare_op(self): def compare_op_int(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): @@ -1818,23 +2072,9 @@ class TestSpecializer(TestBase): self.assert_specialized(compare_op_str, "COMPARE_OP_STR") self.assert_no_opcode(compare_op_str, "COMPARE_OP") - @cpython_only - @requires_specialization_ft - def test_load_const(self): - def load_const(): - def unused(): pass - # Currently, the empty tuple is immortal, and the otherwise - # unused nested function's code object is mortal. This test will - # have to use different values if either of that changes. - return () - - load_const() - self.assert_specialized(load_const, "LOAD_CONST_IMMORTAL") - self.assert_specialized(load_const, "LOAD_CONST_MORTAL") - self.assert_no_opcode(load_const, "LOAD_CONST") @cpython_only - @requires_specialization_ft + @requires_specialization def test_for_iter(self): L = list(range(10)) def for_iter_list(): @@ -1854,6 +2094,15 @@ class TestSpecializer(TestBase): self.assert_specialized(for_iter_tuple, "FOR_ITER_TUPLE") self.assert_no_opcode(for_iter_tuple, "FOR_ITER") + s = "abcdefghij" + def for_iter_str(): + for i in s: + self.assertIn(i, s) + + for_iter_str() + self.assert_specialized(for_iter_str, "FOR_ITER_VIRTUAL") + self.assert_no_opcode(for_iter_str, "FOR_ITER") + r = range(10) def for_iter_range(): for i in r: @@ -1871,6 +2120,126 @@ class TestSpecializer(TestBase): self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") self.assert_no_opcode(for_iter_generator, "FOR_ITER") + @cpython_only + @requires_specialization + def test_get_iter(self): + L = list(range(10)) + def get_iter_list(): + n = 10 + while n: + n -= 1 + for i in L: + break + get_iter_list() + self.assert_specialized(get_iter_list, "GET_ITER_VIRTUAL") + self.assert_no_opcode(get_iter_list, "GET_ITER") + + def get_iter_gen(): + n = 10 + while n: + n -= 1 + for i in (i for i in range(10)): + break + get_iter_gen() + self.assert_specialized(get_iter_gen, "GET_ITER_SELF") + self.assert_no_opcode(get_iter_gen, "GET_ITER") + + @cpython_only + @requires_specialization + def test_call_list_append(self): + # gh-141367: only exact lists should use + # CALL_LIST_APPEND instruction after specialization. + + r = range(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + def list_append(l): + for _ in r: + l.append(1) + + list_append([]) + self.assert_specialized(list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(list_append, "CALL") + + def my_list_append(l): + for _ in r: + l.append(1) + + class MyList(list): pass + my_list_append(MyList()) + self.assert_specialized(my_list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(my_list_append, "CALL") + + @cpython_only + @requires_specialization + def test_load_attr_module_with_getattr(self): + module = types.ModuleType("test_module_with_getattr") + module.__dict__["some_attr"] = "foo" + + def module_getattr(name): + if name == "missing_attr": + return 42 + raise AttributeError(f"module has no attribute {name}") + + module.__dict__["__getattr__"] = module_getattr + + import sys + sys.modules.pop("test_module_with_getattr", None) + sys.modules["test_module_with_getattr"] = module + try: + def load_module_attr_present(): + import test_module_with_getattr + return test_module_with_getattr.some_attr + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(load_module_attr_present(), "foo") + + self.assert_specialized(load_module_attr_present, "LOAD_ATTR_MODULE") + self.assert_no_opcode(load_module_attr_present, "LOAD_ATTR") + + def load_module_attr_missing(): + import test_module_with_getattr + return test_module_with_getattr.missing_attr + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(load_module_attr_missing(), 42) + + self.assert_no_opcode(load_module_attr_missing, "LOAD_ATTR_MODULE") + finally: + sys.modules.pop("test_module_with_getattr", None) + + @cpython_only + @requires_specialization + def test_specialized_iter_doesnt_skip_send_check(self): + def gen_func(seq): + yield from seq + gen = gen_func(list(range(10))) + for _ in range(3): + gen.send(None) + with self.assertRaises(AttributeError): + gen.send(1) + + + @cpython_only + @requires_specialization + def test_load_attr_enum(self): + import enum + + class Color(enum.IntEnum): + RED = 1 + GREEN = 2 + BLUE = 3 + + def load_enum_member(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + x = Color.RED + assert x == 1 + + load_enum_member() + self.assert_specialized(load_enum_member, + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_picklebuffer.py b/Lib/test/test_picklebuffer.py index 748d6f5a5..fff695fa3 100644 --- a/Lib/test/test_picklebuffer.py +++ b/Lib/test/test_picklebuffer.py @@ -64,8 +64,7 @@ class PickleBufferTest(unittest.TestCase): # Idempotency pb.release() - # TODO: RUSTPYTHON; PickleBuffer is not traversed by the cycle collector - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; PickleBuffer is not traversed by the cycle collector def test_cycle(self): b = B(b"foo") pb = PickleBuffer(b) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 9c93427a9..eb854aaa8 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -689,8 +689,7 @@ class CommandLineTest(unittest.TestCase): def load_tests(loader, tests, pattern): - from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON; Remove this - tests.addTest(doctest.DocTestSuite(pickletools, checker=DocTestChecker())) # XXX: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(pickletools)) return tests diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py new file mode 100644 index 000000000..e53875260 --- /dev/null +++ b/Lib/test/test_remote_pdb.py @@ -0,0 +1,1754 @@ +import io +import itertools +import json +import os +import re +import signal +import socket +import subprocess +import sys +import textwrap +import threading +import unittest +import unittest.mock +from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack +from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT, subTests +from test.support.os_helper import TESTFN, unlink +from typing import List + +import pdb +from pdb import _PdbServer, _PdbClient + +try: + import pty +except ImportError: + pty = None + + +if not sys.is_remote_debug_enabled(): + raise unittest.SkipTest('remote debugging is disabled') + + +@contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + + +class MockSocketFile: + """Mock socket file for testing _PdbServer without actual socket connections.""" + + def __init__(self): + self.input_queue = [] + self.output_buffer = [] + + def write(self, data: bytes) -> None: + """Simulate write to socket.""" + self.output_buffer.append(data) + + def flush(self) -> None: + """No-op flush implementation.""" + pass + + def readline(self) -> bytes: + """Read a line from the prepared input queue.""" + if not self.input_queue: + return b"" + return self.input_queue.pop(0) + + def close(self) -> None: + """Close the mock socket file.""" + pass + + def add_input(self, data: dict) -> None: + """Add input that will be returned by readline.""" + self.input_queue.append(json.dumps(data).encode() + b"\n") + + def get_output(self) -> List[dict]: + """Get the output that was written by the object being tested.""" + results = [] + for data in self.output_buffer: + if isinstance(data, bytes) and data.endswith(b"\n"): + try: + results.append(json.loads(data.decode().strip())) + except json.JSONDecodeError: + pass # Ignore non-JSON output + self.output_buffer = [] + return results + + +class PdbClientTestCase(unittest.TestCase): + """Tests for the _PdbClient class.""" + + def do_test( + self, + *, + incoming, + simulate_send_failure=False, + simulate_sigint_during_stdout_write=False, + use_interrupt_socket=False, + expected_outgoing=None, + expected_outgoing_signals=None, + expected_completions=None, + expected_exception=None, + expected_stdout="", + expected_stdout_substring="", + expected_state=None, + ): + if expected_outgoing is None: + expected_outgoing = [] + if expected_outgoing_signals is None: + expected_outgoing_signals = [] + if expected_completions is None: + expected_completions = [] + if expected_state is None: + expected_state = {} + + expected_state.setdefault("write_failed", False) + messages = [m for source, m in incoming if source == "server"] + prompts = [m["prompt"] for source, m in incoming if source == "user"] + + input_iter = (m for source, m in incoming if source == "user") + completions = [] + + def mock_input(prompt): + message = next(input_iter, None) + if message is None: + raise EOFError + + if req := message.get("completion_request"): + readline_mock = unittest.mock.Mock() + readline_mock.get_line_buffer.return_value = req["line"] + readline_mock.get_begidx.return_value = req["begidx"] + readline_mock.get_endidx.return_value = req["endidx"] + unittest.mock.seal(readline_mock) + with unittest.mock.patch.dict(sys.modules, {"readline": readline_mock}): + for param in itertools.count(): + prefix = req["line"][req["begidx"] : req["endidx"]] + completion = client.complete(prefix, param) + if completion is None: + break + completions.append(completion) + + reply = message["input"] + if isinstance(reply, BaseException): + raise reply + if isinstance(reply, str): + return reply + return reply() + + with ExitStack() as stack: + client_sock, server_sock = socket.socketpair() + stack.enter_context(closing(client_sock)) + stack.enter_context(closing(server_sock)) + + server_sock = unittest.mock.Mock(wraps=server_sock) + + client_sock.sendall( + b"".join( + (m if isinstance(m, bytes) else json.dumps(m).encode()) + b"\n" + for m in messages + ) + ) + client_sock.shutdown(socket.SHUT_WR) + + if simulate_send_failure: + server_sock.sendall = unittest.mock.Mock( + side_effect=OSError("sendall failed") + ) + client_sock.shutdown(socket.SHUT_RD) + + stdout = io.StringIO() + + if simulate_sigint_during_stdout_write: + orig_stdout_write = stdout.write + + def sigint_stdout_write(s): + signal.raise_signal(signal.SIGINT) + return orig_stdout_write(s) + + stdout.write = sigint_stdout_write + + input_mock = stack.enter_context( + unittest.mock.patch("pdb.input", side_effect=mock_input) + ) + stack.enter_context(redirect_stdout(stdout)) + + if use_interrupt_socket: + interrupt_sock = unittest.mock.Mock(spec=socket.socket) + mock_kill = None + else: + interrupt_sock = None + mock_kill = stack.enter_context( + unittest.mock.patch("os.kill", spec=os.kill) + ) + + client = _PdbClient( + pid=12345, + server_socket=server_sock, + interrupt_sock=interrupt_sock, + ) + + if expected_exception is not None: + exception = expected_exception["exception"] + msg = expected_exception["msg"] + stack.enter_context(self.assertRaises(exception, msg=msg)) + + client.cmdloop() + + sent_msgs = [msg.args[0] for msg in server_sock.sendall.mock_calls] + for msg in sent_msgs: + assert msg.endswith(b"\n") + actual_outgoing = [json.loads(msg) for msg in sent_msgs] + + self.assertEqual(actual_outgoing, expected_outgoing) + self.assertEqual(completions, expected_completions) + if expected_stdout_substring and not expected_stdout: + self.assertIn(expected_stdout_substring, stdout.getvalue()) + else: + self.assertEqual(stdout.getvalue(), expected_stdout) + input_mock.assert_has_calls([unittest.mock.call(p) for p in prompts]) + actual_state = {k: getattr(client, k) for k in expected_state} + self.assertEqual(actual_state, expected_state) + + if use_interrupt_socket: + outgoing_signals = [ + signal.Signals(int.from_bytes(call.args[0])) + for call in interrupt_sock.sendall.call_args_list + ] + else: + assert mock_kill is not None + outgoing_signals = [] + for call in mock_kill.call_args_list: + pid, signum = call.args + self.assertEqual(pid, 12345) + outgoing_signals.append(signal.Signals(signum)) + self.assertEqual(outgoing_signals, expected_outgoing_signals) + + def test_remote_immediately_closing_the_connection(self): + """Test the behavior when the remote closes the connection immediately.""" + incoming = [] + expected_outgoing = [] + self.do_test( + incoming=incoming, + expected_outgoing=expected_outgoing, + ) + + def test_handling_command_list(self): + """Test handling the command_list message.""" + incoming = [ + ("server", {"command_list": ["help", "list", "continue"]}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_state={ + "pdb_commands": {"help", "list", "continue"}, + }, + ) + + def test_handling_info_message(self): + """Test handling a message payload with type='info'.""" + incoming = [ + ("server", {"message": "Some message or other\n", "type": "info"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout="Some message or other\n", + ) + + def test_handling_error_message(self): + """Test handling a message payload with type='error'.""" + incoming = [ + ("server", {"message": "Some message or other.", "type": "error"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout="*** Some message or other.\n", + ) + + def test_handling_other_message(self): + """Test handling a message payload with an unrecognized type.""" + incoming = [ + ("server", {"message": "Some message.\n", "type": "unknown"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout="Some message.\n", + ) + + @unittest.skipIf(sys.flags.optimize >= 2, "Help not available for -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "Usage: ll | longlist"), + # a request to display a help overview + ({"help": ""}, "type help "), + # a request to display the full PDB manual + ({"help": "pdb"}, ">>> import pdb"), + ], + ) + def test_handling_help_when_available(self, help_request, expected_substring): + """Test handling help requests when help is available.""" + incoming = [ + ("server", help_request), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout_substring=expected_substring, + ) + + @unittest.skipIf(sys.flags.optimize < 2, "Needs -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "No help for 'll'"), + # a request to display a help overview + ({"help": ""}, "Undocumented commands"), + # a request to display the full PDB manual + ({"help": "pdb"}, "No help for 'pdb'"), + ], + ) + def test_handling_help_when_not_available(self, help_request, expected_substring): + """Test handling help requests when help is not available.""" + incoming = [ + ("server", help_request), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout_substring=expected_substring, + ) + + def test_handling_pdb_prompts(self): + """Test responding to pdb's normal prompts.""" + incoming = [ + ("server", {"command_list": ["b"]}), + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": "lst ["}), + ("user", {"prompt": "... ", "input": "0 ]"}), + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": ""}), + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": "b ["}), + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": "! b ["}), + ("user", {"prompt": "... ", "input": "b ]"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "lst [\n0 ]"}, + {"reply": ""}, + {"reply": "b ["}, + {"reply": "!b [\nb ]"}, + ], + expected_state={"state": "pdb"}, + ) + + def test_handling_interact_prompts(self): + """Test responding to pdb's interact mode prompts.""" + incoming = [ + ("server", {"command_list": ["b"]}), + ("server", {"prompt": ">>> ", "state": "interact"}), + ("user", {"prompt": ">>> ", "input": "lst ["}), + ("user", {"prompt": "... ", "input": "0 ]"}), + ("server", {"prompt": ">>> ", "state": "interact"}), + ("user", {"prompt": ">>> ", "input": ""}), + ("server", {"prompt": ">>> ", "state": "interact"}), + ("user", {"prompt": ">>> ", "input": "b ["}), + ("user", {"prompt": "... ", "input": "b ]"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "lst [\n0 ]"}, + {"reply": ""}, + {"reply": "b [\nb ]"}, + ], + expected_state={"state": "interact"}, + ) + + def test_retry_pdb_prompt_on_syntax_error(self): + """Test re-prompting after a SyntaxError in a Python expression.""" + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": " lst ["}), + ("user", {"prompt": "(Pdb) ", "input": "lst ["}), + ("user", {"prompt": "... ", "input": " 0 ]"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "lst [\n 0 ]"}, + ], + expected_stdout_substring="*** IndentationError", + expected_state={"state": "pdb"}, + ) + + def test_retry_interact_prompt_on_syntax_error(self): + """Test re-prompting after a SyntaxError in a Python expression.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ("user", {"prompt": ">>> ", "input": "!lst ["}), + ("user", {"prompt": ">>> ", "input": "lst ["}), + ("user", {"prompt": "... ", "input": " 0 ]"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "lst [\n 0 ]"}, + ], + expected_stdout_substring="*** SyntaxError", + expected_state={"state": "interact"}, + ) + + def test_handling_unrecognized_prompt_type(self): + """Test fallback to "dumb" single-line mode for unknown states.""" + incoming = [ + ("server", {"prompt": "Do it? ", "state": "confirm"}), + ("user", {"prompt": "Do it? ", "input": "! ["}), + ("server", {"prompt": "Do it? ", "state": "confirm"}), + ("user", {"prompt": "Do it? ", "input": "echo hello"}), + ("server", {"prompt": "Do it? ", "state": "confirm"}), + ("user", {"prompt": "Do it? ", "input": ""}), + ("server", {"prompt": "Do it? ", "state": "confirm"}), + ("user", {"prompt": "Do it? ", "input": "echo goodbye"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "! ["}, + {"reply": "echo hello"}, + {"reply": ""}, + {"reply": "echo goodbye"}, + ], + expected_state={"state": "dumb"}, + ) + + def test_sigint_at_prompt(self): + """Test signaling when a prompt gets interrupted.""" + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ( + "user", + { + "prompt": "(Pdb) ", + "input": lambda: signal.raise_signal(signal.SIGINT), + }, + ), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"signal": "INT"}, + ], + expected_state={"state": "pdb"}, + ) + + def test_sigint_at_continuation_prompt(self): + """Test signaling when a continuation prompt gets interrupted.""" + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": "if True:"}), + ( + "user", + { + "prompt": "... ", + "input": lambda: signal.raise_signal(signal.SIGINT), + }, + ), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"signal": "INT"}, + ], + expected_state={"state": "pdb"}, + ) + + def test_sigint_when_writing(self): + """Test siginaling when sys.stdout.write() gets interrupted.""" + incoming = [ + ("server", {"message": "Some message or other\n", "type": "info"}), + ] + for use_interrupt_socket in [False, True]: + with self.subTest(use_interrupt_socket=use_interrupt_socket): + self.do_test( + incoming=incoming, + simulate_sigint_during_stdout_write=True, + use_interrupt_socket=use_interrupt_socket, + expected_outgoing=[], + expected_outgoing_signals=[signal.SIGINT], + expected_stdout="Some message or other\n", + ) + + def test_eof_at_prompt(self): + """Test signaling when a prompt gets an EOFError.""" + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": EOFError()}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"signal": "EOF"}, + ], + expected_state={"state": "pdb"}, + ) + + def test_unrecognized_json_message(self): + """Test failing after getting an unrecognized payload.""" + incoming = [ + ("server", {"monty": "python"}), + ("server", {"message": "Some message or other\n", "type": "info"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_exception={ + "exception": RuntimeError, + "msg": 'Unrecognized payload b\'{"monty": "python"}\'', + }, + ) + + def test_continuing_after_getting_a_non_json_payload(self): + """Test continuing after getting a non JSON payload.""" + incoming = [ + ("server", b"spam"), + ("server", {"message": "Something", "type": "info"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout="\n".join( + [ + "*** Invalid JSON from remote: b'spam\\n'", + "Something", + ] + ), + ) + + def test_write_failing(self): + """Test terminating if write fails due to a half closed socket.""" + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": KeyboardInterrupt()}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[{"signal": "INT"}], + simulate_send_failure=True, + expected_state={"write_failed": True}, + ) + + def test_completion_in_pdb_state(self): + """Test requesting tab completions at a (Pdb) prompt.""" + # GIVEN + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ( + "user", + { + "prompt": "(Pdb) ", + "completion_request": { + "line": " mod._", + "begidx": 8, + "endidx": 9, + }, + "input": "print(\n mod.__name__)", + }, + ), + ("server", {"completions": ["__name__", "__file__"]}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "_", + "line": "mod._", + "begidx": 4, + "endidx": 5, + } + }, + {"reply": "print(\n mod.__name__)"}, + ], + expected_completions=["__name__", "__file__"], + expected_state={"state": "pdb"}, + ) + + def test_multiline_completion_in_pdb_state(self): + """Test requesting tab completions at a (Pdb) continuation prompt.""" + # GIVEN + incoming = [ + ("server", {"prompt": "(Pdb) ", "state": "pdb"}), + ("user", {"prompt": "(Pdb) ", "input": "if True:"}), + ( + "user", + { + "prompt": "... ", + "completion_request": { + "line": " b", + "begidx": 4, + "endidx": 5, + }, + "input": " bool()", + }, + ), + ("server", {"completions": ["bin", "bool", "bytes"]}), + ("user", {"prompt": "... ", "input": ""}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "b", + "line": "! b", + "begidx": 2, + "endidx": 3, + } + }, + {"reply": "if True:\n bool()\n"}, + ], + expected_completions=["bin", "bool", "bytes"], + expected_state={"state": "pdb"}, + ) + + def test_completion_in_interact_state(self): + """Test requesting tab completions at a >>> prompt.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ( + "user", + { + "prompt": ">>> ", + "completion_request": { + "line": " mod.__", + "begidx": 8, + "endidx": 10, + }, + "input": "print(\n mod.__name__)", + }, + ), + ("server", {"completions": ["__name__", "__file__"]}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "__", + "line": "mod.__", + "begidx": 4, + "endidx": 6, + } + }, + {"reply": "print(\n mod.__name__)"}, + ], + expected_completions=["__name__", "__file__"], + expected_state={"state": "interact"}, + ) + + def test_completion_in_unknown_state(self): + """Test requesting tab completions at an unrecognized prompt.""" + incoming = [ + ("server", {"command_list": ["p"]}), + ("server", {"prompt": "Do it? ", "state": "confirm"}), + ( + "user", + { + "prompt": "Do it? ", + "completion_request": { + "line": "_", + "begidx": 0, + "endidx": 1, + }, + "input": "__name__", + }, + ), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + {"reply": "__name__"}, + ], + expected_state={"state": "dumb"}, + ) + + def test_write_failure_during_completion(self): + """Test failing to write to the socket to request tab completions.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ( + "user", + { + "prompt": ">>> ", + "completion_request": { + "line": "xy", + "begidx": 0, + "endidx": 2, + }, + "input": "xyz", + }, + ), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "xy", + "line": "xy", + "begidx": 0, + "endidx": 2, + } + }, + {"reply": "xyz"}, + ], + simulate_send_failure=True, + expected_completions=[], + expected_state={"state": "interact", "write_failed": True}, + ) + + def test_read_failure_during_completion(self): + """Test failing to read tab completions from the socket.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ( + "user", + { + "prompt": ">>> ", + "completion_request": { + "line": "xy", + "begidx": 0, + "endidx": 2, + }, + "input": "xyz", + }, + ), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "xy", + "line": "xy", + "begidx": 0, + "endidx": 2, + } + }, + {"reply": "xyz"}, + ], + expected_completions=[], + expected_state={"state": "interact"}, + ) + + def test_reading_invalid_json_during_completion(self): + """Test receiving invalid JSON when getting tab completions.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ( + "user", + { + "prompt": ">>> ", + "completion_request": { + "line": "xy", + "begidx": 0, + "endidx": 2, + }, + "input": "xyz", + }, + ), + ("server", b'{"completions": '), + ("user", {"prompt": ">>> ", "input": "xyz"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "xy", + "line": "xy", + "begidx": 0, + "endidx": 2, + } + }, + {"reply": "xyz"}, + ], + expected_stdout_substring="*** json.decoder.JSONDecodeError", + expected_completions=[], + expected_state={"state": "interact"}, + ) + + def test_reading_empty_json_during_completion(self): + """Test receiving an empty JSON object when getting tab completions.""" + incoming = [ + ("server", {"prompt": ">>> ", "state": "interact"}), + ( + "user", + { + "prompt": ">>> ", + "completion_request": { + "line": "xy", + "begidx": 0, + "endidx": 2, + }, + "input": "xyz", + }, + ), + ("server", {}), + ("user", {"prompt": ">>> ", "input": "xyz"}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[ + { + "complete": { + "text": "xy", + "line": "xy", + "begidx": 0, + "endidx": 2, + } + }, + {"reply": "xyz"}, + ], + expected_stdout=( + "*** RuntimeError: Failed to get valid completions." + " Got: {}\n" + ), + expected_completions=[], + expected_state={"state": "interact"}, + ) + + +class RemotePdbTestCase(unittest.TestCase): + """Tests for the _PdbServer class.""" + + def setUp(self): + self.sockfile = MockSocketFile() + self.pdb = _PdbServer(self.sockfile) + + # Mock some Bdb attributes that are lazily created when tracing starts + self.pdb.botframe = None + self.pdb.quitting = False + + # Create a frame for testing + self.test_globals = {'a': 1, 'b': 2, '__pdb_convenience_variables': {'x': 100}} + self.test_locals = {'c': 3, 'd': 4} + + # Create a simple test frame + frame_info = unittest.mock.Mock() + frame_info.f_globals = self.test_globals + frame_info.f_locals = self.test_locals + frame_info.f_lineno = 42 + frame_info.f_code = unittest.mock.Mock() + frame_info.f_code.co_filename = "test_file.py" + frame_info.f_code.co_name = "test_function" + + self.pdb.curframe = frame_info + + def test_message_and_error(self): + """Test message and error methods send correct JSON.""" + self.pdb.message("Test message") + self.pdb.error("Test error") + + outputs = self.sockfile.get_output() + self.assertEqual(len(outputs), 2) + self.assertEqual(outputs[0], {"message": "Test message\n", "type": "info"}) + self.assertEqual(outputs[1], {"message": "Test error", "type": "error"}) + + def test_read_command(self): + """Test reading commands from the socket.""" + # Add test input + self.sockfile.add_input({"reply": "help"}) + + # Read the command + cmd = self.pdb._read_reply() + self.assertEqual(cmd, "help") + + def test_read_command_EOF(self): + """Test reading EOF command.""" + # Simulate socket closure + self.pdb._write_failed = True + with self.assertRaises(EOFError): + self.pdb._read_reply() + + def test_completion(self): + """Test handling completion requests.""" + # Mock completenames to return specific values + with unittest.mock.patch.object(self.pdb, 'completenames', + return_value=["continue", "clear"]): + + # Add a completion request + self.sockfile.add_input({ + "complete": { + "text": "c", + "line": "c", + "begidx": 0, + "endidx": 1 + } + }) + + # Add a regular command to break the loop + self.sockfile.add_input({"reply": "help"}) + + # Read command - this should process the completion request first + cmd = self.pdb._read_reply() + + # Verify completion response was sent + outputs = self.sockfile.get_output() + self.assertEqual(len(outputs), 1) + self.assertEqual(outputs[0], {"completions": ["continue", "clear"]}) + + # The actual command should be returned + self.assertEqual(cmd, "help") + + def test_do_help(self): + """Test that do_help sends the help message.""" + self.pdb.do_help("break") + + outputs = self.sockfile.get_output() + self.assertEqual(len(outputs), 1) + self.assertEqual(outputs[0], {"help": "break"}) + + def test_interact_mode(self): + """Test interaction mode setup and execution.""" + # First set up interact mode + self.pdb.do_interact("") + + # Verify _interact_state is properly initialized + self.assertIsNotNone(self.pdb._interact_state) + self.assertIsInstance(self.pdb._interact_state, dict) + + # Test running code in interact mode + with unittest.mock.patch.object(self.pdb, '_error_exc') as mock_error: + self.pdb._run_in_python_repl("print('test')") + mock_error.assert_not_called() + + # Test with syntax error + self.pdb._run_in_python_repl("if:") + mock_error.assert_called_once() + + def test_registering_commands(self): + """Test registering breakpoint commands.""" + # Mock get_bpbynumber + with unittest.mock.patch.object(self.pdb, 'get_bpbynumber'): + # Queue up some input to send + self.sockfile.add_input({"reply": "commands 1"}) + self.sockfile.add_input({"reply": "silent"}) + self.sockfile.add_input({"reply": "print('hi')"}) + self.sockfile.add_input({"reply": "end"}) + self.sockfile.add_input({"signal": "EOF"}) + + # Run the PDB command loop + self.pdb.cmdloop() + + outputs = self.sockfile.get_output() + self.assertIn('command_list', outputs[0]) + self.assertEqual(outputs[1], {"prompt": "(Pdb) ", "state": "pdb"}) + self.assertEqual(outputs[2], {"prompt": "(com) ", "state": "commands"}) + self.assertEqual(outputs[3], {"prompt": "(com) ", "state": "commands"}) + self.assertEqual(outputs[4], {"prompt": "(com) ", "state": "commands"}) + self.assertEqual(outputs[5], {"prompt": "(Pdb) ", "state": "pdb"}) + self.assertEqual(outputs[6], {"message": "\n", "type": "info"}) + self.assertEqual(len(outputs), 7) + + self.assertEqual( + self.pdb.commands[1], + ["_pdbcmd_silence_frame_status", "print('hi')"], + ) + + def test_detach(self): + """Test the detach method.""" + with unittest.mock.patch.object(self.sockfile, 'close') as mock_close: + self.pdb.detach() + mock_close.assert_called_once() + self.assertFalse(self.pdb.quitting) + + def test_cmdloop(self): + """Test the command loop with various commands.""" + # Mock onecmd to track command execution + with unittest.mock.patch.object(self.pdb, 'onecmd', return_value=False) as mock_onecmd: + # Add commands to the queue + self.pdb.cmdqueue = ['help', 'list'] + + # Add a command from the socket for when cmdqueue is empty + self.sockfile.add_input({"reply": "next"}) + + # Add a second command to break the loop + self.sockfile.add_input({"reply": "quit"}) + + # Configure onecmd to exit the loop on "quit" + def side_effect(line): + return line == 'quit' + mock_onecmd.side_effect = side_effect + + # Run the command loop + self.pdb.quitting = False # Set this by hand because we don't want to really call set_trace() + self.pdb.cmdloop() + + # Should have processed 4 commands: 2 from cmdqueue, 2 from socket + self.assertEqual(mock_onecmd.call_count, 4) + mock_onecmd.assert_any_call('help') + mock_onecmd.assert_any_call('list') + mock_onecmd.assert_any_call('next') + mock_onecmd.assert_any_call('quit') + + # Check if prompt was sent to client + outputs = self.sockfile.get_output() + prompts = [o for o in outputs if 'prompt' in o] + self.assertEqual(len(prompts), 2) # Should have sent 2 prompts + + +@requires_subprocess() +@unittest.skipIf(is_wasi, "WASI does not support TCP sockets") +class PdbConnectTestCase(unittest.TestCase): + """Tests for the _connect mechanism using direct socket communication.""" + + def setUp(self): + # Create a server socket that will wait for the debugger to connect + self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server_sock.bind(('127.0.0.1', 0)) # Let OS assign port + self.server_sock.listen(1) + self.port = self.server_sock.getsockname()[1] + + def _create_script(self, script=None): + # Create a file for subprocess script + if script is None: + script = textwrap.dedent( + f""" + import pdb + import sys + import time + + def foo(): + x = 42 + return bar() + + def bar(): + return 42 + + def connect_to_debugger(): + # Create a frame to debug + def dummy_function(): + x = 42 + # Call connect to establish connection + # with the test server + frame = sys._getframe() # Get the current frame + pdb._connect( + host='127.0.0.1', + port={self.port}, + frame=frame, + commands="", + version=pdb._PdbServer.protocol_version(), + signal_raising_thread=False, + colorize=False, + ) + return x # This line won't be reached in debugging + + return dummy_function() + + result = connect_to_debugger() + foo() + print(f"Function returned: {{result}}") + """) + + self.script_path = TESTFN + "_connect_test.py" + with open(self.script_path, 'w') as f: + f.write(script) + + def tearDown(self): + self.server_sock.close() + try: + unlink(self.script_path) + except OSError: + pass + + def _connect_and_get_client_file(self): + """Helper to start subprocess and get connected client file.""" + # Start the subprocess that will connect to our socket + process = subprocess.Popen( + [sys.executable, self.script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Accept the connection from the subprocess + client_sock, _ = self.server_sock.accept() + client_file = client_sock.makefile('rwb') + self.addCleanup(client_file.close) + self.addCleanup(client_sock.close) + + return process, client_file + + def _connect_and_get_client_file_via_pty(self): + """Like _connect_and_get_client_file, but run the target under a pty. + + With a real terminal on stdin/stdout, PyREPL is available *inside the + target process*, which is the condition that exercises pdb's PyREPL + input handling in the remote server. + """ + controller, worker = pty.openpty() + self.addCleanup(os.close, controller) + env = dict(os.environ, TERM="xterm-256color") + env.pop("PYTHON_BASIC_REPL", None) # don't opt out of PyREPL + process = subprocess.Popen( + [sys.executable, self.script_path], + stdin=worker, stdout=worker, stderr=worker, env=env, close_fds=True, + ) + os.close(worker) # only the child keeps the worker end open + + # Continuously drain the terminal so the child never blocks on a write. + drainer = threading.Thread( + target=self._drain_until_eof, args=(controller,), daemon=True + ) + drainer.start() + self.addCleanup(drainer.join, SHORT_TIMEOUT) + + client_sock, _ = self.server_sock.accept() + client_file = client_sock.makefile('rwb') + self.addCleanup(client_file.close) + self.addCleanup(client_sock.close) + + return process, client_file + + @staticmethod + def _drain_until_eof(fd): + try: + while os.read(fd, 1024): + pass + except OSError: + pass # controller closed, or the pty went away with the child + + def _read_until_prompt(self, client_file): + """Helper to read messages until a prompt is received.""" + messages = [] + while True: + data = client_file.readline() + if not data: + break + msg = json.loads(data.decode()) + messages.append(msg) + if 'prompt' in msg: + break + return messages + + def _send_command(self, client_file, command): + """Helper to send a command to the debugger.""" + client_file.write(json.dumps({"reply": command}).encode() + b"\n") + client_file.flush() + + def test_connect_and_basic_commands(self): + """Test connecting to a remote debugger and sending basic commands.""" + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + # We should receive initial data from the debugger + data = client_file.readline() + initial_data = json.loads(data.decode()) + self.assertIn('message', initial_data) + self.assertIn('pdb._connect', initial_data['message']) + + # First, look for command_list message + data = client_file.readline() + command_list = json.loads(data.decode()) + self.assertIn('command_list', command_list) + + # Then, look for the first prompt + data = client_file.readline() + prompt_data = json.loads(data.decode()) + self.assertIn('prompt', prompt_data) + self.assertEqual(prompt_data['state'], 'pdb') + + # Send 'bt' (backtrace) command + self._send_command(client_file, "bt") + + # Check for response - we should get some stack frames + messages = self._read_until_prompt(client_file) + + # Extract text messages containing stack info + text_msg = [msg['message'] for msg in messages + if 'message' in msg and 'connect_to_debugger' in msg['message']] + got_stack_info = bool(text_msg) + + expected_stacks = [ + "", + "connect_to_debugger", + ] + + for stack, msg in zip(expected_stacks, text_msg, strict=True): + self.assertIn(stack, msg) + + self.assertTrue(got_stack_info, "Should have received stack trace information") + + # Send 'c' (continue) command to let the program finish + self._send_command(client_file, "c") + + # Wait for process to finish + stdout, _ = process.communicate(timeout=SHORT_TIMEOUT) + + # Check if we got the expected output + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + + def test_breakpoints(self): + """Test setting and hitting breakpoints.""" + self._create_script() + process, client_file = self._connect_and_get_client_file() + with kill_on_error(process): + # Skip initial messages until we get to the prompt + self._read_until_prompt(client_file) + + # Set a breakpoint at the return statement + self._send_command(client_file, "break bar") + messages = self._read_until_prompt(client_file) + bp_msg = next(msg['message'] for msg in messages if 'message' in msg) + self.assertIn("Breakpoint", bp_msg) + + # Continue execution until breakpoint + self._send_command(client_file, "c") + messages = self._read_until_prompt(client_file) + + # Verify we hit the breakpoint + hit_msg = next(msg['message'] for msg in messages if 'message' in msg) + self.assertIn("bar()", hit_msg) + + # Check breakpoint list + self._send_command(client_file, "b") + messages = self._read_until_prompt(client_file) + list_msg = next(msg['message'] for msg in reversed(messages) if 'message' in msg) + self.assertIn("1 breakpoint", list_msg) + self.assertIn("breakpoint already hit 1 time", list_msg) + + # Clear breakpoint + self._send_command(client_file, "clear 1") + messages = self._read_until_prompt(client_file) + clear_msg = next(msg['message'] for msg in reversed(messages) if 'message' in msg) + self.assertIn("Deleted breakpoint", clear_msg) + + # Continue to end + self._send_command(client_file, "c") + stdout, _ = process.communicate(timeout=SHORT_TIMEOUT) + + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + + def test_keyboard_interrupt(self): + """Test that sending keyboard interrupt breaks into pdb.""" + + script = textwrap.dedent(f""" + import time + import sys + import socket + import pdb + def bar(): + frame = sys._getframe() # Get the current frame + pdb._connect( + host='127.0.0.1', + port={self.port}, + frame=frame, + commands="", + version=pdb._PdbServer.protocol_version(), + signal_raising_thread=True, + colorize=False, + ) + print("Connected to debugger") + iterations = 50 + while iterations > 0: + print("Iteration", iterations, flush=True) + time.sleep(0.2) + iterations -= 1 + return 42 + + if __name__ == "__main__": + print("Function returned:", bar()) + """) + self._create_script(script=script) + process, client_file = self._connect_and_get_client_file() + + # Accept a 2nd connection from the subprocess to tell it about signals + signal_sock, _ = self.server_sock.accept() + self.addCleanup(signal_sock.close) + + with kill_on_error(process): + # Skip initial messages until we get to the prompt + self._read_until_prompt(client_file) + + # Continue execution + self._send_command(client_file, "c") + + # Confirm that the remote is already in the while loop. We know + # it's in bar() and we can exit the loop immediately by setting + # iterations to 0. + while line := process.stdout.readline(): + if line.startswith("Iteration"): + break + + # Inject a script to interrupt the running process + signal_sock.sendall(signal.SIGINT.to_bytes()) + messages = self._read_until_prompt(client_file) + + # Verify we got the keyboard interrupt message. + interrupt_msgs = [msg['message'] for msg in messages if 'message' in msg] + expected_msg = [msg for msg in interrupt_msgs if "bar()" in msg] + self.assertGreater(len(expected_msg), 0) + + # Continue to end as fast as we can + self._send_command(client_file, "iterations = 0") + self._send_command(client_file, "c") + stdout, _ = process.communicate(timeout=SHORT_TIMEOUT) + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + + def test_handle_eof(self): + """Test that EOF signal properly exits the debugger.""" + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + # Skip initial messages until we get to the prompt + self._read_until_prompt(client_file) + + # Send EOF signal to exit the debugger + client_file.write(json.dumps({"signal": "EOF"}).encode() + b"\n") + client_file.flush() + + # The process should complete normally after receiving EOF + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) + + # Verify process completed correctly + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + self.assertEqual(stderr, "") + + def test_colorized_list_has_no_caret_encoded_newlines(self): + """A colorized ``list`` must not append "^J" to each source line. + + The remote server colorizes the source it sends to the client. The + colorizer renders control characters in caret notation, so a source + line's trailing newline has to be stripped *before* it is colorized; + otherwise every listed line ends with a spurious "^J". ``where`` was + unaffected because it strips the line before colorizing. See + gh-154470. + """ + # colorize=True is what attaching from a color-capable terminal passes + # to the server, and it is what makes the server colorize ``list``. + script = textwrap.dedent(f""" + import pdb, sys + def helper(): + x = 42 + return x + def connect(): + frame = sys._getframe() + pdb._connect( + host='127.0.0.1', + port={self.port}, + frame=frame, + commands="", + version=pdb._PdbServer.protocol_version(), + signal_raising_thread=False, + colorize=True, + ) + return helper() + connect() + """) + self._create_script(script=script) + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + self._read_until_prompt(client_file) + self._send_command(client_file, "l 1, 15") + messages = self._read_until_prompt(client_file) + source = "".join(m["message"] for m in messages if "message" in m) + + # Sanity: we really did receive colorized source ... + self.assertIn("helper", source) + self.assertIn("\x1b[", source) # ANSI color escapes are present + # ... and no trailing newline leaked through as caret notation. + self.assertNotIn("^J", source) + self._send_command(client_file, "c") + process.wait(timeout=SHORT_TIMEOUT) + self.assertEqual(process.returncode, 0) + + @unittest.skipUnless(pty, "requires pty") + def test_prompt_with_interactive_terminal(self): + """The server must send "(Pdb) " even when the target owns a terminal. + + The remote server transmits its prompt string to the client, which + displays it. When the target process has an interactive terminal, + PyREPL is enabled inside it; the base pdb machinery then blanks + ``self.prompt`` (a local PyREPL would draw the prompt itself). The + remote server has no local PyREPL -- it reads input from the socket -- + so it must keep the real prompt rather than transmit an empty one. + Regression test for gh-154467, where attaching with ``pdb -p`` to a + process running at an interactive prompt showed a blank prompt. + """ + self._create_script() + process, client_file = self._connect_and_get_client_file_via_pty() + + with kill_on_error(process): + messages = self._read_until_prompt(client_file) + # The message that ended the read is the prompt request. + self.assertEqual(messages[-1], {"prompt": "(Pdb) ", "state": "pdb"}) + + # Let the target run to completion so nothing is left attached. + self._send_command(client_file, "c") + process.wait(timeout=SHORT_TIMEOUT) + self.assertEqual(process.returncode, 0) + + def test_protocol_version(self): + """Test that incompatible protocol versions are properly detected.""" + # Create a script using an incompatible protocol version + script = textwrap.dedent(f''' + import sys + import pdb + + def run_test(): + frame = sys._getframe() + + # Use a fake version number that's definitely incompatible + fake_version = 0x01010101 # A fake version that doesn't match any real Python version + + # Connect with the wrong version + pdb._connect( + host='127.0.0.1', + port={self.port}, + frame=frame, + commands="", + version=fake_version, + signal_raising_thread=False, + colorize=False, + ) + + # This should print if the debugger detaches correctly + print("Debugger properly detected version mismatch") + return True + + if __name__ == "__main__": + print("Test result:", run_test()) + ''') + self._create_script(script=script) + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + # First message should be an error about protocol version mismatch + data = client_file.readline() + message = json.loads(data.decode()) + + self.assertIn('message', message) + self.assertEqual(message['type'], 'error') + self.assertIn('incompatible', message['message']) + self.assertIn('protocol version', message['message']) + + # The process should complete normally + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) + + # Verify the process completed successfully + self.assertIn("Test result: True", stdout) + self.assertIn("Debugger properly detected version mismatch", stdout) + self.assertEqual(process.returncode, 0) + + def test_help_system(self): + """Test that the help system properly sends help text to the client.""" + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + # Skip initial messages until we get to the prompt + self._read_until_prompt(client_file) + + # Request help for different commands + help_commands = ["help", "help break", "help continue", "help pdb"] + + for cmd in help_commands: + self._send_command(client_file, cmd) + + # Look for help message + data = client_file.readline() + message = json.loads(data.decode()) + + self.assertIn('help', message) + + if cmd == "help": + # Should just contain the command itself + self.assertEqual(message['help'], "") + else: + # Should contain the specific command we asked for help with + command = cmd.split()[1] + self.assertEqual(message['help'], command) + + # Skip to the next prompt + self._read_until_prompt(client_file) + + # Continue execution to finish the program + self._send_command(client_file, "c") + + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + + def test_multi_line_commands(self): + """Test that multi-line commands work properly over remote connection.""" + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + # Skip initial messages until we get to the prompt + self._read_until_prompt(client_file) + + # Send a multi-line command + multi_line_commands = [ + # Define a function + "def test_func():\n return 42", + + # For loop + "for i in range(3):\n print(i)", + + # If statement + "if True:\n x = 42\nelse:\n x = 0", + + # Try/except + "try:\n result = 10/2\n print(result)\nexcept ZeroDivisionError:\n print('Error')", + + # Class definition + "class TestClass:\n def __init__(self):\n self.value = 100\n def get_value(self):\n return self.value" + ] + + for cmd in multi_line_commands: + self._send_command(client_file, cmd) + self._read_until_prompt(client_file) + + # Test executing the defined function + self._send_command(client_file, "test_func()") + messages = self._read_until_prompt(client_file) + + # Find the result message + result_msg = next(msg['message'] for msg in messages if 'message' in msg) + self.assertIn("42", result_msg) + + # Test creating an instance of the defined class + self._send_command(client_file, "obj = TestClass()") + self._read_until_prompt(client_file) + + # Test calling a method on the instance + self._send_command(client_file, "obj.get_value()") + messages = self._read_until_prompt(client_file) + + # Find the result message + result_msg = next(msg['message'] for msg in messages if 'message' in msg) + self.assertIn("100", result_msg) + + # Continue execution to finish + self._send_command(client_file, "c") + + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) + self.assertIn("Function returned: 42", stdout) + self.assertEqual(process.returncode, 0) + + def test_exec_in_closure_result_uses_pdb_stdout(self): + """ + Expression results executed via _exec_in_closure() should be written + to the debugger output stream (pdb stdout), not to sys.stdout. + """ + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + self._read_until_prompt(client_file) + + self._send_command(client_file, "(lambda: 123)()") + messages = self._read_until_prompt(client_file) + result_msg = "".join(msg.get("message", "") for msg in messages) + self.assertIn("123", result_msg) + + self._send_command(client_file, "sum(i for i in (1, 2, 3))") + messages = self._read_until_prompt(client_file) + result_msg = "".join(msg.get("message", "") for msg in messages) + self.assertIn("6", result_msg) + + self._send_command(client_file, "c") + stdout, _ = process.communicate(timeout=SHORT_TIMEOUT) + + self.assertNotIn("\n123\n", stdout) + self.assertNotIn("\n6\n", stdout) + self.assertEqual(process.returncode, 0) + + +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False + + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass + + return PROCESS_VM_READV_SUPPORTED + + +@unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") +@unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", + "Test only runs on Linux, Windows and MacOS") +@unittest.skipIf(sys.platform == "linux" and not _supports_remote_attaching(), + "Testing on Linux requires process_vm_readv support") +@cpython_only +@requires_subprocess() +class PdbAttachTestCase(unittest.TestCase): + def setUp(self): + # Create a server socket that will wait for the debugger to connect + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.bind(('127.0.0.1', 0)) # Let OS assign port + self.sock.listen(1) + self.port = self.sock.getsockname()[1] + self._create_script() + + def _create_script(self, script=None): + # Create a file for subprocess script + script = textwrap.dedent( + f""" + import socket + import time + + def foo(): + return bar() + + def bar(): + return baz() + + def baz(): + x = 1 + # Trigger attach + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('127.0.0.1', {self.port})) + sock.close() + count = 0 + while x == 1 and count < 100: + count += 1 + time.sleep(0.1) + return x + + result = foo() + print(f"Function returned: {{result}}") + """ + ) + + self.script_path = TESTFN + "_connect_test.py" + with open(self.script_path, 'w') as f: + f.write(script) + + def tearDown(self): + self.sock.close() + try: + unlink(self.script_path) + except OSError: + pass + + def do_integration_test(self, client_stdin): + process = subprocess.Popen( + [sys.executable, self.script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + self.addCleanup(process.stdout.close) + self.addCleanup(process.stderr.close) + + # Wait for the process to reach our attachment point + self.sock.settimeout(10) + conn, _ = self.sock.accept() + conn.close() + + client_stdin = io.StringIO(client_stdin) + client_stdout = io.StringIO() + client_stderr = io.StringIO() + + self.addCleanup(client_stdin.close) + self.addCleanup(client_stdout.close) + self.addCleanup(client_stderr.close) + self.addCleanup(process.wait) + + with ( + unittest.mock.patch("sys.stdin", client_stdin), + redirect_stdout(client_stdout), + redirect_stderr(client_stderr), + unittest.mock.patch("sys.argv", ["pdb", "-p", str(process.pid)]), + unittest.mock.patch( + "pdb.exit_with_permission_help_text", side_effect=PermissionError + ), + ): + try: + pdb.main() + except PermissionError: + self.skipTest("Insufficient permissions for remote execution") + + process.wait() + server_stdout = process.stdout.read() + server_stderr = process.stderr.read() + + if process.returncode != 0: + print("server failed") + print(f"server stdout:\n{server_stdout}") + print(f"server stderr:\n{server_stderr}") + + self.assertEqual(process.returncode, 0) + return { + "client": { + "stdout": client_stdout.getvalue(), + "stderr": client_stderr.getvalue(), + }, + "server": { + "stdout": server_stdout, + "stderr": server_stderr, + }, + } + + def test_attach_to_process_without_colors(self): + with force_color(False): + output = self.do_integration_test("ll\nx=42\n") + self.assertEqual(output["client"]["stderr"], "") + self.assertEqual(output["server"]["stderr"], "") + + self.assertEqual(output["server"]["stdout"], "Function returned: 42\n") + self.assertIn("while x == 1", output["client"]["stdout"]) + self.assertNotIn("\x1b", output["client"]["stdout"]) + + def test_attach_to_process_with_colors(self): + with force_color(True): + output = self.do_integration_test("ll\nx=42\n") + self.assertEqual(output["client"]["stderr"], "") + self.assertEqual(output["server"]["stderr"], "") + + self.assertEqual(output["server"]["stdout"], "Function returned: 42\n") + self.assertIn("\x1b", output["client"]["stdout"]) + self.assertNotIn("while x == 1", output["client"]["stdout"]) + self.assertIn("while x == 1", re.sub("\x1b[^m]*m", "", output["client"]["stdout"])) + + def test_attach_to_non_existent_process(self): + with force_color(False): + result = subprocess.run([sys.executable, "-m", "pdb", "-p", "999999"], text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0) + if sys.platform == "darwin": + # On MacOS, attaching to a non-existent process gives PermissionError + error = "The specified process cannot be attached to due to insufficient permissions" + else: + error = "Cannot attach to pid 999999, please make sure that the process exists" + self.assertIn(error, result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 7469a1047..849a2afd8 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,9 +1,10 @@ """ Tests for the internal type cache in CPython. """ +import collections.abc import dis import unittest import warnings from test import support -from test.support import import_helper, requires_specialization, requires_specialization_ft +from test.support import import_helper, requires_specialization try: from sys import _clear_type_cache except ImportError: @@ -114,6 +115,25 @@ class TypeCacheTests(unittest.TestCase): Holder.set_value() HolderSub.value + def test_abc_register_invalidates_subclass_versions(self): + class Parent: + pass + + class Child(Parent): + pass + + type_assign_version(Parent) + type_assign_version(Child) + parent_version = type_get_version(Parent) + child_version = type_get_version(Child) + if parent_version == 0 or child_version == 0: + self.skipTest("Could not assign valid type versions") + + collections.abc.Mapping.register(Parent) + + self.assertEqual(type_get_version(Parent), 0) + self.assertEqual(type_get_version(Child), 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): @@ -219,7 +239,7 @@ class TypeCacheWithSpecializationTests(unittest.TestCase): self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) - @requires_specialization_ft + @requires_specialization def test_class_call_specialization_user_type(self): class F: def __init__(self): diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index d015267ce..17db68694 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -259,27 +259,6 @@ class OtherNetworkTests(unittest.TestCase): opener.open(request) self.assertEqual(request.get_header('User-agent'),'Test-Agent') - @unittest.skip('XXX: http://www.imdb.com is gone') - def test_sites_no_connection_close(self): - # Some sites do not send Connection: close header. - # Verify that those work properly. (#issue12576) - - URL = 'http://www.imdb.com' # mangles Connection:close - - with socket_helper.transient_internet(URL): - try: - with urllib.request.urlopen(URL) as res: - pass - except ValueError: - self.fail("urlopen failed for site not sending \ - Connection:close") - else: - self.assertTrue(res) - - req = urllib.request.urlopen(URL) - res = req.read() - self.assertTrue(res) - def _test_urls(self, urls, handlers, retry=True): import time import logging diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py old mode 100644 new mode 100755 diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index a9cd1ecad..beb32d0df 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -18,7 +18,7 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of - # the implementation. need to override __init__ and __str__. + # the implementation. It overrides __init__ and __str__. # It sets self.args for compatibility with other OSError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index f8fe89f6e..b82862dee 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -69,7 +69,8 @@ def read_environ(): # Python 3's http.server.CGIHTTPRequestHandler decodes # using the urllib.unquote default of UTF-8, amongst other - # issues. + # issues. While the CGI handler is removed in 3.15, this + # is kept for legacy reasons. elif ( software.startswith('simplehttp/') and 'python/3' in software