diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index f7bcba064..09b911478 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -10,6 +10,7 @@ import unittest import weakref from test import support +from test.support import threading_helper def _wait(): @@ -31,7 +32,7 @@ class Bunch(object): self.started = [] self.finished = [] self._can_exit = not wait_before_exit - self.wait_thread = support.wait_threads_exit() + self.wait_thread = threading_helper.wait_threads_exit() self.wait_thread.__enter__() def task(): @@ -67,10 +68,10 @@ class Bunch(object): class BaseTestCase(unittest.TestCase): def setUp(self): - self._threads = support.threading_setup() + self._threads = threading_helper.threading_setup() def tearDown(self): - support.threading_cleanup(*self._threads) + threading_helper.threading_cleanup(*self._threads) support.reap_children() def assertTimeout(self, actual, expected): @@ -235,7 +236,7 @@ class LockTests(BaseLockTests): lock.acquire() phase.append(None) - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): start_new_thread(f, ()) while len(phase) == 0: _wait() diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 11514dfa4..8a4de7a4f 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -25,10 +25,11 @@ from test import support from test.support import os_helper from test.support import ( TestFailed, run_with_locale, no_tracing, - _2G, _4G, bigmemtest, reap_threads + _2G, _4G, bigmemtest ) from test.support.import_helper import forget from test.support.os_helper import TESTFN +from test.support import threading_helper from test.support.warnings_helper import save_restore_warnings_filters from pickle import bytes_types @@ -1378,7 +1379,7 @@ class AbstractUnpickleTests: for p in badpickles: self.check_unpickling_error(self.truncated_errors, p) - @reap_threads + @threading_helper.reap_threads def test_unpickle_module_race(self): # https://bugs.python.org/issue34572 locker_module = dedent(""" diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index dab791bf6..667fafefc 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -104,8 +104,6 @@ __all__ = [ 'temp_umask', "reap_children", # logging "TestHandler", - # threads - "threading_setup", "threading_cleanup", "reap_threads", "start_threads", # miscellaneous "check_warnings", "check_no_resource_warning", "check_no_warnings", "EnvironmentVarGuard", @@ -1541,104 +1539,6 @@ print_warning.orig_stderr = sys.stderr # to cleanup threads. environment_altered = False -# NOTE: we use thread._count() rather than threading.enumerate() (or the -# moral equivalent thereof) because a threading.Thread object is still alive -# until its __bootstrap() method has returned, even after it has been -# unregistered from the threading module. -# thread._count(), on the other hand, only gets decremented *after* the -# __bootstrap() method has returned, which gives us reliable reference counts -# at the end of a test run. - -def threading_setup(): - return _thread._count(), threading._dangling.copy() - -def threading_cleanup(*original_values): - global environment_altered - - _MAX_COUNT = 100 - - for count in range(_MAX_COUNT): - values = _thread._count(), threading._dangling - if values == original_values: - break - - if not count: # Display a warning at the first iteration - environment_altered = True - dangling_threads = values[1] - print_warning(f"threading_cleanup() failed to cleanup " - f"{values[0] - original_values[0]} threads " - f"(count: {values[0]}, " - f"dangling: {len(dangling_threads)})") - for thread in dangling_threads: - print_warning(f"Dangling thread: {thread!r}") - - # Don't hold references to threads - dangling_threads = None - values = None - - time.sleep(0.01) - gc_collect() - - -def reap_threads(func): - """Use this function when threads are being used. This will - ensure that the threads are cleaned up even when the test fails. - """ - @functools.wraps(func) - def decorator(*args): - key = threading_setup() - try: - return func(*args) - finally: - threading_cleanup(*key) - return decorator - - -@contextlib.contextmanager -def wait_threads_exit(timeout=60.0): - """ - bpo-31234: Context manager to wait until all threads created in the with - statement exit. - - Use _thread.count() to check if threads exited. Indirectly, wait until - threads exit the internal t_bootstrap() C function of the _thread module. - - threading_setup() and threading_cleanup() are designed to emit a warning - if a test leaves running threads in the background. This context manager - is designed to cleanup threads started by the _thread.start_new_thread() - which doesn't allow to wait for thread exit, whereas thread.Thread has a - join() method. - """ - old_count = _thread._count() - try: - yield - finally: - start_time = time.monotonic() - deadline = start_time + timeout - while True: - count = _thread._count() - if count <= old_count: - break - if time.monotonic() > deadline: - dt = time.monotonic() - start_time - msg = (f"wait_threads() failed to cleanup {count - old_count} " - f"threads after {dt:.1f} seconds " - f"(count: {count}, old count: {old_count})") - raise AssertionError(msg) - time.sleep(0.010) - gc_collect() - - -def join_thread(thread, timeout=30.0): - """Join a thread. Raise an AssertionError if the thread is still alive - after timeout seconds. - """ - thread.join(timeout) - if thread.is_alive(): - msg = f"failed to join the thread in {timeout:.1f} seconds" - raise AssertionError(msg) - - def reap_children(): """Use this function at the end of test_main() whenever sub-processes are started. This will help ensure that no extra children (zombies) @@ -1667,42 +1567,6 @@ def reap_children(): environment_altered = True -@contextlib.contextmanager -def start_threads(threads, unlock=None): - threads = list(threads) - started = [] - try: - try: - for t in threads: - t.start() - started.append(t) - except: - if verbose: - print("Can't start %d threads, only %d threads started" % - (len(threads), len(started))) - raise - yield - finally: - try: - if unlock: - unlock() - endtime = starttime = time.monotonic() - for timeout in range(1, 16): - endtime += 60 - for t in started: - t.join(max(endtime - time.monotonic(), 0.01)) - started = [t for t in started if t.is_alive()] - if not started: - break - if verbose: - print('Unable to join %d threads during a period of ' - '%d minutes' % (len(started), timeout)) - finally: - started = [t for t in started if t.is_alive()] - if started: - faulthandler.dump_traceback(sys.stdout) - raise AssertionError('Unable to join %d threads' % len(started)) - @contextlib.contextmanager def swap_attr(obj, attr, new_val): """Temporary swap out an attribute with a new object. @@ -2507,63 +2371,6 @@ class catch_unraisable_exception: del self.unraisable -class catch_threading_exception: - """ - Context manager catching threading.Thread exception using - threading.excepthook. - - Attributes set when an exception is catched: - - * exc_type - * exc_value - * exc_traceback - * thread - - See threading.excepthook() documentation for these attributes. - - These attributes are deleted at the context manager exit. - - Usage: - - with support.catch_threading_exception() as cm: - # code spawning a thread which raises an exception - ... - - # check the thread exception, use cm attributes: - # exc_type, exc_value, exc_traceback, thread - ... - - # exc_type, exc_value, exc_traceback, thread attributes of cm no longer - # exists at this point - # (to avoid reference cycles) - """ - - def __init__(self): - self.exc_type = None - self.exc_value = None - self.exc_traceback = None - self.thread = None - self._old_hook = None - - def _hook(self, args): - self.exc_type = args.exc_type - self.exc_value = args.exc_value - self.exc_traceback = args.exc_traceback - self.thread = args.thread - - def __enter__(self): - self._old_hook = threading.excepthook - threading.excepthook = self._hook - return self - - def __exit__(self, *exc_info): - threading.excepthook = self._old_hook - del self.exc_type - del self.exc_value - del self.exc_traceback - del self.thread - - def wait_process(pid, *, exitcode, timeout=None): """ Wait until process pid completes and check that the process exit code is diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py new file mode 100644 index 000000000..92a64e835 --- /dev/null +++ b/Lib/test/support/threading_helper.py @@ -0,0 +1,209 @@ +import _thread +import contextlib +import functools +import sys +import threading +import time + +from test import support + + +#======================================================================= +# Threading support to prevent reporting refleaks when running regrtest.py -R + +# NOTE: we use thread._count() rather than threading.enumerate() (or the +# moral equivalent thereof) because a threading.Thread object is still alive +# until its __bootstrap() method has returned, even after it has been +# unregistered from the threading module. +# thread._count(), on the other hand, only gets decremented *after* the +# __bootstrap() method has returned, which gives us reliable reference counts +# at the end of a test run. + + +def threading_setup(): + return _thread._count(), threading._dangling.copy() + + +def threading_cleanup(*original_values): + _MAX_COUNT = 100 + + for count in range(_MAX_COUNT): + values = _thread._count(), threading._dangling + if values == original_values: + break + + if not count: + # Display a warning at the first iteration + support.environment_altered = True + dangling_threads = values[1] + support.print_warning(f"threading_cleanup() failed to cleanup " + f"{values[0] - original_values[0]} threads " + f"(count: {values[0]}, " + f"dangling: {len(dangling_threads)})") + for thread in dangling_threads: + support.print_warning(f"Dangling thread: {thread!r}") + + # Don't hold references to threads + dangling_threads = None + values = None + + time.sleep(0.01) + support.gc_collect() + + +def reap_threads(func): + """Use this function when threads are being used. This will + ensure that the threads are cleaned up even when the test fails. + """ + @functools.wraps(func) + def decorator(*args): + key = threading_setup() + try: + return func(*args) + finally: + threading_cleanup(*key) + return decorator + + +@contextlib.contextmanager +def wait_threads_exit(timeout=None): + """ + bpo-31234: Context manager to wait until all threads created in the with + statement exit. + + Use _thread.count() to check if threads exited. Indirectly, wait until + threads exit the internal t_bootstrap() C function of the _thread module. + + threading_setup() and threading_cleanup() are designed to emit a warning + if a test leaves running threads in the background. This context manager + is designed to cleanup threads started by the _thread.start_new_thread() + which doesn't allow to wait for thread exit, whereas thread.Thread has a + join() method. + """ + if timeout is None: + timeout = support.SHORT_TIMEOUT + old_count = _thread._count() + try: + yield + finally: + start_time = time.monotonic() + deadline = start_time + timeout + while True: + count = _thread._count() + if count <= old_count: + break + if time.monotonic() > deadline: + dt = time.monotonic() - start_time + msg = (f"wait_threads() failed to cleanup {count - old_count} " + f"threads after {dt:.1f} seconds " + f"(count: {count}, old count: {old_count})") + raise AssertionError(msg) + time.sleep(0.010) + support.gc_collect() + + +def join_thread(thread, timeout=None): + """Join a thread. Raise an AssertionError if the thread is still alive + after timeout seconds. + """ + if timeout is None: + timeout = support.SHORT_TIMEOUT + thread.join(timeout) + if thread.is_alive(): + msg = f"failed to join the thread in {timeout:.1f} seconds" + raise AssertionError(msg) + + +@contextlib.contextmanager +def start_threads(threads, unlock=None): + import faulthandler + threads = list(threads) + started = [] + try: + try: + for t in threads: + t.start() + started.append(t) + except: + if support.verbose: + print("Can't start %d threads, only %d threads started" % + (len(threads), len(started))) + raise + yield + finally: + try: + if unlock: + unlock() + endtime = time.monotonic() + for timeout in range(1, 16): + endtime += 60 + for t in started: + t.join(max(endtime - time.monotonic(), 0.01)) + started = [t for t in started if t.is_alive()] + if not started: + break + if support.verbose: + print('Unable to join %d threads during a period of ' + '%d minutes' % (len(started), timeout)) + finally: + started = [t for t in started if t.is_alive()] + if started: + faulthandler.dump_traceback(sys.stdout) + raise AssertionError('Unable to join %d threads' % len(started)) + + +class catch_threading_exception: + """ + Context manager catching threading.Thread exception using + threading.excepthook. + + Attributes set when an exception is caught: + + * exc_type + * exc_value + * exc_traceback + * thread + + See threading.excepthook() documentation for these attributes. + + These attributes are deleted at the context manager exit. + + Usage: + + with threading_helper.catch_threading_exception() as cm: + # code spawning a thread which raises an exception + ... + + # check the thread exception, use cm attributes: + # exc_type, exc_value, exc_traceback, thread + ... + + # exc_type, exc_value, exc_traceback, thread attributes of cm no longer + # exists at this point + # (to avoid reference cycles) + """ + + def __init__(self): + self.exc_type = None + self.exc_value = None + self.exc_traceback = None + self.thread = None + self._old_hook = None + + def _hook(self, args): + self.exc_type = args.exc_type + self.exc_value = args.exc_value + self.exc_traceback = args.exc_traceback + self.thread = args.thread + + def __enter__(self): + self._old_hook = threading.excepthook + threading.excepthook = self._hook + return self + + def __exit__(self, *exc_info): + threading.excepthook = self._old_hook + del self.exc_type + del self.exc_value + del self.exc_traceback + del self.thread diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index e9a1a40fd..1fcc882ce 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -2,6 +2,7 @@ from test import support from test.support import socket_helper +from test.support import threading_helper import asynchat @@ -105,10 +106,10 @@ class TestAsynchat(unittest.TestCase): usepoll = False def setUp(self): - self._threads = support.threading_setup() + self._threads = threading_helper.threading_setup() def tearDown(self): - support.threading_cleanup(*self._threads) + threading_helper.threading_cleanup(*self._threads) def line_terminator_check(self, term, server_chunk): event = threading.Event() @@ -124,7 +125,7 @@ class TestAsynchat(unittest.TestCase): c.push(b"I'm not dead yet!" + term) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -155,7 +156,7 @@ class TestAsynchat(unittest.TestCase): c.push(data) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [data[:termlen]]) @@ -175,7 +176,7 @@ class TestAsynchat(unittest.TestCase): c.push(data) c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, []) self.assertEqual(c.buffer, data) @@ -187,7 +188,7 @@ class TestAsynchat(unittest.TestCase): p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8) c.push_with_producer(p) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -197,7 +198,7 @@ class TestAsynchat(unittest.TestCase): data = b"hello world\nI'm not dead yet!\n" c.push_with_producer(data+SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) @@ -208,7 +209,7 @@ class TestAsynchat(unittest.TestCase): c.push(b"hello world\n\nI'm not dead yet!\n") c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [b"hello world", b"", b"I'm not dead yet!"]) @@ -227,7 +228,7 @@ class TestAsynchat(unittest.TestCase): # where the server echoes all of its data before we can check that it # got any down below. s.start_resend_event.set() - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, []) # the server might have been able to send a byte or two back, but this @@ -248,7 +249,7 @@ class TestAsynchat(unittest.TestCase): self.assertRaises(TypeError, c.push, 'unicode') c.push(SERVER_QUIT) asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - support.join_thread(s, timeout=TIMEOUT) + threading_helper.join_thread(s) self.assertEqual(c.contents, [b'bytes', b'bytes', b'bytes']) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index e13104a12..a58744823 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -12,6 +12,7 @@ import threading from test import support from test.support import os_helper from test.support import socket_helper +from test.support import threading_helper from io import BytesIO if support.PGO: @@ -325,7 +326,7 @@ class DispatcherWithSendTests(unittest.TestCase): def tearDown(self): asyncore.close_all() - @support.reap_threads + @threading_helper.reap_threads def test_send(self): evt = threading.Event() sock = socket.socket() @@ -362,7 +363,7 @@ class DispatcherWithSendTests(unittest.TestCase): self.assertEqual(cap.getvalue(), data*2) finally: - support.join_thread(t, timeout=TIMEOUT) + threading_helper.join_thread(t) @unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), @@ -768,7 +769,7 @@ class BaseTestAPI: self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) - @support.reap_threads + @threading_helper.reap_threads def test_quick_connect(self): # see: http://bugs.python.org/issue10340 if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): @@ -790,7 +791,7 @@ class BaseTestAPI: except OSError: pass finally: - support.join_thread(t, timeout=TIMEOUT) + threading_helper.join_thread(t) class TestAPI_UseIPv4Sockets(BaseTestAPI): family = socket.AF_INET diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 7b5c1d8b5..5e56bec46 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -8,7 +8,7 @@ from collections import OrderedDict from enum import Enum, IntEnum, EnumMeta, Flag, IntFlag, unique, auto from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL -from test import support +from test.support import ALWAYS_EQ, check__all__, threading_helper from datetime import timedelta @@ -2489,7 +2489,7 @@ class TestFlag(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading") - @support.reap_threads + @threading_helper.reap_threads def test_unique_composite(self): # override __eq__ to be identity only class TestFlag(Flag): @@ -2519,7 +2519,7 @@ class TestFlag(unittest.TestCase): threading.Thread(target=cycle_enum) for _ in range(8) ] - with support.start_threads(threads): + with threading_helper.start_threads(threads): pass # check that only 248 members were created self.assertFalse( @@ -2922,7 +2922,7 @@ class TestIntFlag(unittest.TestCase): # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading") - @support.reap_threads + @threading_helper.reap_threads def test_unique_composite(self): # override __eq__ to be identity only class TestFlag(IntFlag): @@ -2952,7 +2952,7 @@ class TestIntFlag(unittest.TestCase): threading.Thread(target=cycle_enum) for _ in range(8) ] - with support.start_threads(threads): + with threading_helper.start_threads(threads): pass # check that only 248 members were created self.assertFalse( @@ -3183,7 +3183,7 @@ class TestStdLib(unittest.TestCase): class MiscTestCase(unittest.TestCase): def test__all__(self): - support.check__all__(self, enum) + check__all__(self, enum) # These are unordered here on purpose to ensure that declaration order diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 848e9cead..d48b9bea3 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -20,6 +20,7 @@ except ImportError: from unittest import TestCase, skipUnless from test import support +from test.support import threading_helper from test.support import socket_helper from test.support.socket_helper import HOST, HOSTv6 @@ -1094,11 +1095,11 @@ def test_main(): TestTLS_FTPClassMixin, TestTLS_FTPClass, MiscTestCase] - thread_info = support.threading_setup() + thread_info = threading_helper.threading_setup() try: support.run_unittest(*tests) finally: - support.threading_cleanup(*thread_info) + threading_helper.threading_cleanup(*thread_info) if __name__ == '__main__': diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index b4ff23750..7e9e96378 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -8,7 +8,6 @@ import pickle from random import choice import sys from test import support -from test.support import import_helper import threading import time import typing @@ -17,6 +16,9 @@ import unittest.mock from weakref import proxy import contextlib +from test.support import import_helper +from test.support import threading_helper + import functools py_functools = import_helper.import_fresh_module('functools', blocked=['_functools']) @@ -1570,7 +1572,7 @@ class TestLRU: # create n threads in order to fill cache threads = [threading.Thread(target=full, args=[k]) for k in range(n)] - with support.start_threads(threads): + with threading_helper.start_threads(threads): start.set() hits, misses, maxsize, currsize = f.cache_info() @@ -1588,7 +1590,7 @@ class TestLRU: threads += [threading.Thread(target=full, args=[k]) for k in range(n)] start.clear() - with support.start_threads(threads): + with threading_helper.start_threads(threads): start.set() finally: sys.setswitchinterval(orig_si) @@ -1610,7 +1612,7 @@ class TestLRU: self.assertEqual(f(i), 3 * i) stop.wait(10) threads = [threading.Thread(target=test) for k in range(n)] - with support.start_threads(threads): + with threading_helper.start_threads(threads): for i in range(m): start.wait(10) stop.reset() @@ -1630,7 +1632,7 @@ class TestLRU: self.assertEqual(f(x), 3 * x, i) threads = [threading.Thread(target=test, args=(i, v)) for i, v in enumerate([1, 2, 2, 3, 2])] - with support.start_threads(threads): + with threading_helper.start_threads(threads): pass def test_need_for_rlock(self): @@ -2526,7 +2528,7 @@ class TestCachedProperty(unittest.TestCase): threading.Thread(target=lambda: item.cost) for k in range(num_threads) ] - with support.start_threads(threads): + with threading_helper.start_threads(threads): go.set() finally: sys.setswitchinterval(orig_si) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index a6a55be19..054e0ab95 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -20,6 +20,7 @@ import warnings from test import support from test.support import _4G, bigmemtest from test.support.import_helper import import_fresh_module +from test.support import threading_helper from http.client import HTTPException # Were we compiled --with-pydebug or with #define Py_DEBUG? @@ -898,7 +899,7 @@ class HashLibTestCase(unittest.TestCase): '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94' ) - @support.reap_threads + @threading_helper.reap_threads def test_threaded_hashing(self): # Updating the same hash object from several threads at once # using data chunk sizes containing the same byte sequences. diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 2a4587b9d..28a1ec372 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -30,6 +30,7 @@ from io import BytesIO import unittest from test import support from test.support import os_helper +from test.support import threading_helper class NoLogRequestHandler: @@ -64,7 +65,7 @@ class TestServerThread(threading.Thread): class BaseTestCase(unittest.TestCase): def setUp(self): - self._threads = support.threading_setup() + self._threads = threading_helper.threading_setup() os.environ = os_helper.EnvironmentVarGuard() self.server_started = threading.Event() self.thread = TestServerThread(self, self.request_handler) @@ -75,7 +76,7 @@ class BaseTestCase(unittest.TestCase): self.thread.stop() self.thread = None os.environ.__exit__() - support.threading_cleanup(*self._threads) + threading_helper.threading_cleanup(*self._threads) def request(self, uri, method='GET', body=None, headers={}): self.connection = http.client.HTTPConnection(self.HOST, self.PORT) diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index e97a50811..9afc1559b 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -4,10 +4,11 @@ init = test_util.import_importlib('importlib') import sys import threading -import weakref import unittest +import weakref from test import support +from test.support import threading_helper from test import lock_tests @@ -142,7 +143,7 @@ class LifetimeTests: # ) = test_util.test_both(LifetimeTests, init=init) -@support.reap_threads +@threading_helper.reap_threads def test_main(): support.run_unittest(Frozen_ModuleLockAsRLockTests, Source_ModuleLockAsRLockTests, diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 8bfa3af08..2437a77eb 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -37,8 +37,11 @@ import weakref from collections import deque, UserList from itertools import cycle, count from test import support -from test.support import os_helper, import_helper -from test.support.script_helper import assert_python_ok, run_python_until_end +from test.support.script_helper import ( + assert_python_ok, assert_python_failure, run_python_until_end) +from test.support import import_helper +from test.support import os_helper +from test.support import threading_helper from test.support.os_helper import FakePath import codecs @@ -1482,7 +1485,7 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): errors.append(e) raise threads = [threading.Thread(target=f) for x in range(20)] - with support.start_threads(threads): + with threading_helper.start_threads(threads): time.sleep(0.02) # yield self.assertFalse(errors, "the following exceptions were caught: %r" % errors) @@ -1851,7 +1854,7 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests): errors.append(e) raise threads = [threading.Thread(target=f) for x in range(20)] - with support.start_threads(threads): + with threading_helper.start_threads(threads): time.sleep(0.02) # yield self.assertFalse(errors, "the following exceptions were caught: %r" % errors) @@ -3317,7 +3320,7 @@ class TextIOWrapperTest(unittest.TestCase): f.write(text) threads = [threading.Thread(target=run, args=(x,)) for x in range(20)] - with support.start_threads(threads, event.set): + with threading_helper.start_threads(threads, event.set): time.sleep(0.02) with self.open(os_helper.TESTFN) as f: content = f.read() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 76f7cf852..ed313f027 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,8 +34,8 @@ from test import support from test.support import import_helper from test.support import os_helper from test.support import socket_helper +from test.support import threading_helper # TODO: RUSTPYTHON -# from test.support import threading_helper # from test.support import warnings_helper from platform import win32_is_iot @@ -3343,12 +3343,12 @@ class TestSendfile(unittest.TestCase): @classmethod def setUpClass(cls): - cls.key = support.threading_setup() + cls.key = threading_helper.threading_setup() create_file(os_helper.TESTFN, cls.DATA) @classmethod def tearDownClass(cls): - support.threading_cleanup(*cls.key) + threading_helper.threading_cleanup(*cls.key) os_helper.unlink(os_helper.TESTFN) def setUp(self): diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index a428e8100..1a1f3a963 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -8,6 +8,8 @@ import unittest import weakref from test import support from test.support import import_helper +from test.support import threading_helper + py_queue = import_helper.import_fresh_module('queue', blocked=['_queue']) c_queue = import_helper.import_fresh_module('queue', fresh=['_queue']) @@ -64,7 +66,7 @@ class BlockingTestMixin: block_func) return self.result finally: - support.join_thread(thread, 10) # make sure the thread terminates + threading_helper.join_thread(thread) # make sure the thread terminates # Call this instead if block_func is supposed to raise an exception. def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, @@ -80,7 +82,7 @@ class BlockingTestMixin: self.fail("expected exception of kind %r" % expected_exception_class) finally: - support.join_thread(thread, 10) # make sure the thread terminates + threading_helper.join_thread(thread) # make sure the thread terminates if not thread.startedEvent.is_set(): self.fail("trigger thread ended but event never set") @@ -485,7 +487,7 @@ class BaseSimpleQueueTest: args=(q, results, sentinel)) for i in range(n_consumers)] - with support.start_threads(feeders + consumers): + with threading_helper.start_threads(feeders + consumers): pass self.assertFalse(exceptions) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 54866dfa9..789e65c52 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -5,6 +5,7 @@ import unittest import urllib.robotparser from test import support from test.support import socket_helper +from test.support import threading_helper from http.server import BaseHTTPRequestHandler, HTTPServer @@ -327,7 +328,7 @@ class PasswordProtectedSiteTestCase(unittest.TestCase): self.t.join() self.server.server_close() - @support.reap_threads + @threading_helper.reap_threads def testPasswordProtectedSite(self): addr = self.server.server_address url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index 3f84af2a4..491d7b3a7 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -4,9 +4,10 @@ import threading import time import unittest from test import support +from test.support import threading_helper -TIMEOUT = 10 +TIMEOUT = support.SHORT_TIMEOUT class Timer: @@ -82,7 +83,7 @@ class TestCase(unittest.TestCase): self.assertEqual(q.get(timeout=TIMEOUT), 5) self.assertTrue(q.empty()) timer.advance(1000) - support.join_thread(t, timeout=TIMEOUT) + threading_helper.join_thread(t) self.assertTrue(q.empty()) self.assertEqual(timer.time(), 5) @@ -137,7 +138,7 @@ class TestCase(unittest.TestCase): self.assertEqual(q.get(timeout=TIMEOUT), 4) self.assertTrue(q.empty()) timer.advance(1000) - support.join_thread(t, timeout=TIMEOUT) + threading_helper.join_thread(t) self.assertTrue(q.empty()) self.assertEqual(timer.time(), 4) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b1c291b8b..3875eac93 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2,6 +2,7 @@ import unittest from test import support from test.support import os_helper from test.support import socket_helper +from test.support import threading_helper import errno import io @@ -303,7 +304,7 @@ class ThreadableTest: self.server_ready.set() def _setUp(self): - self.wait_threads = support.wait_threads_exit() + self.wait_threads = threading_helper.wait_threads_exit() self.wait_threads.__enter__() self.server_ready = threading.Event() @@ -6294,9 +6295,9 @@ def test_main(): ]) tests.append(TestMSWindowsTCPFlags) - thread_info = support.threading_setup() + thread_info = threading_helper.threading_setup() support.run_unittest(*tests) - support.threading_cleanup(*thread_info) + threading_helper.threading_cleanup(*thread_info) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 17a89d766..b05d02ccd 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -14,9 +14,10 @@ import unittest import socketserver import test.support -from test.support import reap_children, reap_threads, verbose +from test.support import reap_children, verbose from test.support import os_helper from test.support import socket_helper +from test.support import threading_helper test.support.requires("network") @@ -123,7 +124,7 @@ class SocketServerTest(unittest.TestCase): self.assertEqual(server.server_address, server.socket.getsockname()) return server - @reap_threads + @threading_helper.reap_threads def run_server(self, svrcls, hdlrbase, testfunc): server = self.make_server(self.pickaddr(svrcls.address_family), svrcls, hdlrbase) @@ -252,7 +253,7 @@ class SocketServerTest(unittest.TestCase): socketserver.DatagramRequestHandler, self.dgram_examine) - @reap_threads + @threading_helper.reap_threads def test_shutdown(self): # Issue #2302: shutdown() should always succeed in making an # other thread leave serve_forever(). diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 666f6a7cd..a08b29e52 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1,6 +1,3 @@ -from test import support -from test.support import os_helper -from test.support.script_helper import assert_python_ok, assert_python_failure import builtins import codecs # import gc @@ -12,6 +9,10 @@ import subprocess import sys import sysconfig import test.support +from test import support +from test.support import os_helper +from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import threading_helper import textwrap import unittest import warnings @@ -388,7 +389,7 @@ class SysModuleTest(unittest.TestCase): # sys._current_frames() is a CPython-only gimmick. # XXX RUSTPYTHON: above comment is from original cpython test; not sure why the cpython_only decorator wasn't added @test.support.cpython_only - @test.support.reap_threads + @threading_helper.reap_threads def test_current_frames(self): import threading import traceback diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 82e61c488..81625429b 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -2,6 +2,7 @@ import os import unittest import random from test import support +from test.support import threading_helper import _thread as thread import time import weakref @@ -32,8 +33,8 @@ class BasicThreadTest(unittest.TestCase): self.running = 0 self.next_ident = 0 - key = support.threading_setup() - self.addCleanup(support.threading_cleanup, *key) + key = threading_helper.threading_setup() + self.addCleanup(threading_helper.threading_cleanup, *key) class ThreadRunningTests(BasicThreadTest): @@ -58,7 +59,7 @@ class ThreadRunningTests(BasicThreadTest): self.done_mutex.release() def test_starting_threads(self): - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): # Basic test for thread creation. for i in range(NUMTASKS): self.newtask() @@ -94,7 +95,7 @@ class ThreadRunningTests(BasicThreadTest): verbose_print("trying stack_size = (%d)" % tss) self.next_ident = 0 self.created = 0 - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): for i in range(NUMTASKS): self.newtask() @@ -117,7 +118,7 @@ class ThreadRunningTests(BasicThreadTest): mut.acquire() mut.release() - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): thread.start_new_thread(task, ()) while not started: time.sleep(POLL_SLEEP) @@ -143,7 +144,7 @@ class ThreadRunningTests(BasicThreadTest): started = thread.allocate_lock() with support.catch_unraisable_exception() as cm: - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): started.acquire() thread.start_new_thread(task, ()) started.acquire() @@ -183,7 +184,7 @@ class Barrier: class BarrierTest(BasicThreadTest): def test_barrier(self): - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): self.bar = Barrier(NUMTASKS) self.running = NUMTASKS for i in range(NUMTASKS): @@ -226,7 +227,7 @@ class TestForkInThread(unittest.TestCase): self.read_fd, self.write_fd = os.pipe() @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') - @support.reap_threads + @threading_helper.reap_threads def test_forkinthread(self): status = "not set" @@ -247,7 +248,7 @@ class TestForkInThread(unittest.TestCase): os.close(self.write_fd) pid, status = os.waitpid(pid, 0) - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): thread.start_new_thread(thread1, ()) self.assertEqual(os.read(self.read_fd, 2), b"OK", "Unable to fork() in thread") diff --git a/Lib/test/test_threadedtempfile.py b/Lib/test/test_threadedtempfile.py index 526c6a45e..909563732 100644 --- a/Lib/test/test_threadedtempfile.py +++ b/Lib/test/test_threadedtempfile.py @@ -15,7 +15,7 @@ provoking a 2.0 failure under Linux. import tempfile -from test.support import start_threads +from test.support import threading_helper import unittest import io import threading @@ -52,7 +52,7 @@ class ThreadedTempFileTest(unittest.TestCase): @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_main(self): threads = [TempFileGreedy() for i in range(NUM_THREADS)] - with start_threads(threads, startEvent.set): + with threading_helper.start_threads(threads, startEvent.set): pass ok = sum(t.ok_count for t in threads) errors = [str(t.name) + str(t.errors.getvalue()) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 27b15034e..4080b0a74 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,10 @@ Tests for the threading module. """ import test.support -from test.support import (verbose, cpython_only, - requires_type_collecting) -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import threading_helper +from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok, assert_python_failure import random import sys @@ -76,10 +76,10 @@ class TestThread(threading.Thread): class BaseTestCase(unittest.TestCase): def setUp(self): - self._threads = test.support.threading_setup() + self._threads = threading_helper.threading_setup() def tearDown(self): - test.support.threading_cleanup(*self._threads) + threading_helper.threading_cleanup(*self._threads) test.support.reap_children() @@ -131,7 +131,7 @@ class ThreadTests(BaseTestCase): done.set() done = threading.Event() ident = [] - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): tid = _thread.start_new_thread(f, ()) done.wait() self.assertEqual(ident[0], tid) @@ -172,7 +172,7 @@ class ThreadTests(BaseTestCase): mutex = threading.Lock() mutex.acquire() - with support.wait_threads_exit(): + with threading_helper.wait_threads_exit(): tid = _thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() @@ -560,7 +560,6 @@ class ThreadTests(BaseTestCase): # TODO: RUSTPYTHON @unittest.expectedFailure - @requires_type_collecting def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread # at shutdown @@ -1127,7 +1126,6 @@ class ThreadingExceptionTests(BaseTestCase): self.assertIn("ZeroDivisionError", err) self.assertNotIn("Unhandled exception", err) - @requires_type_collecting def test_print_exception_stderr_is_none_1(self): script = r"""if True: import sys diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index afbca37a7..5485d3e0e 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -2,6 +2,7 @@ import sys import unittest from doctest import DocTestSuite from test import support +from test.support import threading_helper import weakref # import gc @@ -66,8 +67,8 @@ class BaseLocalTest: # Simply check that the variable is correctly set self.assertEqual(local.x, i) - with support.start_threads(threading.Thread(target=f, args=(i,)) - for i in range(10)): + with threading_helper.start_threads(threading.Thread(target=f, args=(i,)) + for i in range(10)): pass def test_derived_cycle_dealloc(self): diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 21bc91afc..9d9a0ea05 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -10,6 +10,7 @@ import hashlib from test import support from test.support import hashlib_helper +from test.support import threading_helper try: import ssl @@ -669,11 +670,11 @@ def setUpModule(): # Store the threading_setup in a key and ensure that it is cleaned up # in the tearDown global threads_key - threads_key = support.threading_setup() + threads_key = threading_helper.threading_setup() def tearDownModule(): if threads_key: - support.threading_cleanup(*threads_key) + threading_helper.threading_cleanup(*threads_key) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 877e2d2dc..f31e5058c 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -16,6 +16,8 @@ import io import contextlib from test import support from test.support import os_helper +from test.support import socket_helper +from test.support import threading_helper try: import gzip @@ -358,7 +360,7 @@ class XMLRPCTestCase(unittest.TestCase): server.handle_request() # First request and attempt at second server.handle_request() # Retried second request - server = http.server.HTTPServer((support.HOST, 0), RequestHandler) + server = http.server.HTTPServer((socket_helper.HOST, 0), RequestHandler) self.addCleanup(server.server_close) thread = threading.Thread(target=run_server) thread.start() @@ -1557,7 +1559,7 @@ class UseBuiltinTypesTestCase(unittest.TestCase): self.assertTrue(server.use_builtin_types) -@support.reap_threads +@threading_helper.reap_threads def test_main(): support.run_unittest(XMLRPCTestCase, HelperTestCase, DateTimeTestCase, BinaryTestCase, FaultTestCase, UseBuiltinTypesTestCase,