From bf08f2f6a303e87177f4af385bffe8084ebe41f5 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Wed, 16 Oct 2019 20:07:48 -0500 Subject: [PATCH] Enable async stuff in stdlib modules --- Lib/_collections_abc.py | 260 ++++++++++++++-------------- Lib/contextlib.py | 363 ++++++++++++++++++++-------------------- Lib/types.py | 14 +- 3 files changed, 318 insertions(+), 319 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3158a4f16..53789e729 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -40,7 +40,7 @@ dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) list_iterator = type(iter([])) -# list_reverseiterator = type(iter(reversed([]))) +list_reverseiterator = type(iter(reversed([]))) range_iterator = type(iter(range(0))) longrange_iterator = type(iter(range(1 << 1000))) set_iterator = type(iter(set())) @@ -53,13 +53,14 @@ dict_values = type({}.values()) dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) -# generator = type((lambda: (yield))()) +generator = type((lambda: (yield))()) ## coroutine ## -# async def _coro(): pass -# _coro = _coro() -# coroutine = type(_coro) -# _coro.close() # Prevent ResourceWarning -# del _coro +async def _coro(): pass +_coro = _coro() +coroutine = type(_coro) +_coro.close() # Prevent ResourceWarning +del _coro +# XXX RustPython TODO: async generators # ## asynchronous generator ## # async def _ag(): yield # _ag = _ag() @@ -96,145 +97,145 @@ class Hashable(metaclass=ABCMeta): return NotImplemented -# class Awaitable(metaclass=ABCMeta): +class Awaitable(metaclass=ABCMeta): -# __slots__ = () + __slots__ = () -# @abstractmethod -# def __await__(self): -# yield + @abstractmethod + def __await__(self): + yield -# @classmethod -# def __subclasshook__(cls, C): -# if cls is Awaitable: -# return _check_methods(C, "__await__") -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is Awaitable: + return _check_methods(C, "__await__") + return NotImplemented -# class Coroutine(Awaitable): +class Coroutine(Awaitable): -# __slots__ = () + __slots__ = () -# @abstractmethod -# def send(self, value): -# """Send a value into the coroutine. -# Return next yielded value or raise StopIteration. -# """ -# raise StopIteration + @abstractmethod + def send(self, value): + """Send a value into the coroutine. + Return next yielded value or raise StopIteration. + """ + raise StopIteration -# @abstractmethod -# def throw(self, typ, val=None, tb=None): -# """Raise an exception in the coroutine. -# Return next yielded value or raise StopIteration. -# """ -# if val is None: -# if tb is None: -# raise typ -# val = typ() -# if tb is not None: -# val = val.with_traceback(tb) -# raise val + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the coroutine. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val -# def close(self): -# """Raise GeneratorExit inside coroutine. -# """ -# try: -# self.throw(GeneratorExit) -# except (GeneratorExit, StopIteration): -# pass -# else: -# raise RuntimeError("coroutine ignored GeneratorExit") + def close(self): + """Raise GeneratorExit inside coroutine. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("coroutine ignored GeneratorExit") -# @classmethod -# def __subclasshook__(cls, C): -# if cls is Coroutine: -# return _check_methods(C, '__await__', 'send', 'throw', 'close') -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is Coroutine: + return _check_methods(C, '__await__', 'send', 'throw', 'close') + return NotImplemented -# Coroutine.register(coroutine) +Coroutine.register(coroutine) -# class AsyncIterable(metaclass=ABCMeta): +class AsyncIterable(metaclass=ABCMeta): -# __slots__ = () + __slots__ = () -# @abstractmethod -# def __aiter__(self): -# return AsyncIterator() + @abstractmethod + def __aiter__(self): + return AsyncIterator() -# @classmethod -# def __subclasshook__(cls, C): -# if cls is AsyncIterable: -# return _check_methods(C, "__aiter__") -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterable: + return _check_methods(C, "__aiter__") + return NotImplemented -# class AsyncIterator(AsyncIterable): +class AsyncIterator(AsyncIterable): -# __slots__ = () + __slots__ = () -# @abstractmethod -# async def __anext__(self): -# """Return the next item or raise StopAsyncIteration when exhausted.""" -# raise StopAsyncIteration + @abstractmethod + async def __anext__(self): + """Return the next item or raise StopAsyncIteration when exhausted.""" + raise StopAsyncIteration -# def __aiter__(self): -# return self + def __aiter__(self): + return self -# @classmethod -# def __subclasshook__(cls, C): -# if cls is AsyncIterator: -# return _check_methods(C, "__anext__", "__aiter__") -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterator: + return _check_methods(C, "__anext__", "__aiter__") + return NotImplemented -# class AsyncGenerator(AsyncIterator): +class AsyncGenerator(AsyncIterator): -# __slots__ = () + __slots__ = () -# async def __anext__(self): -# """Return the next item from the asynchronous generator. -# When exhausted, raise StopAsyncIteration. -# """ -# return await self.asend(None) + async def __anext__(self): + """Return the next item from the asynchronous generator. + When exhausted, raise StopAsyncIteration. + """ + return await self.asend(None) -# @abstractmethod -# async def asend(self, value): -# """Send a value into the asynchronous generator. -# Return next yielded value or raise StopAsyncIteration. -# """ -# raise StopAsyncIteration + @abstractmethod + async def asend(self, value): + """Send a value into the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + raise StopAsyncIteration -# @abstractmethod -# async def athrow(self, typ, val=None, tb=None): -# """Raise an exception in the asynchronous generator. -# Return next yielded value or raise StopAsyncIteration. -# """ -# if val is None: -# if tb is None: -# raise typ -# val = typ() -# if tb is not None: -# val = val.with_traceback(tb) -# raise val + @abstractmethod + async def athrow(self, typ, val=None, tb=None): + """Raise an exception in the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val -# async def aclose(self): -# """Raise GeneratorExit inside coroutine. -# """ -# try: -# await self.athrow(GeneratorExit) -# except (GeneratorExit, StopAsyncIteration): -# pass -# else: -# raise RuntimeError("asynchronous generator ignored GeneratorExit") + async def aclose(self): + """Raise GeneratorExit inside coroutine. + """ + try: + await self.athrow(GeneratorExit) + except (GeneratorExit, StopAsyncIteration): + pass + else: + raise RuntimeError("asynchronous generator ignored GeneratorExit") -# @classmethod -# def __subclasshook__(cls, C): -# if cls is AsyncGenerator: -# return _check_methods(C, '__aiter__', '__anext__', -# 'asend', 'athrow', 'aclose') -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncGenerator: + return _check_methods(C, '__aiter__', '__anext__', + 'asend', 'athrow', 'aclose') + return NotImplemented # AsyncGenerator.register(async_generator) @@ -274,20 +275,20 @@ class Iterator(Iterable): return _check_methods(C, '__iter__', '__next__') return NotImplemented -# Iterator.register(bytes_iterator) -# Iterator.register(bytearray_iterator) +Iterator.register(bytes_iterator) +Iterator.register(bytearray_iterator) # Iterator.register(callable_iterator) -# Iterator.register(dict_keyiterator) -# Iterator.register(dict_valueiterator) -# Iterator.register(dict_itemiterator) -# Iterator.register(list_iterator) -# Iterator.register(list_reverseiterator) -# Iterator.register(range_iterator) -# Iterator.register(longrange_iterator) -# Iterator.register(set_iterator) -# Iterator.register(str_iterator) -# Iterator.register(tuple_iterator) -# Iterator.register(zip_iterator) +Iterator.register(dict_keyiterator) +Iterator.register(dict_valueiterator) +Iterator.register(dict_itemiterator) +Iterator.register(list_iterator) +Iterator.register(list_reverseiterator) +Iterator.register(range_iterator) +Iterator.register(longrange_iterator) +Iterator.register(set_iterator) +Iterator.register(str_iterator) +Iterator.register(tuple_iterator) +Iterator.register(zip_iterator) class Reversible(Iterable): @@ -353,8 +354,7 @@ class Generator(Iterator): 'send', 'throw', 'close') return NotImplemented -# Generator.register(generator) - +Generator.register(generator) class Sized(metaclass=ABCMeta): diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 345a04297..1ff8cdf1c 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -31,26 +31,25 @@ class AbstractContextManager(abc.ABC): return NotImplemented -# XXX -# class AbstractAsyncContextManager(abc.ABC): +class AbstractAsyncContextManager(abc.ABC): -# """An abstract base class for asynchronous context managers.""" + """An abstract base class for asynchronous context managers.""" -# async def __aenter__(self): -# """Return `self` upon entering the runtime context.""" -# return self + async def __aenter__(self): + """Return `self` upon entering the runtime context.""" + return self -# @abc.abstractmethod -# async def __aexit__(self, exc_type, exc_value, traceback): -# """Raise any exception triggered within the runtime context.""" -# return None + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None -# @classmethod -# def __subclasshook__(cls, C): -# if cls is AbstractAsyncContextManager: -# return _collections_abc._check_methods(C, "__aenter__", -# "__aexit__") -# return NotImplemented + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractAsyncContextManager: + return _collections_abc._check_methods(C, "__aenter__", + "__aexit__") + return NotImplemented class ContextDecorator(object): @@ -162,51 +161,50 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, raise RuntimeError("generator didn't stop after throw()") -# XXX -# class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, -# AbstractAsyncContextManager): -# """Helper for @asynccontextmanager.""" +class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, + AbstractAsyncContextManager): + """Helper for @asynccontextmanager.""" -# async def __aenter__(self): -# try: -# return await self.gen.__anext__() -# except StopAsyncIteration: -# raise RuntimeError("generator didn't yield") from None + async def __aenter__(self): + try: + return await self.gen.__anext__() + except StopAsyncIteration: + raise RuntimeError("generator didn't yield") from None -# async def __aexit__(self, typ, value, traceback): -# if typ is None: -# try: -# await self.gen.__anext__() -# except StopAsyncIteration: -# return -# else: -# raise RuntimeError("generator didn't stop") -# else: -# if value is None: -# value = typ() -# # See _GeneratorContextManager.__exit__ for comments on subtleties -# # in this implementation -# try: -# await self.gen.athrow(typ, value, traceback) -# raise RuntimeError("generator didn't stop after throw()") -# except StopAsyncIteration as exc: -# return exc is not value -# except RuntimeError as exc: -# if exc is value: -# return False -# # Avoid suppressing if a StopIteration exception -# # was passed to throw() and later wrapped into a RuntimeError -# # (see PEP 479 for sync generators; async generators also -# # have this behavior). But do this only if the exception wrapped -# # by the RuntimeError is actully Stop(Async)Iteration (see -# # issue29692). -# if isinstance(value, (StopIteration, StopAsyncIteration)): -# if exc.__cause__ is value: -# return False -# raise -# except BaseException as exc: -# if exc is not value: -# raise + async def __aexit__(self, typ, value, traceback): + if typ is None: + try: + await self.gen.__anext__() + except StopAsyncIteration: + return + else: + raise RuntimeError("generator didn't stop") + else: + if value is None: + value = typ() + # See _GeneratorContextManager.__exit__ for comments on subtleties + # in this implementation + try: + await self.gen.athrow(typ, value, traceback) + raise RuntimeError("generator didn't stop after throw()") + except StopAsyncIteration as exc: + return exc is not value + except RuntimeError as exc: + if exc is value: + return False + # Avoid suppressing if a StopIteration exception + # was passed to throw() and later wrapped into a RuntimeError + # (see PEP 479 for sync generators; async generators also + # have this behavior). But do this only if the exception wrapped + # by the RuntimeError is actully Stop(Async)Iteration (see + # issue29692). + if isinstance(value, (StopIteration, StopAsyncIteration)): + if exc.__cause__ is value: + return False + raise + except BaseException as exc: + if exc is not value: + raise def contextmanager(func): @@ -269,12 +267,10 @@ def asynccontextmanager(func): finally: """ - # XXX - # @wraps(func) - # def helper(*args, **kwds): - # return _AsyncGeneratorContextManager(func, args, kwds) - # return helper - raise RuntimeError("async not supported") + @wraps(func) + def helper(*args, **kwds): + return _AsyncGeneratorContextManager(func, args, kwds) + return helper class closing(AbstractContextManager): @@ -523,142 +519,141 @@ class ExitStack(_BaseExitStack, AbstractContextManager): self.__exit__(None, None, None) -# XXX -# # Inspired by discussions on https://bugs.python.org/issue29302 -# class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): -# """Async context manager for dynamic management of a stack of exit -# callbacks. +# Inspired by discussions on https://bugs.python.org/issue29302 +class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): + """Async context manager for dynamic management of a stack of exit + callbacks. -# For example: -# async with AsyncExitStack() as stack: -# connections = [await stack.enter_async_context(get_connection()) -# for i in range(5)] -# # All opened connections will automatically be released at the -# # end of the async with statement, even if attempts to open a -# # connection later in the list raise an exception. -# """ + For example: + async with AsyncExitStack() as stack: + connections = [await stack.enter_async_context(get_connection()) + for i in range(5)] + # All opened connections will automatically be released at the + # end of the async with statement, even if attempts to open a + # connection later in the list raise an exception. + """ -# @staticmethod -# def _create_async_exit_wrapper(cm, cm_exit): -# async def _exit_wrapper(exc_type, exc, tb): -# return await cm_exit(cm, exc_type, exc, tb) -# return _exit_wrapper + @staticmethod + def _create_async_exit_wrapper(cm, cm_exit): + async def _exit_wrapper(exc_type, exc, tb): + return await cm_exit(cm, exc_type, exc, tb) + return _exit_wrapper -# @staticmethod -# def _create_async_cb_wrapper(callback, *args, **kwds): -# async def _exit_wrapper(exc_type, exc, tb): -# await callback(*args, **kwds) -# return _exit_wrapper + @staticmethod + def _create_async_cb_wrapper(callback, *args, **kwds): + async def _exit_wrapper(exc_type, exc, tb): + await callback(*args, **kwds) + return _exit_wrapper -# async def enter_async_context(self, cm): -# """Enters the supplied async context manager. + async def enter_async_context(self, cm): + """Enters the supplied async context manager. -# If successful, also pushes its __aexit__ method as a callback and -# returns the result of the __aenter__ method. -# """ -# _cm_type = type(cm) -# _exit = _cm_type.__aexit__ -# result = await _cm_type.__aenter__(cm) -# self._push_async_cm_exit(cm, _exit) -# return result + If successful, also pushes its __aexit__ method as a callback and + returns the result of the __aenter__ method. + """ + _cm_type = type(cm) + _exit = _cm_type.__aexit__ + result = await _cm_type.__aenter__(cm) + self._push_async_cm_exit(cm, _exit) + return result -# def push_async_exit(self, exit): -# """Registers a coroutine function with the standard __aexit__ method -# signature. + def push_async_exit(self, exit): + """Registers a coroutine function with the standard __aexit__ method + signature. -# Can suppress exceptions the same way __aexit__ method can. -# Also accepts any object with an __aexit__ method (registering a call -# to the method instead of the object itself). -# """ -# _cb_type = type(exit) -# try: -# exit_method = _cb_type.__aexit__ -# except AttributeError: -# # Not an async context manager, so assume it's a coroutine function -# self._push_exit_callback(exit, False) -# else: -# self._push_async_cm_exit(exit, exit_method) -# return exit # Allow use as a decorator + Can suppress exceptions the same way __aexit__ method can. + Also accepts any object with an __aexit__ method (registering a call + to the method instead of the object itself). + """ + _cb_type = type(exit) + try: + exit_method = _cb_type.__aexit__ + except AttributeError: + # Not an async context manager, so assume it's a coroutine function + self._push_exit_callback(exit, False) + else: + self._push_async_cm_exit(exit, exit_method) + return exit # Allow use as a decorator -# def push_async_callback(self, callback, *args, **kwds): -# """Registers an arbitrary coroutine function and arguments. + def push_async_callback(self, callback, *args, **kwds): + """Registers an arbitrary coroutine function and arguments. -# Cannot suppress exceptions. -# """ -# _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) + Cannot suppress exceptions. + """ + _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) -# # We changed the signature, so using @wraps is not appropriate, but -# # setting __wrapped__ may still help with introspection. -# _exit_wrapper.__wrapped__ = callback -# self._push_exit_callback(_exit_wrapper, False) -# return callback # Allow use as a decorator + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection. + _exit_wrapper.__wrapped__ = callback + self._push_exit_callback(_exit_wrapper, False) + return callback # Allow use as a decorator -# async def aclose(self): -# """Immediately unwind the context stack.""" -# await self.__aexit__(None, None, None) + async def aclose(self): + """Immediately unwind the context stack.""" + await self.__aexit__(None, None, None) -# def _push_async_cm_exit(self, cm, cm_exit): -# """Helper to correctly register coroutine function to __aexit__ -# method.""" -# _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) -# _exit_wrapper.__self__ = cm -# self._push_exit_callback(_exit_wrapper, False) + def _push_async_cm_exit(self, cm, cm_exit): + """Helper to correctly register coroutine function to __aexit__ + method.""" + _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) + _exit_wrapper.__self__ = cm + self._push_exit_callback(_exit_wrapper, False) -# async def __aenter__(self): -# return self + async def __aenter__(self): + return self -# async def __aexit__(self, *exc_details): -# received_exc = exc_details[0] is not None + async def __aexit__(self, *exc_details): + received_exc = exc_details[0] is not None -# # We manipulate the exception state so it behaves as though -# # we were actually nesting multiple with statements -# frame_exc = sys.exc_info()[1] -# def _fix_exception_context(new_exc, old_exc): -# # Context may not be correct, so find the end of the chain -# while 1: -# exc_context = new_exc.__context__ -# if exc_context is old_exc: -# # Context is already set correctly (see issue 20317) -# return -# if exc_context is None or exc_context is frame_exc: -# break -# new_exc = exc_context -# # Change the end of the chain to point to the exception -# # we expect it to reference -# new_exc.__context__ = old_exc + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exc_info()[1] + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is None or exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc -# # Callbacks are invoked in LIFO order to match the behaviour of -# # nested context managers -# suppressed_exc = False -# pending_raise = False -# while self._exit_callbacks: -# is_sync, cb = self._exit_callbacks.pop() -# try: -# if is_sync: -# cb_suppress = cb(*exc_details) -# else: -# cb_suppress = await cb(*exc_details) + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + is_sync, cb = self._exit_callbacks.pop() + try: + if is_sync: + cb_suppress = cb(*exc_details) + else: + cb_suppress = await cb(*exc_details) -# if cb_suppress: -# suppressed_exc = True -# pending_raise = False -# exc_details = (None, None, None) -# except: -# new_exc_details = sys.exc_info() -# # simulate the stack of exceptions by setting the context -# _fix_exception_context(new_exc_details[1], exc_details[1]) -# pending_raise = True -# exc_details = new_exc_details -# if pending_raise: -# try: -# # bare "raise exc_details[1]" replaces our carefully -# # set-up context -# fixed_ctx = exc_details[1].__context__ -# raise exc_details[1] -# except BaseException: -# exc_details[1].__context__ = fixed_ctx -# raise -# return received_exc and suppressed_exc + if cb_suppress: + suppressed_exc = True + pending_raise = False + exc_details = (None, None, None) + except: + new_exc_details = sys.exc_info() + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc_details[1], exc_details[1]) + pending_raise = True + exc_details = new_exc_details + if pending_raise: + try: + # bare "raise exc_details[1]" replaces our carefully + # set-up context + fixed_ctx = exc_details[1].__context__ + raise exc_details[1] + except BaseException: + exc_details[1].__context__ = fixed_ctx + raise + return received_exc and suppressed_exc class nullcontext(AbstractContextManager): diff --git a/Lib/types.py b/Lib/types.py index a7034901c..d088912d4 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -19,11 +19,12 @@ def _g(): yield 1 GeneratorType = type(_g()) -# async def _c(): pass -# _c = _c() -# CoroutineType = type(_c) -# _c.close() # Prevent ResourceWarning +async def _c(): pass +_c = _c() +CoroutineType = type(_c) +_c.close() # Prevent ResourceWarning +# XXX RUSTPYTHON TODO: async generators # async def _ag(): # yield # _ag = _ag() @@ -240,7 +241,8 @@ def coroutine(func): if not callable(func): raise TypeError('types.coroutine() expects a callable') - if (func.__class__ is FunctionType and + # XXX RUSTPYTHON TODO: iterable coroutine + if (False and func.__class__ is FunctionType and getattr(func, '__code__', None).__class__ is CodeType): co_flags = func.__code__.co_flags @@ -276,6 +278,8 @@ def coroutine(func): def wrapped(*args, **kwargs): coro = func(*args, **kwargs) if (coro.__class__ is CoroutineType or + # XXX RUSTPYTHON TODO: iterable coroutine + False and coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100): # 'coro' is a native coroutine object or an iterable coroutine return coro