diff --git a/Lib/selectors.py b/Lib/selectors.py index 89680a20a..bb15a1cb1 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -6,7 +6,8 @@ This module allows high-level and efficient I/O multiplexing, built upon the from abc import ABCMeta, abstractmethod -from collections import namedtuple, Mapping +from collections import namedtuple +from collections.abc import Mapping import math import select import sys @@ -56,6 +57,7 @@ if sys.version_info >= (3, 5): SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. For example, this could be used to store a per-client session ID.''') + class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" @@ -251,7 +253,6 @@ class _BaseSelectorImpl(BaseSelector): return key def modify(self, fileobj, events, data=None): - # TODO: Subclasses can probably optimize this even further. try: key = self._fd_to_key[self._fileobj_lookup(fileobj)] except KeyError: @@ -338,92 +339,115 @@ class SelectSelector(_BaseSelectorImpl): return ready +class _PollLikeSelector(_BaseSelectorImpl): + """Base class shared between poll, epoll and devpoll selectors.""" + _selector_cls = None + _EVENT_READ = None + _EVENT_WRITE = None + + def __init__(self): + super().__init__() + self._selector = self._selector_cls() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poller_events = 0 + if events & EVENT_READ: + poller_events |= self._EVENT_READ + if events & EVENT_WRITE: + poller_events |= self._EVENT_WRITE + try: + self._selector.register(key.fd, poller_events) + except: + super().unregister(fileobj) + raise + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + try: + self._selector.unregister(key.fd) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + return key + + def modify(self, fileobj, events, data=None): + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError(f"{fileobj!r} is not registered") from None + + changed = False + if events != key.events: + selector_events = 0 + if events & EVENT_READ: + selector_events |= self._EVENT_READ + if events & EVENT_WRITE: + selector_events |= self._EVENT_WRITE + try: + self._selector.modify(key.fd, selector_events) + except: + super().unregister(fileobj) + raise + changed = True + if data != key.data: + changed = True + + if changed: + key = key._replace(events=events, data=data) + self._fd_to_key[key.fd] = key + return key + + def select(self, timeout=None): + # This is shared between poll() and epoll(). + # epoll() has a different signature and handling of timeout parameter. + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # poll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._selector.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~self._EVENT_READ: + events |= EVENT_WRITE + if event & ~self._EVENT_WRITE: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + if hasattr(select, 'poll'): - class PollSelector(_BaseSelectorImpl): + class PollSelector(_PollLikeSelector): """Poll-based selector.""" - - def __init__(self): - super().__init__() - self._poll = select.poll() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - poll_events = 0 - if events & EVENT_READ: - poll_events |= select.POLLIN - if events & EVENT_WRITE: - poll_events |= select.POLLOUT - self._poll.register(key.fd, poll_events) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - self._poll.unregister(key.fd) - return key - - def select(self, timeout=None): - if timeout is None: - timeout = None - elif timeout <= 0: - timeout = 0 - else: - # poll() has a resolution of 1 millisecond, round away from - # zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - ready = [] - try: - fd_event_list = self._poll.poll(timeout) - except InterruptedError: - return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.POLLIN: - events |= EVENT_WRITE - if event & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready + _selector_cls = select.poll + _EVENT_READ = select.POLLIN + _EVENT_WRITE = select.POLLOUT if hasattr(select, 'epoll'): - class EpollSelector(_BaseSelectorImpl): + class EpollSelector(_PollLikeSelector): """Epoll-based selector.""" - - def __init__(self): - super().__init__() - self._epoll = select.epoll() + _selector_cls = select.epoll + _EVENT_READ = select.EPOLLIN + _EVENT_WRITE = select.EPOLLOUT def fileno(self): - return self._epoll.fileno() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - epoll_events = 0 - if events & EVENT_READ: - epoll_events |= select.EPOLLIN - if events & EVENT_WRITE: - epoll_events |= select.EPOLLOUT - try: - self._epoll.register(key.fd, epoll_events) - except BaseException: - super().unregister(fileobj) - raise - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - try: - self._epoll.unregister(key.fd) - except OSError: - # This can happen if the FD was closed since it - # was registered. - pass - return key + return self._selector.fileno() def select(self, timeout=None): if timeout is None: @@ -442,7 +466,7 @@ if hasattr(select, 'epoll'): ready = [] try: - fd_event_list = self._epoll.poll(timeout, max_ev) + fd_event_list = self._selector.poll(timeout, max_ev) except InterruptedError: return ready for fd, event in fd_event_list: @@ -458,65 +482,23 @@ if hasattr(select, 'epoll'): return ready def close(self): - self._epoll.close() + self._selector.close() super().close() if hasattr(select, 'devpoll'): - class DevpollSelector(_BaseSelectorImpl): + class DevpollSelector(_PollLikeSelector): """Solaris /dev/poll selector.""" - - def __init__(self): - super().__init__() - self._devpoll = select.devpoll() + _selector_cls = select.devpoll + _EVENT_READ = select.POLLIN + _EVENT_WRITE = select.POLLOUT def fileno(self): - return self._devpoll.fileno() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - poll_events = 0 - if events & EVENT_READ: - poll_events |= select.POLLIN - if events & EVENT_WRITE: - poll_events |= select.POLLOUT - self._devpoll.register(key.fd, poll_events) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - self._devpoll.unregister(key.fd) - return key - - def select(self, timeout=None): - if timeout is None: - timeout = None - elif timeout <= 0: - timeout = 0 - else: - # devpoll() has a resolution of 1 millisecond, round away from - # zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - ready = [] - try: - fd_event_list = self._devpoll.poll(timeout) - except InterruptedError: - return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.POLLIN: - events |= EVENT_WRITE - if event & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready + return self._selector.fileno() def close(self): - self._devpoll.close() + self._selector.close() super().close() @@ -527,10 +509,10 @@ if hasattr(select, 'kqueue'): def __init__(self): super().__init__() - self._kqueue = select.kqueue() + self._selector = select.kqueue() def fileno(self): - return self._kqueue.fileno() + return self._selector.fileno() def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) @@ -538,12 +520,12 @@ if hasattr(select, 'kqueue'): if events & EVENT_READ: kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) - self._kqueue.control([kev], 0, 0) + self._selector.control([kev], 0, 0) if events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) - self._kqueue.control([kev], 0, 0) - except BaseException: + self._selector.control([kev], 0, 0) + except: super().unregister(fileobj) raise return key @@ -554,7 +536,7 @@ if hasattr(select, 'kqueue'): kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) try: - self._kqueue.control([kev], 0, 0) + self._selector.control([kev], 0, 0) except OSError: # This can happen if the FD was closed since it # was registered. @@ -563,7 +545,7 @@ if hasattr(select, 'kqueue'): kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) try: - self._kqueue.control([kev], 0, 0) + self._selector.control([kev], 0, 0) except OSError: # See comment above. pass @@ -571,10 +553,13 @@ if hasattr(select, 'kqueue'): def select(self, timeout=None): timeout = None if timeout is None else max(timeout, 0) - max_ev = len(self._fd_to_key) + # If max_ev is 0, kqueue will ignore the timeout. For consistent + # behavior with the other selector classes, we prevent that here + # (using max). See https://bugs.python.org/issue29255 + max_ev = max(len(self._fd_to_key), 1) ready = [] try: - kev_list = self._kqueue.control(None, max_ev, timeout) + kev_list = self._selector.control(None, max_ev, timeout) except InterruptedError: return ready for kev in kev_list: @@ -592,20 +577,43 @@ if hasattr(select, 'kqueue'): return ready def close(self): - self._kqueue.close() + self._selector.close() super().close() +def _can_use(method): + """Check if we can use the selector depending upon the + operating system. """ + # Implementation based upon https://github.com/sethmlarson/selectors2/blob/master/selectors2.py + selector = getattr(select, method, None) + if selector is None: + # select module does not implement method + return False + # check if the OS and Kernel actually support the method. Call may fail with + # OSError: [Errno 38] Function not implemented + try: + selector_obj = selector() + if method == 'poll': + # check that poll actually works + selector_obj.poll(0) + else: + # close epoll, kqueue, and devpoll fd + selector_obj.close() + return True + except OSError: + return False + + # Choose the best implementation, roughly: # epoll|kqueue|devpoll > poll > select. # select() also can't accept a FD > FD_SETSIZE (usually around 1024) -if 'KqueueSelector' in globals(): +if _can_use('kqueue'): DefaultSelector = KqueueSelector -elif 'EpollSelector' in globals(): +elif _can_use('epoll'): DefaultSelector = EpollSelector -elif 'DevpollSelector' in globals(): +elif _can_use('devpoll'): DefaultSelector = DevpollSelector -elif 'PollSelector' in globals(): +elif _can_use('poll'): DefaultSelector = PollSelector else: DefaultSelector = SelectSelector diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index c98ee55b1..4a080d87d 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -52,12 +52,18 @@ class TestUserObjects(unittest.TestCase): self.assertEqual(obj.data, obj_copy.data) self.assertIs(obj.test, obj_copy.test) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_str_protocol(self): self._superset_test(UserString, str) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_list_protocol(self): self._superset_test(UserList, list) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dict_protocol(self): self._superset_test(UserDict, dict) @@ -686,6 +692,8 @@ class TestNamedTuple(unittest.TestCase): self.assertEqual(np.x, 1) self.assertEqual(np.y, 2) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_new_builtins_issue_43102(self): obj = namedtuple('C', ()) new_func = obj.__new__ @@ -1999,6 +2007,8 @@ class TestCollectionABCs(ABCTestCase): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py index 68fc44978..65376a37e 100644 --- a/Lib/test/test_defaultdict.py +++ b/Lib/test/test_defaultdict.py @@ -150,6 +150,8 @@ class TestDefaultDict(unittest.TestCase): o = pickle.loads(s) self.assertEqual(d, o) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union(self): i = defaultdict(int, {1: 1, 2: 2}) s = defaultdict(str, {0: "zero", 1: "one"}) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 86825bce2..bebf40d21 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -691,11 +691,6 @@ class TypingCallableTests(BaseCallableTests, BaseTestCase): class CollectionsCallableTests(BaseCallableTests, BaseTestCase): Callable = collections.abc.Callable - # TODO: RUSTPYTHON, pickle.PicklingError: Can't pickle : it's not found as collections.abc._CallableGenericAlias - @unittest.expectedFailure - def test_pickle(self): # TODO: RUSTPYTHON, remove when this passes - super().test_pickle() # TODO: RUSTPYTHON, remove when this passes - # TODO: RUSTPYTHON, AssertionError: 'collections.abc.Callable[__main__.ParamSpec, typing.TypeVar]' != 'collections.abc.Callable[~P, ~T]' @unittest.expectedFailure def test_paramspec(self): # TODO: RUSTPYTHON, remove when this passes @@ -5423,7 +5418,6 @@ class AllTests(BaseTestCase): self.assertIn('SupportsComplex', a) def test_all_exported_names(self): - actual_all = set(typing.__all__) computed_all = { k for k, v in vars(typing).items()