adjust failing markers for test_{collections,defaultdict}

Lib/selectors.py

@@ -6,7 +6,8 @@ This module allows high-level and efficient I/O multiplexing, built upon the
 
 
 from abc import ABCMeta, abstractmethod
-from collections import namedtuple, Mapping
+from collections import namedtuple
+from collections.abc import Mapping
 import math
 import select
 import sys
@@ -56,6 +57,7 @@ if sys.version_info >= (3, 5):
     SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object.
     For example, this could be used to store a per-client session ID.''')
 
+
 class _SelectorMapping(Mapping):
     """Mapping of file objects to selector keys."""
 
@@ -251,7 +253,6 @@ class _BaseSelectorImpl(BaseSelector):
         return key
 
     def modify(self, fileobj, events, data=None):
-        # TODO: Subclasses can probably optimize this even further.
         try:
             key = self._fd_to_key[self._fileobj_lookup(fileobj)]
         except KeyError:
@@ -338,92 +339,115 @@ class SelectSelector(_BaseSelectorImpl):
         return ready
 
 
+class _PollLikeSelector(_BaseSelectorImpl):
+    """Base class shared between poll, epoll and devpoll selectors."""
+    _selector_cls = None
+    _EVENT_READ = None
+    _EVENT_WRITE = None
+
+    def __init__(self):
+        super().__init__()
+        self._selector = self._selector_cls()
+
+    def register(self, fileobj, events, data=None):
+        key = super().register(fileobj, events, data)
+        poller_events = 0
+        if events & EVENT_READ:
+            poller_events |= self._EVENT_READ
+        if events & EVENT_WRITE:
+            poller_events |= self._EVENT_WRITE
+        try:
+            self._selector.register(key.fd, poller_events)
+        except:
+            super().unregister(fileobj)
+            raise
+        return key
+
+    def unregister(self, fileobj):
+        key = super().unregister(fileobj)
+        try:
+            self._selector.unregister(key.fd)
+        except OSError:
+            # This can happen if the FD was closed since it
+            # was registered.
+            pass
+        return key
+
+    def modify(self, fileobj, events, data=None):
+        try:
+            key = self._fd_to_key[self._fileobj_lookup(fileobj)]
+        except KeyError:
+            raise KeyError(f"{fileobj!r} is not registered") from None
+
+        changed = False
+        if events != key.events:
+            selector_events = 0
+            if events & EVENT_READ:
+                selector_events |= self._EVENT_READ
+            if events & EVENT_WRITE:
+                selector_events |= self._EVENT_WRITE
+            try:
+                self._selector.modify(key.fd, selector_events)
+            except:
+                super().unregister(fileobj)
+                raise
+            changed = True
+        if data != key.data:
+            changed = True
+
+        if changed:
+            key = key._replace(events=events, data=data)
+            self._fd_to_key[key.fd] = key
+        return key
+
+    def select(self, timeout=None):
+        # This is shared between poll() and epoll().
+        # epoll() has a different signature and handling of timeout parameter.
+        if timeout is None:
+            timeout = None
+        elif timeout <= 0:
+            timeout = 0
+        else:
+            # poll() has a resolution of 1 millisecond, round away from
+            # zero to wait *at least* timeout seconds.
+            timeout = math.ceil(timeout * 1e3)
+        ready = []
+        try:
+            fd_event_list = self._selector.poll(timeout)
+        except InterruptedError:
+            return ready
+        for fd, event in fd_event_list:
+            events = 0
+            if event & ~self._EVENT_READ:
+                events |= EVENT_WRITE
+            if event & ~self._EVENT_WRITE:
+                events |= EVENT_READ
+
+            key = self._key_from_fd(fd)
+            if key:
+                ready.append((key, events & key.events))
+        return ready
+
+
 if hasattr(select, 'poll'):
 
-    class PollSelector(_BaseSelectorImpl):
+    class PollSelector(_PollLikeSelector):
         """Poll-based selector."""
-
-        def __init__(self):
-            super().__init__()
-            self._poll = select.poll()
-
-        def register(self, fileobj, events, data=None):
-            key = super().register(fileobj, events, data)
-            poll_events = 0
-            if events & EVENT_READ:
-                poll_events |= select.POLLIN
-            if events & EVENT_WRITE:
-                poll_events |= select.POLLOUT
-            self._poll.register(key.fd, poll_events)
-            return key
-
-        def unregister(self, fileobj):
-            key = super().unregister(fileobj)
-            self._poll.unregister(key.fd)
-            return key
-
-        def select(self, timeout=None):
-            if timeout is None:
-                timeout = None
-            elif timeout <= 0:
-                timeout = 0
-            else:
-                # poll() has a resolution of 1 millisecond, round away from
-                # zero to wait *at least* timeout seconds.
-                timeout = math.ceil(timeout * 1e3)
-            ready = []
-            try:
-                fd_event_list = self._poll.poll(timeout)
-            except InterruptedError:
-                return ready
-            for fd, event in fd_event_list:
-                events = 0
-                if event & ~select.POLLIN:
-                    events |= EVENT_WRITE
-                if event & ~select.POLLOUT:
-                    events |= EVENT_READ
-
-                key = self._key_from_fd(fd)
-                if key:
-                    ready.append((key, events & key.events))
-            return ready
+        _selector_cls = select.poll
+        _EVENT_READ = select.POLLIN
+        _EVENT_WRITE = select.POLLOUT
 
 
 if hasattr(select, 'epoll'):
 
-    class EpollSelector(_BaseSelectorImpl):
+    class EpollSelector(_PollLikeSelector):
         """Epoll-based selector."""
-
-        def __init__(self):
-            super().__init__()
-            self._epoll = select.epoll()
+        _selector_cls = select.epoll
+        _EVENT_READ = select.EPOLLIN
+        _EVENT_WRITE = select.EPOLLOUT
 
         def fileno(self):
-            return self._epoll.fileno()
-
-        def register(self, fileobj, events, data=None):
-            key = super().register(fileobj, events, data)
-            epoll_events = 0
-            if events & EVENT_READ:
-                epoll_events |= select.EPOLLIN
-            if events & EVENT_WRITE:
-                epoll_events |= select.EPOLLOUT
-            try:
-                self._epoll.register(key.fd, epoll_events)
-            except BaseException:
-                super().unregister(fileobj)
-                raise
-            return key
-
-        def unregister(self, fileobj):
-            key = super().unregister(fileobj)
-            try:
-                self._epoll.unregister(key.fd)
-            except OSError:
-                # This can happen if the FD was closed since it
-                # was registered.
-                pass
-            return key
+            return self._selector.fileno()
 
         def select(self, timeout=None):
             if timeout is None:
@@ -442,7 +466,7 @@ if hasattr(select, 'epoll'):
 
             ready = []
             try:
-                fd_event_list = self._epoll.poll(timeout, max_ev)
+                fd_event_list = self._selector.poll(timeout, max_ev)
             except InterruptedError:
                 return ready
             for fd, event in fd_event_list:
@@ -458,65 +482,23 @@ if hasattr(select, 'epoll'):
             return ready
 
         def close(self):
-            self._epoll.close()
+            self._selector.close()
             super().close()
 
 
 if hasattr(select, 'devpoll'):
 
-    class DevpollSelector(_BaseSelectorImpl):
+    class DevpollSelector(_PollLikeSelector):
         """Solaris /dev/poll selector."""
-
-        def __init__(self):
-            super().__init__()
-            self._devpoll = select.devpoll()
+        _selector_cls = select.devpoll
+        _EVENT_READ = select.POLLIN
+        _EVENT_WRITE = select.POLLOUT
 
         def fileno(self):
-            return self._devpoll.fileno()
-
-        def register(self, fileobj, events, data=None):
-            key = super().register(fileobj, events, data)
-            poll_events = 0
-            if events & EVENT_READ:
-                poll_events |= select.POLLIN
-            if events & EVENT_WRITE:
-                poll_events |= select.POLLOUT
-            self._devpoll.register(key.fd, poll_events)
-            return key
-
-        def unregister(self, fileobj):
-            key = super().unregister(fileobj)
-            self._devpoll.unregister(key.fd)
-            return key
-
-        def select(self, timeout=None):
-            if timeout is None:
-                timeout = None
-            elif timeout <= 0:
-                timeout = 0
-            else:
-                # devpoll() has a resolution of 1 millisecond, round away from
-                # zero to wait *at least* timeout seconds.
-                timeout = math.ceil(timeout * 1e3)
-            ready = []
-            try:
-                fd_event_list = self._devpoll.poll(timeout)
-            except InterruptedError:
-                return ready
-            for fd, event in fd_event_list:
-                events = 0
-                if event & ~select.POLLIN:
-                    events |= EVENT_WRITE
-                if event & ~select.POLLOUT:
-                    events |= EVENT_READ
-
-                key = self._key_from_fd(fd)
-                if key:
-                    ready.append((key, events & key.events))
-            return ready
+            return self._selector.fileno()
 
         def close(self):
-            self._devpoll.close()
+            self._selector.close()
             super().close()
 
 
@@ -527,10 +509,10 @@ if hasattr(select, 'kqueue'):
 
         def __init__(self):
             super().__init__()
-            self._kqueue = select.kqueue()
+            self._selector = select.kqueue()
 
         def fileno(self):
-            return self._kqueue.fileno()
+            return self._selector.fileno()
 
         def register(self, fileobj, events, data=None):
             key = super().register(fileobj, events, data)
@@ -538,12 +520,12 @@ if hasattr(select, 'kqueue'):
                 if events & EVENT_READ:
                     kev = select.kevent(key.fd, select.KQ_FILTER_READ,
                                         select.KQ_EV_ADD)
-                    self._kqueue.control([kev], 0, 0)
+                    self._selector.control([kev], 0, 0)
                 if events & EVENT_WRITE:
                     kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
                                         select.KQ_EV_ADD)
-                    self._kqueue.control([kev], 0, 0)
-            except BaseException:
+                    self._selector.control([kev], 0, 0)
+            except:
                 super().unregister(fileobj)
                 raise
             return key
@@ -554,7 +536,7 @@ if hasattr(select, 'kqueue'):
                 kev = select.kevent(key.fd, select.KQ_FILTER_READ,
                                     select.KQ_EV_DELETE)
                 try:
-                    self._kqueue.control([kev], 0, 0)
+                    self._selector.control([kev], 0, 0)
                 except OSError:
                     # This can happen if the FD was closed since it
                     # was registered.
@@ -563,7 +545,7 @@ if hasattr(select, 'kqueue'):
                 kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
                                     select.KQ_EV_DELETE)
                 try:
-                    self._kqueue.control([kev], 0, 0)
+                    self._selector.control([kev], 0, 0)
                 except OSError:
                     # See comment above.
                     pass
@@ -571,10 +553,13 @@ if hasattr(select, 'kqueue'):
 
         def select(self, timeout=None):
             timeout = None if timeout is None else max(timeout, 0)
-            max_ev = len(self._fd_to_key)
+            # If max_ev is 0, kqueue will ignore the timeout. For consistent
+            # behavior with the other selector classes, we prevent that here
+            # (using max). See https://bugs.python.org/issue29255
+            max_ev = max(len(self._fd_to_key), 1)
             ready = []
             try:
-                kev_list = self._kqueue.control(None, max_ev, timeout)
+                kev_list = self._selector.control(None, max_ev, timeout)
             except InterruptedError:
                 return ready
             for kev in kev_list:
@@ -592,20 +577,43 @@ if hasattr(select, 'kqueue'):
             return ready
 
         def close(self):
-            self._kqueue.close()
+            self._selector.close()
             super().close()
 
 
+def _can_use(method):
+    """Check if we can use the selector depending upon the
+    operating system. """
+    # Implementation based upon https://github.com/sethmlarson/selectors2/blob/master/selectors2.py
+    selector = getattr(select, method, None)
+    if selector is None:
+        # select module does not implement method
+        return False
+    # check if the OS and Kernel actually support the method. Call may fail with
+    # OSError: [Errno 38] Function not implemented
+    try:
+        selector_obj = selector()
+        if method == 'poll':
+            # check that poll actually works
+            selector_obj.poll(0)
+        else:
+            # close epoll, kqueue, and devpoll fd
+            selector_obj.close()
+        return True
+    except OSError:
+        return False
+
+
 # Choose the best implementation, roughly:
 #    epoll|kqueue|devpoll > poll > select.
 # select() also can't accept a FD > FD_SETSIZE (usually around 1024)
-if 'KqueueSelector' in globals():
+if _can_use('kqueue'):
     DefaultSelector = KqueueSelector
-elif 'EpollSelector' in globals():
+elif _can_use('epoll'):
     DefaultSelector = EpollSelector
-elif 'DevpollSelector' in globals():
+elif _can_use('devpoll'):
     DefaultSelector = DevpollSelector
-elif 'PollSelector' in globals():
+elif _can_use('poll'):
     DefaultSelector = PollSelector
 else:
     DefaultSelector = SelectSelector

Lib/test/test_collections.py

@@ -52,12 +52,18 @@ class TestUserObjects(unittest.TestCase):
         self.assertEqual(obj.data, obj_copy.data)
         self.assertIs(obj.test, obj_copy.test)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_str_protocol(self):
         self._superset_test(UserString, str)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_list_protocol(self):
         self._superset_test(UserList, list)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_dict_protocol(self):
         self._superset_test(UserDict, dict)
 
@@ -686,6 +692,8 @@ class TestNamedTuple(unittest.TestCase):
         self.assertEqual(np.x, 1)
         self.assertEqual(np.y, 2)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_new_builtins_issue_43102(self):
         obj = namedtuple('C', ())
         new_func = obj.__new__
@@ -1999,6 +2007,8 @@ class TestCollectionABCs(ABCTestCase):
         self.assertEqual(len(mss), len(mss2))
         self.assertEqual(list(mss), list(mss2))
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_illegal_patma_flags(self):
         with self.assertRaises(TypeError):
             class Both(Collection):

Lib/test/test_defaultdict.py

@@ -150,6 +150,8 @@ class TestDefaultDict(unittest.TestCase):
             o = pickle.loads(s)
             self.assertEqual(d, o)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_union(self):
         i = defaultdict(int, {1: 1, 2: 2})
         s = defaultdict(str, {0: "zero", 1: "one"})

Lib/test/test_typing.py

@@ -691,11 +691,6 @@ class TypingCallableTests(BaseCallableTests, BaseTestCase):
 class CollectionsCallableTests(BaseCallableTests, BaseTestCase):
     Callable = collections.abc.Callable
 
-    # TODO: RUSTPYTHON, pickle.PicklingError: Can't pickle <class 'collections.abc._CallableGenericAlias'>: it's not found as collections.abc._CallableGenericAlias
-    @unittest.expectedFailure
-    def test_pickle(self): # TODO: RUSTPYTHON, remove when this passes
-        super().test_pickle() # TODO: RUSTPYTHON, remove when this passes
-
     # TODO: RUSTPYTHON, AssertionError: 'collections.abc.Callable[__main__.ParamSpec, typing.TypeVar]' != 'collections.abc.Callable[~P, ~T]'
     @unittest.expectedFailure
     def test_paramspec(self): # TODO: RUSTPYTHON, remove when this passes
@@ -5423,7 +5418,6 @@ class AllTests(BaseTestCase):
         self.assertIn('SupportsComplex', a)
 
     def test_all_exported_names(self):
-
         actual_all = set(typing.__all__)
         computed_all = {
             k for k, v in vars(typing).items()