Merge pull request #2544 from RustPython/coolreader18/misc-fixes

Misc fixes related to IPython

Cargo.lock

@@ -2091,6 +2091,7 @@ dependencies = [
  "socket2",
  "sre-engine",
  "static_assertions",
+ "termios",
  "thiserror",
  "thread_local",
  "timsort",
@@ -2428,6 +2429,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "termios"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"

Lib/collections/_defaultdict.py

@@ -1,7 +1,11 @@
+from reprlib import recursive_repr as _recursive_repr
+
 class defaultdict(dict):
     def __init__(self, *args, **kwargs):
         if len(args) >= 1:
             default_factory = args[0]
+            if default_factory is not None and not callable(default_factory):
+                raise TypeError("first argument must be callable or None")
             args = args[1:]
         else:
             default_factory = None
@@ -9,11 +13,30 @@ class defaultdict(dict):
         self.default_factory = default_factory
 
     def __missing__(self, key):
-        if self.default_factory:
-            return self.default_factory()
+        if self.default_factory is not None:
+            val = self.default_factory()
         else:
             raise KeyError(key)
+        self[key] = val
+        return val
+
+    @_recursive_repr()
+    def __repr_factory(factory):
+        return repr(factory)
 
     def __repr__(self):
-        return f"defaultdict({self.default_factory}, {dict.__repr__(self)})"
+        return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})"
 
+    def copy(self):
+        return type(self)(self.default_factory, self)
+
+    __copy__ = copy
+
+    def __reduce__(self):
+        if self.default_factory is not None:
+            args = self.default_factory,
+        else:
+            args = ()
+        return type(self), args, None, None, iter(self.items())
+
+defaultdict.__module__ = 'collections'

Lib/pty.py

@@ -0,0 +1,172 @@
+"""Pseudo terminal utilities."""
+
+# Bugs: No signal handling.  Doesn't set slave termios and window size.
+#       Only tested on Linux.
+# See:  W. Richard Stevens. 1992.  Advanced Programming in the
+#       UNIX Environment.  Chapter 19.
+# Author: Steen Lumholt -- with additions by Guido.
+
+from select import select
+import os
+import sys
+import tty
+
+__all__ = ["openpty","fork","spawn"]
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+CHILD = 0
+
+def openpty():
+    """openpty() -> (master_fd, slave_fd)
+    Open a pty master/slave pair, using os.openpty() if possible."""
+
+    try:
+        return os.openpty()
+    except (AttributeError, OSError):
+        pass
+    master_fd, slave_name = _open_terminal()
+    slave_fd = slave_open(slave_name)
+    return master_fd, slave_fd
+
+def master_open():
+    """master_open() -> (master_fd, slave_name)
+    Open a pty master and return the fd, and the filename of the slave end.
+    Deprecated, use openpty() instead."""
+
+    try:
+        master_fd, slave_fd = os.openpty()
+    except (AttributeError, OSError):
+        pass
+    else:
+        slave_name = os.ttyname(slave_fd)
+        os.close(slave_fd)
+        return master_fd, slave_name
+
+    return _open_terminal()
+
+def _open_terminal():
+    """Open pty master and return (master_fd, tty_name)."""
+    for x in 'pqrstuvwxyzPQRST':
+        for y in '0123456789abcdef':
+            pty_name = '/dev/pty' + x + y
+            try:
+                fd = os.open(pty_name, os.O_RDWR)
+            except OSError:
+                continue
+            return (fd, '/dev/tty' + x + y)
+    raise OSError('out of pty devices')
+
+def slave_open(tty_name):
+    """slave_open(tty_name) -> slave_fd
+    Open the pty slave and acquire the controlling terminal, returning
+    opened filedescriptor.
+    Deprecated, use openpty() instead."""
+
+    result = os.open(tty_name, os.O_RDWR)
+    try:
+        from fcntl import ioctl, I_PUSH
+    except ImportError:
+        return result
+    try:
+        ioctl(result, I_PUSH, "ptem")
+        ioctl(result, I_PUSH, "ldterm")
+    except OSError:
+        pass
+    return result
+
+def fork():
+    """fork() -> (pid, master_fd)
+    Fork and make the child a session leader with a controlling terminal."""
+
+    try:
+        pid, fd = os.forkpty()
+    except (AttributeError, OSError):
+        pass
+    else:
+        if pid == CHILD:
+            try:
+                os.setsid()
+            except OSError:
+                # os.forkpty() already set us session leader
+                pass
+        return pid, fd
+
+    master_fd, slave_fd = openpty()
+    pid = os.fork()
+    if pid == CHILD:
+        # Establish a new session.
+        os.setsid()
+        os.close(master_fd)
+
+        # Slave becomes stdin/stdout/stderr of child.
+        os.dup2(slave_fd, STDIN_FILENO)
+        os.dup2(slave_fd, STDOUT_FILENO)
+        os.dup2(slave_fd, STDERR_FILENO)
+        if (slave_fd > STDERR_FILENO):
+            os.close (slave_fd)
+
+        # Explicitly open the tty to make it become a controlling tty.
+        tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
+        os.close(tmp_fd)
+    else:
+        os.close(slave_fd)
+
+    # Parent and child process.
+    return pid, master_fd
+
+def _writen(fd, data):
+    """Write all the data to a descriptor."""
+    while data:
+        n = os.write(fd, data)
+        data = data[n:]
+
+def _read(fd):
+    """Default read function."""
+    return os.read(fd, 1024)
+
+def _copy(master_fd, master_read=_read, stdin_read=_read):
+    """Parent copy loop.
+    Copies
+            pty master -> standard output   (master_read)
+            standard input -> pty master    (stdin_read)"""
+    fds = [master_fd, STDIN_FILENO]
+    while True:
+        rfds, wfds, xfds = select(fds, [], [])
+        if master_fd in rfds:
+            data = master_read(master_fd)
+            if not data:  # Reached EOF.
+                fds.remove(master_fd)
+            else:
+                os.write(STDOUT_FILENO, data)
+        if STDIN_FILENO in rfds:
+            data = stdin_read(STDIN_FILENO)
+            if not data:
+                fds.remove(STDIN_FILENO)
+            else:
+                _writen(master_fd, data)
+
+def spawn(argv, master_read=_read, stdin_read=_read):
+    """Create a spawned process."""
+    if type(argv) == type(''):
+        argv = (argv,)
+    sys.audit('pty.spawn', argv)
+    pid, master_fd = fork()
+    if pid == CHILD:
+        os.execlp(argv[0], *argv)
+    try:
+        mode = tty.tcgetattr(STDIN_FILENO)
+        tty.setraw(STDIN_FILENO)
+        restore = 1
+    except tty.error:    # This is the same as termios.error
+        restore = 0
+    try:
+        _copy(master_fd, master_read, stdin_read)
+    except OSError:
+        if restore:
+            tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
+
+    os.close(master_fd)
+    return os.waitpid(pid, 0)[1]

Lib/test/test_copy.py

@@ -714,8 +714,6 @@ class TestCopy(unittest.TestCase):
         self.assertEqual(x.foo, y.foo)
         self.assertIsNot(x.foo, y.foo)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_deepcopy_dict_subclass(self):
         class C(dict):
             def __init__(self, d=None):

Lib/test/test_defaultdict.py

@@ -0,0 +1,187 @@
+"""Unit tests for collections.defaultdict."""
+
+import os
+import copy
+import pickle
+import tempfile
+import unittest
+
+from collections import defaultdict
+
+def foobar():
+    return list
+
+class TestDefaultDict(unittest.TestCase):
+
+    def test_basic(self):
+        d1 = defaultdict()
+        self.assertEqual(d1.default_factory, None)
+        d1.default_factory = list
+        d1[12].append(42)
+        self.assertEqual(d1, {12: [42]})
+        d1[12].append(24)
+        self.assertEqual(d1, {12: [42, 24]})
+        d1[13]
+        d1[14]
+        self.assertEqual(d1, {12: [42, 24], 13: [], 14: []})
+        self.assertTrue(d1[12] is not d1[13] is not d1[14])
+        d2 = defaultdict(list, foo=1, bar=2)
+        self.assertEqual(d2.default_factory, list)
+        self.assertEqual(d2, {"foo": 1, "bar": 2})
+        self.assertEqual(d2["foo"], 1)
+        self.assertEqual(d2["bar"], 2)
+        self.assertEqual(d2[42], [])
+        self.assertIn("foo", d2)
+        self.assertIn("foo", d2.keys())
+        self.assertIn("bar", d2)
+        self.assertIn("bar", d2.keys())
+        self.assertIn(42, d2)
+        self.assertIn(42, d2.keys())
+        self.assertNotIn(12, d2)
+        self.assertNotIn(12, d2.keys())
+        d2.default_factory = None
+        self.assertEqual(d2.default_factory, None)
+        try:
+            d2[15]
+        except KeyError as err:
+            self.assertEqual(err.args, (15,))
+        else:
+            self.fail("d2[15] didn't raise KeyError")
+        self.assertRaises(TypeError, defaultdict, 1)
+
+    def test_missing(self):
+        d1 = defaultdict()
+        self.assertRaises(KeyError, d1.__missing__, 42)
+        d1.default_factory = list
+        self.assertEqual(d1.__missing__(42), [])
+
+    def test_repr(self):
+        d1 = defaultdict()
+        self.assertEqual(d1.default_factory, None)
+        self.assertEqual(repr(d1), "defaultdict(None, {})")
+        self.assertEqual(eval(repr(d1)), d1)
+        d1[11] = 41
+        self.assertEqual(repr(d1), "defaultdict(None, {11: 41})")
+        d2 = defaultdict(int)
+        self.assertEqual(d2.default_factory, int)
+        d2[12] = 42
+        self.assertEqual(repr(d2), "defaultdict(<class 'int'>, {12: 42})")
+        def foo(): return 43
+        d3 = defaultdict(foo)
+        self.assertTrue(d3.default_factory is foo)
+        d3[13]
+        self.assertEqual(repr(d3), "defaultdict(%s, {13: 43})" % repr(foo))
+
+    def test_print(self):
+        d1 = defaultdict()
+        def foo(): return 42
+        d2 = defaultdict(foo, {1: 2})
+        # NOTE: We can't use tempfile.[Named]TemporaryFile since this
+        # code must exercise the tp_print C code, which only gets
+        # invoked for *real* files.
+        tfn = tempfile.mktemp()
+        try:
+            f = open(tfn, "w+")
+            try:
+                print(d1, file=f)
+                print(d2, file=f)
+                f.seek(0)
+                self.assertEqual(f.readline(), repr(d1) + "\n")
+                self.assertEqual(f.readline(), repr(d2) + "\n")
+            finally:
+                f.close()
+        finally:
+            os.remove(tfn)
+
+    def test_copy(self):
+        d1 = defaultdict()
+        d2 = d1.copy()
+        self.assertEqual(type(d2), defaultdict)
+        self.assertEqual(d2.default_factory, None)
+        self.assertEqual(d2, {})
+        d1.default_factory = list
+        d3 = d1.copy()
+        self.assertEqual(type(d3), defaultdict)
+        self.assertEqual(d3.default_factory, list)
+        self.assertEqual(d3, {})
+        d1[42]
+        d4 = d1.copy()
+        self.assertEqual(type(d4), defaultdict)
+        self.assertEqual(d4.default_factory, list)
+        self.assertEqual(d4, {42: []})
+        d4[12]
+        self.assertEqual(d4, {42: [], 12: []})
+
+        # Issue 6637: Copy fails for empty default dict
+        d = defaultdict()
+        d['a'] = 42
+        e = d.copy()
+        self.assertEqual(e['a'], 42)
+
+    def test_shallow_copy(self):
+        d1 = defaultdict(foobar, {1: 1})
+        d2 = copy.copy(d1)
+        self.assertEqual(d2.default_factory, foobar)
+        self.assertEqual(d2, d1)
+        d1.default_factory = list
+        d2 = copy.copy(d1)
+        self.assertEqual(d2.default_factory, list)
+        self.assertEqual(d2, d1)
+
+    def test_deep_copy(self):
+        d1 = defaultdict(foobar, {1: [1]})
+        d2 = copy.deepcopy(d1)
+        self.assertEqual(d2.default_factory, foobar)
+        self.assertEqual(d2, d1)
+        self.assertTrue(d1[1] is not d2[1])
+        d1.default_factory = list
+        d2 = copy.deepcopy(d1)
+        self.assertEqual(d2.default_factory, list)
+        self.assertEqual(d2, d1)
+
+    def test_keyerror_without_factory(self):
+        d1 = defaultdict()
+        try:
+            d1[(1,)]
+        except KeyError as err:
+            self.assertEqual(err.args[0], (1,))
+        else:
+            self.fail("expected KeyError")
+
+    def test_recursive_repr(self):
+        # Issue2045: stack overflow when default_factory is a bound method
+        class sub(defaultdict):
+            def __init__(self):
+                self.default_factory = self._factory
+            def _factory(self):
+                return []
+        d = sub()
+        self.assertRegex(repr(d),
+            r"sub\(<bound method .*sub\._factory "
+            r"of sub\(\.\.\., \{\}\)>, \{\}\)")
+
+        # NOTE: printing a subclass of a builtin type does not call its
+        # tp_print slot. So this part is essentially the same test as above.
+        tfn = tempfile.mktemp()
+        try:
+            f = open(tfn, "w+")
+            try:
+                print(d, file=f)
+            finally:
+                f.close()
+        finally:
+            os.remove(tfn)
+
+    def test_callable_arg(self):
+        self.assertRaises(TypeError, defaultdict, {})
+
+    def test_pickling(self):
+        d = defaultdict(int)
+        d[1]
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            s = pickle.dumps(d, proto)
+            o = pickle.loads(s)
+            self.assertEqual(d, o)
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_imp.py

@@ -103,8 +103,6 @@ class ImportTests(unittest.TestCase):
             self.assertEqual(fp.readline(),
                              '"""Tokenization help for Python programs.\n')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_issue3594(self):
         temp_mod_name = 'test_imp_helper'
         sys.path.insert(0, '.')
@@ -119,8 +117,6 @@ class ImportTests(unittest.TestCase):
             support.unlink(temp_mod_name + '.py')
             support.unlink(temp_mod_name + '.pyc')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_issue5604(self):
         # Test cannot cover imp.load_compiled function.
         # Martin von Loewis note what shared library cannot have non-ascii
@@ -215,8 +211,6 @@ class ImportTests(unittest.TestCase):
         self.assertRaises(SyntaxError,
                           imp.find_module, "badsyntax_pep3120", [path])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_load_from_source(self):
         # Verify that the imp module can correctly load and find .py files
         # XXX (ncoghlan): It would be nice to use support.CleanImport

Lib/test/test_importlib/import_/test_caching.py

@@ -61,8 +61,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase):
 
     # __import__ inconsistent between loaders and built-in import when it comes
     #   to when to use the module in sys.modules and when not to.
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_using_cache_after_loader(self):
         # [from cache on return]
         with self.create_mock('module') as mock:
@@ -71,8 +69,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase):
                 self.assertEqual(id(module), id(sys.modules['module']))
 
     # See test_using_cache_after_loader() for reasoning.
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_using_cache_for_assigning_to_attribute(self):
         # [from cache to attribute]
         with self.create_mock('pkg.__init__', 'pkg.module') as importer:
@@ -83,8 +79,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase):
                                  id(sys.modules['pkg.module']))
 
     # See test_using_cache_after_loader() for reasoning.
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_using_cache_for_fromlist(self):
         # [from cache for fromlist]
         with self.create_mock('pkg.__init__', 'pkg.module') as importer:

Lib/test/test_importlib/import_/test_meta_path.py

@@ -67,8 +67,6 @@ class CallSignature:
             return fxn(*args, **kwargs)
         return log, wrapper
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_no_path(self):
         # [no path]
         mod_name = 'top_level'
@@ -84,8 +82,6 @@ class CallSignature:
                 self.assertEqual(args[0], mod_name)
                 self.assertIsNone(args[1])
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_with_path(self):
         # [path set]
         pkg_name = 'pkg'

Lib/test/test_posix.py

@@ -1769,8 +1769,6 @@ class _PosixSpawnMixin:
 
         support.wait_process(pid, exitcode=-signal.SIGUSR1)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_setsigdef_wrong_type(self):
         with self.assertRaises(TypeError):
             self.spawn_func(sys.executable,

Lib/token.py

@@ -1,15 +1,8 @@
-"""Token constants (from "token.h")."""
+"""Token constants."""
+# Auto-generated by Tools/scripts/generate_token.py
 
 __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
 
-#  This file is automatically generated; please don't muck it up!
-#
-#  To update the symbols in this file, 'cd' to the top directory of
-#  the python source tree after building the interpreter and run:
-#
-#    ./python Lib/token.py
-
-#--start constants--
 ENDMARKER = 0
 NAME = 1
 NUMBER = 2
@@ -63,19 +56,76 @@ AT = 49
 ATEQUAL = 50
 RARROW = 51
 ELLIPSIS = 52
-OP = 53
-AWAIT = 54
-ASYNC = 55
-ERRORTOKEN = 56
-N_TOKENS = 57
+COLONEQUAL = 53
+OP = 54
+AWAIT = 55
+ASYNC = 56
+TYPE_IGNORE = 57
+TYPE_COMMENT = 58
+# These aren't used by the C tokenizer but are needed for tokenize.py
+ERRORTOKEN = 59
+COMMENT = 60
+NL = 61
+ENCODING = 62
+N_TOKENS = 63
+# Special definitions for cooperation with parser
 NT_OFFSET = 256
-#--end constants--
 
 tok_name = {value: name
             for name, value in globals().items()
             if isinstance(value, int) and not name.startswith('_')}
 __all__.extend(tok_name.values())
 
+EXACT_TOKEN_TYPES = {
+    '!=': NOTEQUAL,
+    '%': PERCENT,
+    '%=': PERCENTEQUAL,
+    '&': AMPER,
+    '&=': AMPEREQUAL,
+    '(': LPAR,
+    ')': RPAR,
+    '*': STAR,
+    '**': DOUBLESTAR,
+    '**=': DOUBLESTAREQUAL,
+    '*=': STAREQUAL,
+    '+': PLUS,
+    '+=': PLUSEQUAL,
+    ',': COMMA,
+    '-': MINUS,
+    '-=': MINEQUAL,
+    '->': RARROW,
+    '.': DOT,
+    '...': ELLIPSIS,
+    '/': SLASH,
+    '//': DOUBLESLASH,
+    '//=': DOUBLESLASHEQUAL,
+    '/=': SLASHEQUAL,
+    ':': COLON,
+    ':=': COLONEQUAL,
+    ';': SEMI,
+    '<': LESS,
+    '<<': LEFTSHIFT,
+    '<<=': LEFTSHIFTEQUAL,
+    '<=': LESSEQUAL,
+    '=': EQUAL,
+    '==': EQEQUAL,
+    '>': GREATER,
+    '>=': GREATEREQUAL,
+    '>>': RIGHTSHIFT,
+    '>>=': RIGHTSHIFTEQUAL,
+    '@': AT,
+    '@=': ATEQUAL,
+    '[': LSQB,
+    ']': RSQB,
+    '^': CIRCUMFLEX,
+    '^=': CIRCUMFLEXEQUAL,
+    '{': LBRACE,
+    '|': VBAR,
+    '|=': VBAREQUAL,
+    '}': RBRACE,
+    '~': TILDE,
+}
+
 def ISTERMINAL(x):
     return x < NT_OFFSET
 
@@ -84,60 +134,3 @@ def ISNONTERMINAL(x):
 
 def ISEOF(x):
     return x == ENDMARKER
-
-
-def _main():
-    import re
-    import sys
-    args = sys.argv[1:]
-    inFileName = args and args[0] or "Include/token.h"
-    outFileName = "Lib/token.py"
-    if len(args) > 1:
-        outFileName = args[1]
-    try:
-        fp = open(inFileName)
-    except OSError as err:
-        sys.stdout.write("I/O error: %s\n" % str(err))
-        sys.exit(1)
-    with fp:
-        lines = fp.read().split("\n")
-    prog = re.compile(
-        "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
-        re.IGNORECASE)
-    tokens = {}
-    for line in lines:
-        match = prog.match(line)
-        if match:
-            name, val = match.group(1, 2)
-            val = int(val)
-            tokens[val] = name          # reverse so we can sort them...
-    keys = sorted(tokens.keys())
-    # load the output skeleton from the target:
-    try:
-        fp = open(outFileName)
-    except OSError as err:
-        sys.stderr.write("I/O error: %s\n" % str(err))
-        sys.exit(2)
-    with fp:
-        format = fp.read().split("\n")
-    try:
-        start = format.index("#--start constants--") + 1
-        end = format.index("#--end constants--")
-    except ValueError:
-        sys.stderr.write("target does not contain format markers")
-        sys.exit(3)
-    lines = []
-    for val in keys:
-        lines.append("%s = %d" % (tokens[val], val))
-    format[start:end] = lines
-    try:
-        fp = open(outFileName, 'w')
-    except OSError as err:
-        sys.stderr.write("I/O error: %s\n" % str(err))
-        sys.exit(4)
-    with fp:
-        fp.write("\n".join(format))
-
-
-if __name__ == "__main__":
-    _main()

Lib/tokenize.py

@@ -0,0 +1,682 @@
+"""Tokenization help for Python programs.
+
+tokenize(readline) is a generator that breaks a stream of bytes into
+Python tokens.  It decodes the bytes according to PEP-0263 for
+determining source file encoding.
+
+It accepts a readline-like method which is called repeatedly to get the
+next line of input (or b"" for EOF).  It generates 5-tuples with these
+members:
+
+    the token type (see token.py)
+    the token (a string)
+    the starting (row, column) indices of the token (a 2-tuple of ints)
+    the ending (row, column) indices of the token (a 2-tuple of ints)
+    the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators.  Additionally, all token lists start with an ENCODING token
+which tells you which encoding was used to decode the bytes stream.
+"""
+
+__author__ = 'Ka-Ping Yee <ping@lfw.org>'
+__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
+               'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
+               'Michael Foord')
+from builtins import open as _builtin_open
+from codecs import lookup, BOM_UTF8
+import collections
+from io import TextIOWrapper
+import itertools as _itertools
+import re
+import sys
+from token import *
+from token import EXACT_TOKEN_TYPES
+
+cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
+blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
+
+import token
+__all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding",
+                           "untokenize", "TokenInfo"]
+del token
+
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
+    def __repr__(self):
+        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
+        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
+                self._replace(type=annotated_type))
+
+    @property
+    def exact_type(self):
+        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
+            return EXACT_TOKEN_TYPES[self.string]
+        else:
+            return self.type
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'\w+'
+
+Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
+Binnumber = r'0[bB](?:_?[01])+'
+Octnumber = r'0[oO](?:_?[0-7])+'
+Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
+Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
+                   r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
+Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+# Return the empty string, plus all of the valid string prefixes.
+def _all_string_prefixes():
+    # The valid string prefixes. Only contain the lower case versions,
+    #  and don't contain any permutations (include 'fr', but not
+    #  'rf'). The various permutations will be generated.
+    _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr']
+    # if we add binary f-strings, add: ['fb', 'fbr']
+    result = {''}
+    for prefix in _valid_string_prefixes:
+        for t in _itertools.permutations(prefix):
+            # create a list with upper and lower versions of each
+            #  character
+            for u in _itertools.product(*[(c, c.upper()) for c in t]):
+                result.add(''.join(u))
+    return result
+
+def _compile(expr):
+    return re.compile(expr, re.UNICODE)
+
+# Note that since _all_string_prefixes includes the empty string,
+#  StringPrefix can be the empty string (making it optional).
+StringPrefix = group(*_all_string_prefixes())
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group(StringPrefix + "'''", StringPrefix + '"""')
+# Single-line ' or " string.
+String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Sorting in reverse order puts the long operators before their prefixes.
+# Otherwise if = came before ==, == would get recognized as two instances
+# of =.
+Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True)))
+Funny = group(r'\r?\n', Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+                group("'", r'\\\r?\n'),
+                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+# For a given string prefix plus quotes, endpats maps it to a regex
+#  to match the remainder of that string. _prefix can be empty, for
+#  a normal single or triple quoted string (with no prefix).
+endpats = {}
+for _prefix in _all_string_prefixes():
+    endpats[_prefix + "'"] = Single
+    endpats[_prefix + '"'] = Double
+    endpats[_prefix + "'''"] = Single3
+    endpats[_prefix + '"""'] = Double3
+
+# A set of all of the single and triple quoted string prefixes,
+#  including the opening quotes.
+single_quoted = set()
+triple_quoted = set()
+for t in _all_string_prefixes():
+    for u in (t + '"', t + "'"):
+        single_quoted.add(u)
+    for u in (t + '"""', t + "'''"):
+        triple_quoted.add(u)
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+class StopTokenizing(Exception): pass
+
+
+class Untokenizer:
+
+    def __init__(self):
+        self.tokens = []
+        self.prev_row = 1
+        self.prev_col = 0
+        self.encoding = None
+
+    def add_whitespace(self, start):
+        row, col = start
+        if row < self.prev_row or row == self.prev_row and col < self.prev_col:
+            raise ValueError("start ({},{}) precedes previous end ({},{})"
+                             .format(row, col, self.prev_row, self.prev_col))
+        row_offset = row - self.prev_row
+        if row_offset:
+            self.tokens.append("\\\n" * row_offset)
+            self.prev_col = 0
+        col_offset = col - self.prev_col
+        if col_offset:
+            self.tokens.append(" " * col_offset)
+
+    def untokenize(self, iterable):
+        it = iter(iterable)
+        indents = []
+        startline = False
+        for t in it:
+            if len(t) == 2:
+                self.compat(t, it)
+                break
+            tok_type, token, start, end, line = t
+            if tok_type == ENCODING:
+                self.encoding = token
+                continue
+            if tok_type == ENDMARKER:
+                break
+            if tok_type == INDENT:
+                indents.append(token)
+                continue
+            elif tok_type == DEDENT:
+                indents.pop()
+                self.prev_row, self.prev_col = end
+                continue
+            elif tok_type in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                indent = indents[-1]
+                if start[1] >= len(indent):
+                    self.tokens.append(indent)
+                    self.prev_col = len(indent)
+                startline = False
+            self.add_whitespace(start)
+            self.tokens.append(token)
+            self.prev_row, self.prev_col = end
+            if tok_type in (NEWLINE, NL):
+                self.prev_row += 1
+                self.prev_col = 0
+        return "".join(self.tokens)
+
+    def compat(self, token, iterable):
+        indents = []
+        toks_append = self.tokens.append
+        startline = token[0] in (NEWLINE, NL)
+        prevstring = False
+
+        for tok in _itertools.chain([token], iterable):
+            toknum, tokval = tok[:2]
+            if toknum == ENCODING:
+                self.encoding = tokval
+                continue
+
+            if toknum in (NAME, NUMBER):
+                tokval += ' '
+
+            # Insert a space between two consecutive strings
+            if toknum == STRING:
+                if prevstring:
+                    tokval = ' ' + tokval
+                prevstring = True
+            else:
+                prevstring = False
+
+            if toknum == INDENT:
+                indents.append(tokval)
+                continue
+            elif toknum == DEDENT:
+                indents.pop()
+                continue
+            elif toknum in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                toks_append(indents[-1])
+                startline = False
+            toks_append(tokval)
+
+
+def untokenize(iterable):
+    """Transform tokens back into Python source code.
+    It returns a bytes object, encoded using the ENCODING
+    token, which is the first token sequence output by tokenize.
+
+    Each element returned by the iterable must be a token sequence
+    with at least two elements, a token number and token value.  If
+    only two tokens are passed, the resulting output is poor.
+
+    Round-trip invariant for full input:
+        Untokenized source will match input source exactly
+
+    Round-trip invariant for limited input:
+        # Output bytes will tokenize back to the input
+        t1 = [tok[:2] for tok in tokenize(f.readline)]
+        newcode = untokenize(t1)
+        readline = BytesIO(newcode).readline
+        t2 = [tok[:2] for tok in tokenize(readline)]
+        assert t1 == t2
+    """
+    ut = Untokenizer()
+    out = ut.untokenize(iterable)
+    if ut.encoding is not None:
+        out = out.encode(ut.encoding)
+    return out
+
+
+def _get_normal_name(orig_enc):
+    """Imitates get_normal_name in tokenizer.c."""
+    # Only care about the first 12 characters.
+    enc = orig_enc[:12].lower().replace("_", "-")
+    if enc == "utf-8" or enc.startswith("utf-8-"):
+        return "utf-8"
+    if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
+       enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
+        return "iso-8859-1"
+    return orig_enc
+
+def detect_encoding(readline):
+    """
+    The detect_encoding() function is used to detect the encoding that should
+    be used to decode a Python source file.  It requires one argument, readline,
+    in the same way as the tokenize() generator.
+
+    It will call readline a maximum of twice, and return the encoding used
+    (as a string) and a list of any lines (left as bytes) it has read in.
+
+    It detects the encoding from the presence of a utf-8 bom or an encoding
+    cookie as specified in pep-0263.  If both a bom and a cookie are present,
+    but disagree, a SyntaxError will be raised.  If the encoding cookie is an
+    invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    'utf-8-sig' is returned.
+
+    If no encoding is specified, then the default of 'utf-8' will be returned.
+    """
+    try:
+        filename = readline.__self__.name
+    except AttributeError:
+        filename = None
+    bom_found = False
+    encoding = None
+    default = 'utf-8'
+    def read_or_stop():
+        try:
+            return readline()
+        except StopIteration:
+            return b''
+
+    def find_cookie(line):
+        try:
+            # Decode as UTF-8. Either the line is an encoding declaration,
+            # in which case it should be pure ASCII, or it must be UTF-8
+            # per default encoding.
+            line_string = line.decode('utf-8')
+        except UnicodeDecodeError:
+            msg = "invalid or missing encoding declaration"
+            if filename is not None:
+                msg = '{} for {!r}'.format(msg, filename)
+            raise SyntaxError(msg)
+
+        match = cookie_re.match(line_string)
+        if not match:
+            return None
+        encoding = _get_normal_name(match.group(1))
+        try:
+            codec = lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            if filename is None:
+                msg = "unknown encoding: " + encoding
+            else:
+                msg = "unknown encoding for {!r}: {}".format(filename,
+                        encoding)
+            raise SyntaxError(msg)
+
+        if bom_found:
+            if encoding != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                if filename is None:
+                    msg = 'encoding problem: utf-8'
+                else:
+                    msg = 'encoding problem for {!r}: utf-8'.format(filename)
+                raise SyntaxError(msg)
+            encoding += '-sig'
+        return encoding
+
+    first = read_or_stop()
+    if first.startswith(BOM_UTF8):
+        bom_found = True
+        first = first[3:]
+        default = 'utf-8-sig'
+    if not first:
+        return default, []
+
+    encoding = find_cookie(first)
+    if encoding:
+        return encoding, [first]
+    if not blank_re.match(first):
+        return default, [first]
+
+    second = read_or_stop()
+    if not second:
+        return default, [first]
+
+    encoding = find_cookie(second)
+    if encoding:
+        return encoding, [first, second]
+
+    return default, [first, second]
+
+
+def open(filename):
+    """Open a file in read only mode using the encoding detected by
+    detect_encoding().
+    """
+    buffer = _builtin_open(filename, 'rb')
+    try:
+        encoding, lines = detect_encoding(buffer.readline)
+        buffer.seek(0)
+        text = TextIOWrapper(buffer, encoding, line_buffering=True)
+        text.mode = 'r'
+        return text
+    except:
+        buffer.close()
+        raise
+
+
+def tokenize(readline):
+    """
+    The tokenize() generator requires one argument, readline, which
+    must be a callable object which provides the same interface as the
+    readline() method of built-in file objects.  Each call to the function
+    should return one line of input as bytes.  Alternatively, readline
+    can be a callable function terminating with StopIteration:
+        readline = open(myfile, 'rb').__next__  # Example of alternate readline
+
+    The generator produces 5-tuples with these members: the token type; the
+    token string; a 2-tuple (srow, scol) of ints specifying the row and
+    column where the token begins in the source; a 2-tuple (erow, ecol) of
+    ints specifying the row and column where the token ends in the source;
+    and the line on which the token was found.  The line passed is the
+    physical line.
+
+    The first token sequence will always be an ENCODING token
+    which tells you which encoding was used to decode the bytes stream.
+    """
+    encoding, consumed = detect_encoding(readline)
+    empty = _itertools.repeat(b"")
+    rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)
+    return _tokenize(rl_gen.__next__, encoding)
+
+
+def _tokenize(readline, encoding):
+    lnum = parenlev = continued = 0
+    numchars = '0123456789'
+    contstr, needcont = '', 0
+    contline = None
+    indents = [0]
+
+    if encoding is not None:
+        if encoding == "utf-8-sig":
+            # BOM will already have been stripped.
+            encoding = "utf-8"
+        yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
+    last_line = b''
+    line = b''
+    while True:                                # loop over lines in stream
+        try:
+            # We capture the value of the line variable here because
+            # readline uses the empty string '' to signal end of input,
+            # hence `line` itself will always be overwritten at the end
+            # of this loop.
+            last_line = line
+            line = readline()
+        except StopIteration:
+            line = b''
+
+        if encoding is not None:
+            line = line.decode(encoding)
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:                            # continued string
+            if not line:
+                raise TokenError("EOF in multi-line string", strstart)
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield TokenInfo(STRING, contstr + line[:end],
+                       strstart, (lnum, end), contline + line)
+                contstr, needcont = '', 0
+                contline = None
+            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+                yield TokenInfo(ERRORTOKEN, contstr + line,
+                           strstart, (lnum, len(line)), contline)
+                contstr = ''
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line: break
+            column = 0
+            while pos < max:                   # measure leading whitespace
+                if line[pos] == ' ':
+                    column += 1
+                elif line[pos] == '\t':
+                    column = (column//tabsize + 1)*tabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in '#\r\n':           # skip comments or blank lines
+                if line[pos] == '#':
+                    comment_token = line[pos:].rstrip('\r\n')
+                    yield TokenInfo(COMMENT, comment_token,
+                           (lnum, pos), (lnum, pos + len(comment_token)), line)
+                    pos += len(comment_token)
+
+                yield TokenInfo(NL, line[pos:],
+                           (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:           # count indents or dedents
+                indents.append(column)
+                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level",
+                        ("<tokenize>", lnum, pos, line))
+                indents = indents[:-1]
+
+                yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+        else:                                  # continued statement
+            if not line:
+                raise TokenError("EOF in multi-line statement", (lnum, 0))
+            continued = 0
+
+        while pos < max:
+            pseudomatch = _compile(PseudoToken).match(line, pos)
+            if pseudomatch:                                # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
+                token, initial = line[start:end], line[start]
+
+                if (initial in numchars or                 # ordinary number
+                    (initial == '.' and token != '.' and token != '...')):
+                    yield TokenInfo(NUMBER, token, spos, epos, line)
+                elif initial in '\r\n':
+                    if parenlev > 0:
+                        yield TokenInfo(NL, token, spos, epos, line)
+                    else:
+                        yield TokenInfo(NEWLINE, token, spos, epos, line)
+
+                elif initial == '#':
+                    assert not token.endswith("\n")
+                    yield TokenInfo(COMMENT, token, spos, epos, line)
+
+                elif token in triple_quoted:
+                    endprog = _compile(endpats[token])
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:                           # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)           # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+
+                # Check up to the first 3 chars of the token to see if
+                #  they're in the single_quoted set. If so, they start
+                #  a string.
+                # We're using the first 3, because we're looking for
+                #  "rb'" (for example) at the start of the token. If
+                #  we switch to longer prefixes, this needs to be
+                #  adjusted.
+                # Note that initial == token[:1].
+                # Also note that single quote checking must come after
+                #  triple quote checking (above).
+                elif (initial in single_quoted or
+                      token[:2] in single_quoted or
+                      token[:3] in single_quoted):
+                    if token[-1] == '\n':                  # continued string
+                        strstart = (lnum, start)
+                        # Again, using the first 3 chars of the
+                        #  token. This is looking for the matching end
+                        #  regex for the correct type of quote
+                        #  character. So it's really looking for
+                        #  endpats["'"] or endpats['"'], by trying to
+                        #  skip string prefix characters, if any.
+                        endprog = _compile(endpats.get(initial) or
+                                           endpats.get(token[1]) or
+                                           endpats.get(token[2]))
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:                                  # ordinary string
+                        yield TokenInfo(STRING, token, spos, epos, line)
+
+                elif initial.isidentifier():               # ordinary name
+                    yield TokenInfo(NAME, token, spos, epos, line)
+                elif initial == '\\':                      # continued stmt
+                    continued = 1
+                else:
+                    if initial in '([{':
+                        parenlev += 1
+                    elif initial in ')]}':
+                        parenlev -= 1
+                    yield TokenInfo(OP, token, spos, epos, line)
+            else:
+                yield TokenInfo(ERRORTOKEN, line[pos],
+                           (lnum, pos), (lnum, pos+1), line)
+                pos += 1
+
+    # Add an implicit NEWLINE if the input doesn't end in one
+    if last_line and last_line[-1] not in '\r\n':
+        yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
+    for indent in indents[1:]:                 # pop remaining indent levels
+        yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
+    yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+
+
+def generate_tokens(readline):
+    """Tokenize a source reading Python code as unicode strings.
+
+    This has the same API as tokenize(), except that it expects the *readline*
+    callable to return str objects instead of bytes.
+    """
+    return _tokenize(readline, None)
+
+def main():
+    import argparse
+
+    # Helper error handling routines
+    def perror(message):
+        sys.stderr.write(message)
+        sys.stderr.write('\n')
+
+    def error(message, filename=None, location=None):
+        if location:
+            args = (filename,) + location + (message,)
+            perror("%s:%d:%d: error: %s" % args)
+        elif filename:
+            perror("%s: error: %s" % (filename, message))
+        else:
+            perror("error: %s" % message)
+        sys.exit(1)
+
+    # Parse the arguments and options
+    parser = argparse.ArgumentParser(prog='python -m tokenize')
+    parser.add_argument(dest='filename', nargs='?',
+                        metavar='filename.py',
+                        help='the file to tokenize; defaults to stdin')
+    parser.add_argument('-e', '--exact', dest='exact', action='store_true',
+                        help='display token names using the exact type')
+    args = parser.parse_args()
+
+    try:
+        # Tokenize the input
+        if args.filename:
+            filename = args.filename
+            with _builtin_open(filename, 'rb') as f:
+                tokens = list(tokenize(f.readline))
+        else:
+            filename = "<stdin>"
+            tokens = _tokenize(sys.stdin.readline, None)
+
+        # Output the tokenization
+        for token in tokens:
+            token_type = token.type
+            if args.exact:
+                token_type = token.exact_type
+            token_range = "%d,%d-%d,%d:" % (token.start + token.end)
+            print("%-20s%-15s%-15r" %
+                  (token_range, tok_name[token_type], token.string))
+    except IndentationError as err:
+        line, column = err.args[1][1:3]
+        error(err.args[0], filename, (line, column))
+    except TokenError as err:
+        line, column = err.args[1]
+        error(err.args[0], filename, (line, column))
+    except SyntaxError as err:
+        error(err, filename)
+    except OSError as err:
+        error(err)
+    except KeyboardInterrupt:
+        print("interrupted\n")
+    except Exception as err:
+        perror("unexpected error: %s" % err)
+        raise
+
+if __name__ == "__main__":
+    main()

Lib/tty.py

@@ -0,0 +1,36 @@
+"""Terminal utilities."""
+
+# Author: Steen Lumholt.
+
+from termios import *
+
+__all__ = ["setraw", "setcbreak"]
+
+# Indexes for termios list.
+IFLAG = 0
+OFLAG = 1
+CFLAG = 2
+LFLAG = 3
+ISPEED = 4
+OSPEED = 5
+CC = 6
+
+def setraw(fd, when=TCSAFLUSH):
+    """Put terminal into a raw mode."""
+    mode = tcgetattr(fd)
+    mode[IFLAG] = mode[IFLAG] & ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON)
+    mode[OFLAG] = mode[OFLAG] & ~(OPOST)
+    mode[CFLAG] = mode[CFLAG] & ~(CSIZE | PARENB)
+    mode[CFLAG] = mode[CFLAG] | CS8
+    mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON | IEXTEN | ISIG)
+    mode[CC][VMIN] = 1
+    mode[CC][VTIME] = 0
+    tcsetattr(fd, when, mode)
+
+def setcbreak(fd, when=TCSAFLUSH):
+    """Put terminal into a cbreak mode."""
+    mode = tcgetattr(fd)
+    mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON)
+    mode[CC][VMIN] = 1
+    mode[CC][VTIME] = 0
+    tcsetattr(fd, when, mode)

Lib/types.py

@@ -53,7 +53,7 @@ except TypeError:
 
 # For Jython, the following two types are identical
 GetSetDescriptorType = type(FunctionType.__code__)
-# MemberDescriptorType = type(FunctionType.__globals__)
+MemberDescriptorType = type(FunctionType.__globals__)
 
 del sys, _f, _g, _C  # Not for export
 

bytecode/src/lib.rs

@@ -978,7 +978,7 @@ impl Instruction {
             }
             GetAIter => 0,
             GetANext => 1,
-            EndAsyncFor => -1,
+            EndAsyncFor => -2,
         }
     }
 

compiler/src/compile.rs

@@ -1391,22 +1391,18 @@ impl Compiler {
         // The thing iterated:
         self.compile_expression(iter)?;
 
-        let check_asynciter_block = if is_async {
-            let check_asynciter_block = self.new_block();
-
+        if is_async {
             self.emit(Instruction::GetAIter);
 
             self.switch_to_block(for_block);
             self.emit(Instruction::SetupExcept {
-                handler: check_asynciter_block,
+                handler: else_block,
             });
             self.emit(Instruction::GetANext);
             self.emit_constant(ConstantData::None);
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
-
-            Some(check_asynciter_block)
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
@@ -1416,7 +1412,6 @@ impl Compiler {
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
-            None
         };
 
         let was_in_loop = self.ctx.loop_data;
@@ -1425,12 +1420,10 @@ impl Compiler {
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump { target: for_block });
 
-        if let Some(check_asynciter_block) = check_asynciter_block {
-            self.switch_to_block(check_asynciter_block);
+        self.switch_to_block(else_block);
+        if is_async {
             self.emit(Instruction::EndAsyncFor);
         }
-
-        self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
         self.compile_statements(orelse)?;
 

src/shell/helper.rs

@@ -58,16 +58,16 @@ impl<'vm> ShellHelper<'vm> {
     fn get_available_completions<'w>(
         &self,
         words: &'w [String],
-    ) -> Option<(&'w str, Box<dyn Iterator<Item = PyResult<PyStrRef>> + 'vm>)> {
+    ) -> Option<(&'w str, impl Iterator<Item = PyResult<PyStrRef>> + 'vm)> {
         // the very first word and then all the ones after the dot
         let (first, rest) = words.split_first().unwrap();
 
         let str_iter_method = |obj, name| {
-            let iter = self.vm.call_method(obj, name, ())?;
+            let iter = self.vm.call_special_method(obj, name, ())?;
             PyIterable::<PyStrRef>::try_from_object(self.vm, iter)?.iter(self.vm)
         };
 
-        if let Some((last, parents)) = rest.split_last() {
+        let (word_start, iter1, iter2) = if let Some((last, parents)) = rest.split_last() {
             // we need to get an attribute based off of the dir() of an object
 
             // last: the last word, could be empty if it ends with a dot
@@ -82,16 +82,17 @@ impl<'vm> ShellHelper<'vm> {
                 current = self.vm.get_attribute(current.clone(), attr.as_str()).ok()?;
             }
 
-            let current_iter = str_iter_method(&current, "__dir__").ok()?;
+            let current_iter = str_iter_method(current, "__dir__").ok()?;
 
-            Some((&last, Box::new(current_iter) as _))
+            (last, current_iter, None)
         } else {
             // we need to get a variable based off of globals/builtins
 
-            let globals = str_iter_method(self.globals.as_object(), "keys").ok()?;
-            let builtins = str_iter_method(&self.vm.builtins, "__dir__").ok()?;
-            Some((&first, Box::new(Iterator::chain(globals, builtins)) as _))
-        }
+            let globals = str_iter_method(self.globals.as_object().clone(), "keys").ok()?;
+            let builtins = str_iter_method(self.vm.builtins.clone(), "__dir__").ok()?;
+            (first, globals, Some(builtins))
+        };
+        Some((word_start, iter1.chain(iter2.into_iter().flatten())))
     }
 
     fn complete_opt(&self, line: &str) -> Option<(usize, Vec<String>)> {

vm/Cargo.toml

@@ -134,6 +134,9 @@ num_cpus = "1"
 [target.'cfg(not(any(target_arch = "wasm32", target_os = "redox")))'.dependencies]
 dns-lookup = "1.0"
 
+[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies]
+termios = "0.3"
+
 [target.'cfg(windows)'.dependencies]
 winreg = "0.8.0"
 schannel = "0.1"

vm/Lib/python_builtins/__reducelib.py

@@ -63,7 +63,7 @@ def reduce_2(obj):
         state = getstate()
 
     listitems = iter(obj) if isinstance(obj, list) else None
-    dictitems = obj.iteritems() if isinstance(obj, dict) else None
+    dictitems = iter(obj.items()) if isinstance(obj, dict) else None
 
     newobj = copyreg.__newobj__
 

vm/src/builtins/function.rs

@@ -38,8 +38,7 @@ pub struct PyFunction {
     jitted_code: OnceCell<CompiledCode>,
     globals: PyDictRef,
     closure: Option<PyTupleTyped<PyCellRef>>,
-    defaults: Option<PyTupleRef>,
-    kw_only_defaults: Option<PyDictRef>,
+    defaults_and_kwdefaults: PyMutex<(Option<PyTupleRef>, Option<PyDictRef>)>,
     name: PyMutex<PyStrRef>,
 }
 
@@ -58,8 +57,7 @@ impl PyFunction {
             jitted_code: OnceCell::new(),
             globals,
             closure,
-            defaults,
-            kw_only_defaults,
+            defaults_and_kwdefaults: PyMutex::new((defaults, kw_only_defaults)),
             name,
         }
     }
@@ -161,10 +159,20 @@ impl PyFunction {
             )));
         }
 
+        let mut defaults_and_kwdefaults = None;
+        // can't be a closure cause it returns a reference to a captured variable :/
+        macro_rules! get_defaults {
+            () => {{
+                defaults_and_kwdefaults
+                    .get_or_insert_with(|| self.defaults_and_kwdefaults.lock().clone())
+            }};
+        }
+
         // Add missing positional arguments, if we have fewer positional arguments than the
         // function definition calls for
         if nargs < nexpected_args {
-            let ndefs = self.defaults.as_ref().map_or(0, |d| d.borrow_value().len());
+            let defaults = get_defaults!().0.as_ref().map(|tup| tup.borrow_value());
+            let ndefs = defaults.map_or(0, |d| d.len());
 
             let nrequired = code.arg_count - ndefs;
 
@@ -184,9 +192,7 @@ impl PyFunction {
                 )));
             }
 
-            if let Some(defaults) = &self.defaults {
-                let defaults = defaults.borrow_value();
-
+            if let Some(defaults) = defaults {
                 let n = std::cmp::min(nargs, nexpected_args);
                 let i = n.saturating_sub(nrequired);
 
@@ -212,10 +218,8 @@ impl PyFunction {
                 .take(code.kwonlyarg_count)
             {
                 if slot.is_none() {
-                    if let Some(kw_only_defaults) = &self.kw_only_defaults {
-                        if let Some(default) =
-                            kw_only_defaults.get_item_option(kwarg.clone(), vm)?
-                        {
+                    if let Some(defaults) = &get_defaults!().1 {
+                        if let Some(default) = defaults.get_item_option(kwarg.clone(), vm)? {
                             *slot = Some(default);
                             continue;
                         }
@@ -311,12 +315,20 @@ impl PyFunction {
 
     #[pyproperty(magic)]
     fn defaults(&self) -> Option<PyTupleRef> {
-        self.defaults.clone()
+        self.defaults_and_kwdefaults.lock().0.clone()
+    }
+    #[pyproperty(magic, setter)]
+    fn set_defaults(&self, defaults: Option<PyTupleRef>) {
+        self.defaults_and_kwdefaults.lock().0 = defaults
     }
 
     #[pyproperty(magic)]
     fn kwdefaults(&self) -> Option<PyDictRef> {
-        self.kw_only_defaults.clone()
+        self.defaults_and_kwdefaults.lock().1.clone()
+    }
+    #[pyproperty(magic, setter)]
+    fn set_kwdefaults(&self, kwdefaults: Option<PyDictRef>) {
+        self.defaults_and_kwdefaults.lock().1 = kwdefaults
     }
 
     #[pyproperty(magic)]
@@ -425,11 +437,32 @@ impl PyBoundMethod {
         PyBoundMethod { object, function }
     }
 
+    #[pyslot]
+    fn tp_new(
+        cls: PyTypeRef,
+        function: PyObjectRef,
+        object: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyRef<Self>> {
+        PyBoundMethod::new(object, function).into_ref_with_type(vm, cls)
+    }
+
     #[pymethod(magic)]
     fn repr(&self, vm: &VirtualMachine) -> PyResult<String> {
+        let funcname = if let Some(qname) =
+            vm.get_attribute_opt(self.function.clone(), "__qualname__")?
+        {
+            Some(qname)
+        } else if let Some(name) = vm.get_attribute_opt(self.function.clone(), "__qualname__")? {
+            Some(name)
+        } else {
+            None
+        };
+        let funcname: Option<PyStrRef> = funcname.and_then(|o| o.downcast().ok());
         Ok(format!(
-            "<bound method of {}>",
-            vm.to_repr(&self.object)?.borrow_value()
+            "<bound method {} of {}>",
+            funcname.as_ref().map_or("?", |s| s.borrow_value()),
+            vm.to_repr(&self.object)?.borrow_value(),
         ))
     }
 

vm/src/builtins/function/jitfunc.rs

@@ -169,8 +169,10 @@ pub(crate) fn get_jit_args<'a>(
         }
     }
 
+    let (defaults, kwdefaults) = func.defaults_and_kwdefaults.lock().clone();
+
     // fill in positional defaults
-    if let Some(defaults) = &func.defaults {
+    if let Some(defaults) = defaults {
         let defaults = defaults.borrow_value();
         for (i, default) in defaults.iter().enumerate() {
             let arg_idx = i + func.code.arg_count - defaults.len();
@@ -181,7 +183,7 @@ pub(crate) fn get_jit_args<'a>(
     }
 
     // fill in keyword only defaults
-    if let Some(kw_only_defaults) = &func.kw_only_defaults {
+    if let Some(kw_only_defaults) = kwdefaults {
         for (i, name) in arg_names.kwonlyargs.iter().enumerate() {
             let arg_idx = i + func.code.arg_count;
             if !jit_args.is_set(arg_idx) {

vm/src/builtins/getset.rs

@@ -336,6 +336,19 @@ impl PyGetSet {
             }
         }
     }
+    #[pymethod]
+    fn __set__(
+        zelf: PyObjectRef,
+        obj: PyObjectRef,
+        value: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        Self::descr_set(zelf, obj, Some(value), vm)
+    }
+    #[pymethod]
+    fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        Self::descr_set(zelf, obj, None, vm)
+    }
 
     #[pyproperty(magic)]
     fn name(&self) -> String {

vm/src/builtins/make_module.rs

@@ -157,7 +157,8 @@ mod decl {
                 // TODO: compile::compile should probably get bytes
                 let source = match &source {
                     Either::A(string) => string.borrow_value(),
-                    Either::B(bytes) => std::str::from_utf8(bytes).unwrap(),
+                    Either::B(bytes) => std::str::from_utf8(bytes)
+                        .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?,
                 };
 
                 let flags = args

vm/src/builtins/mod.rs

@@ -35,7 +35,7 @@ pub use int::{PyInt, PyIntRef};
 pub(crate) mod iter;
 pub use iter::*;
 pub(crate) mod list;
-pub use list::PyList;
+pub use list::{PyList, PyListRef};
 pub(crate) mod map;
 pub use map::PyMap;
 pub(crate) mod mappingproxy;

vm/src/builtins/module.rs

@@ -1,7 +1,7 @@
 use super::dict::PyDictRef;
 use super::pystr::{PyStr, PyStrRef};
 use super::pytype::PyTypeRef;
-use crate::function::{FuncArgs, OptionalOption};
+use crate::function::FuncArgs;
 use crate::pyobject::{
     BorrowValue, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult,
     PyValue,
@@ -43,6 +43,14 @@ pub fn init_module_dict(
         .expect("Failed to set __spec__ on module");
 }
 
+#[derive(FromArgs)]
+struct ModuleInitArgs {
+    #[pyarg(any)]
+    name: PyStrRef,
+    #[pyarg(any, default)]
+    doc: Option<PyStrRef>,
+}
+
 #[pyimpl(with(SlotGetattro), flags(BASETYPE, HAS_DICT))]
 impl PyModule {
     #[pyslot]
@@ -51,7 +59,7 @@ impl PyModule {
     }
 
     #[pymethod(magic)]
-    fn init(zelf: PyRef<Self>, name: PyStrRef, doc: OptionalOption<PyStrRef>, vm: &VirtualMachine) {
+    fn init(zelf: PyRef<Self>, args: ModuleInitArgs, vm: &VirtualMachine) {
         debug_assert!(crate::pyobject::TypeProtocol::class(zelf.as_object())
             .slots
             .flags
@@ -59,8 +67,8 @@ impl PyModule {
         init_module_dict(
             vm,
             &zelf.as_object().dict().unwrap(),
-            name.into_object(),
-            doc.flatten().into_pyobject(vm),
+            args.name.into_object(),
+            args.doc.into_pyobject(vm),
         );
     }
 

vm/src/builtins/property.rs

@@ -136,6 +136,19 @@ impl PyProperty {
             }
         }
     }
+    #[pymethod]
+    fn __set__(
+        zelf: PyObjectRef,
+        obj: PyObjectRef,
+        value: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        Self::descr_set(zelf, obj, Some(value), vm)
+    }
+    #[pymethod]
+    fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        Self::descr_set(zelf, obj, None, vm)
+    }
 
     // Access functions
 

vm/src/builtins/pytype.rs

@@ -536,8 +536,6 @@ impl SlotGetattro for PyType {
         } else if let Some(attr) = mcl_attr {
             drop(mcl);
             vm.call_if_get_descriptor(attr, zelf.into_object())
-        } else if let Some(ref getter) = zelf.get_attr("__getattr__") {
-            vm.invoke(getter, (PyLease::into_pyref(mcl), name_str))
         } else {
             Err(vm.new_attribute_error(format!(
                 "type object '{}' has no attribute '{}'",

vm/src/frame.rs

@@ -786,8 +786,8 @@ impl ExecutingFrame<'_> {
                         Ok(None)
                     }
                 } else {
-                    unreachable!(
-                        "Block type must be finally handler when reaching EndFinally instruction!"
+                    self.fatal(
+                        "Block type must be finally handler when reaching EndFinally instruction!",
                     );
                 }
             }
@@ -820,7 +820,7 @@ impl ExecutingFrame<'_> {
                 let block = self.current_block().unwrap();
                 let reason = match block.typ {
                     BlockType::FinallyHandler { reason, .. } => reason,
-                    _ => unreachable!("WithCleanupStart expects a FinallyHandler block on stack"),
+                    _ => self.fatal("WithCleanupStart expects a FinallyHandler block on stack"),
                 };
                 let exc = match reason {
                     Some(UnwindReason::Raising { exception }) => Some(exception),
@@ -843,7 +843,7 @@ impl ExecutingFrame<'_> {
                 let block = self.pop_block();
                 let (reason, prev_exc) = match block.typ {
                     BlockType::FinallyHandler { reason, prev_exc } => (reason, prev_exc),
-                    _ => unreachable!("WithCleanupFinish expects a FinallyHandler block on stack"),
+                    _ => self.fatal("WithCleanupFinish expects a FinallyHandler block on stack"),
                 };
 
                 let suppress_exception = pybool::boolval(vm, self.pop_value())?;
@@ -904,6 +904,7 @@ impl ExecutingFrame<'_> {
             }
             bytecode::Instruction::EndAsyncFor => {
                 let exc = self.pop_value();
+                self.pop_value(); // async iterator we were calling __anext__ on
                 if exc.isinstance(&vm.ctx.exceptions.stop_async_iteration) {
                     vm.take_exception().expect("Should have exception in stack");
                     Ok(None)
@@ -1072,7 +1073,7 @@ impl ExecutingFrame<'_> {
                     vm.set_exception(prev_exc);
                     Ok(None)
                 } else {
-                    unreachable!("block type must be ExceptHandler here.")
+                    self.fatal("block type must be ExceptHandler here.")
                 }
             }
             bytecode::Instruction::Reverse { amount } => {
@@ -1229,7 +1230,7 @@ impl ExecutingFrame<'_> {
             UnwindReason::Raising { exception } => Err(exception),
             UnwindReason::Returning { value } => Ok(Some(ExecutionResult::Return(value))),
             UnwindReason::Break { .. } | UnwindReason::Continue { .. } => {
-                unreachable!("break or continue must occur within a loop block.")
+                self.fatal("break or continue must occur within a loop block.")
             } // UnwindReason::NoWorries => Ok(None),
         }
     }
@@ -1237,7 +1238,7 @@ impl ExecutingFrame<'_> {
     fn execute_rotate(&mut self, amount: u32) -> FrameResult {
         // Shuffles top of stack amount down
         if amount < 2 {
-            panic!("Can only rotate two or more values");
+            self.fatal("Can only rotate two or more values");
         }
 
         let mut values = Vec::new();
@@ -1811,17 +1812,14 @@ impl ExecutingFrame<'_> {
     fn push_value(&mut self, obj: PyObjectRef) {
         match self.state.stack.try_push(obj) {
             Ok(()) => {}
-            Err(_e) => {
-                dbg!(self);
-                panic!("tried to push value onto stack but overflowed max_stacksize")
-            }
+            Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stacksize"),
         }
     }
 
     fn pop_value(&mut self) -> PyObjectRef {
         match self.state.stack.pop() {
             Some(x) => x,
-            None => panic!("tried to pop value but there was nothing on the stack"),
+            None => self.fatal("tried to pop value but there was nothing on the stack"),
         }
     }
 
@@ -1838,13 +1836,22 @@ impl ExecutingFrame<'_> {
     fn last_value_ref(&self) -> &PyObjectRef {
         match &*self.state.stack {
             [.., last] => last,
-            [] => panic!("tried to get top of stack but stack is empty"),
+            [] => self.fatal("tried to get top of stack but stack is empty"),
         }
     }
 
     fn nth_value(&self, depth: u32) -> PyObjectRef {
         self.state.stack[self.state.stack.len() - depth as usize - 1].clone()
     }
+
+    // redox still has an old nightly, and edition 2021 won't be out for a while
+    #[allow(non_fmt_panic)]
+    #[cold]
+    #[inline(never)]
+    fn fatal(&self, msg: &'static str) -> ! {
+        dbg!(self);
+        panic!(msg)
+    }
 }
 
 impl fmt::Debug for Frame {

vm/src/stdlib/io.rs

@@ -314,8 +314,12 @@ mod _io {
         fn truncate(zelf: PyObjectRef, _pos: OptionalArg, vm: &VirtualMachine) -> PyResult {
             _unsupported(vm, &zelf, "truncate")
         }
-        #[pyattr]
+        #[pymethod]
+        fn fileno(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult {
+            _unsupported(vm, &zelf, "truncate")
+        }
 
+        #[pyattr]
         fn __closed(ctx: &PyContext) -> PyObjectRef {
             ctx.new_bool(false)
         }
@@ -1787,6 +1791,8 @@ mod _io {
         errors: Option<PyStrRef>,
         #[pyarg(any, default)]
         newline: Option<PyStrRef>,
+        #[pyarg(any, default = "false")]
+        line_buffering: bool,
     }
 
     impl TextIOWrapperArgs {
@@ -1813,6 +1819,8 @@ mod _io {
         errors: PyStrRef,
         // TODO: respect newline
         newline: Option<PyStrRef>,
+        // TODO: respect line_buffering
+        line_buffering: bool,
     }
     #[pyattr]
     #[pyclass(name = "TextIOWrapper", base = "_TextIOBase")]
@@ -1872,6 +1880,7 @@ mod _io {
                 encoding,
                 errors,
                 newline: args.newline,
+                line_buffering: args.line_buffering,
             });
 
             Ok(())

vm/src/stdlib/mod.rs

@@ -37,8 +37,6 @@ mod sysconfigdata;
 #[cfg(feature = "threading")]
 mod thread;
 mod time_module;
-#[cfg(feature = "rustpython-parser")]
-mod tokenize;
 mod unicodedata;
 mod warnings;
 mod weakref;
@@ -64,6 +62,8 @@ mod select;
 pub mod signal;
 #[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))]
 mod ssl;
+#[cfg(all(unix, not(target_os = "redox")))]
+mod termios;
 #[cfg(windows)]
 mod winapi;
 #[cfg(windows)]
@@ -118,7 +118,6 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc, ahash::RandomState>
     #[cfg(feature = "rustpython-parser")]
     {
         modules.insert("keyword".to_owned(), Box::new(keyword::make_module));
-        modules.insert("tokenize".to_owned(), Box::new(tokenize::make_module));
     }
 
     // Insert compiler related modules:
@@ -156,6 +155,9 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc, ahash::RandomState>
         modules.insert("pwd".to_owned(), Box::new(pwd::make_module));
     }
 
+    #[cfg(all(unix, not(target_os = "redox")))]
+    modules.insert("termios".to_owned(), Box::new(termios::make_module));
+
     #[cfg(unix)]
     {
         modules.insert(

vm/src/stdlib/signal.rs

@@ -164,6 +164,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
         "SIGINT" => ctx.new_int(libc::SIGINT as u8),
         "SIGSEGV" => ctx.new_int(libc::SIGSEGV as u8),
         "SIGTERM" => ctx.new_int(libc::SIGTERM as u8),
+        "NSIG" => ctx.new_int(NSIG),
         "default_int_handler" => int_handler.clone(),
     });
     extend_module_platform_specific(vm, &module);

vm/src/stdlib/ssl.rs

@@ -564,6 +564,32 @@ impl PySslContext {
             SslServerOrClient::Client
         };
 
+        if let Some(hostname) = &args.server_hostname {
+            let hostname = hostname.borrow_value();
+            if hostname.is_empty() || hostname.starts_with('.') {
+                return Err(vm.new_value_error(
+                    "server_hostname cannot be an empty string or start with a leading dot."
+                        .to_owned(),
+                ));
+            }
+            let ip = hostname.parse::<std::net::IpAddr>();
+            if ip.is_err() {
+                ssl.set_hostname(hostname)
+                    .map_err(|e| convert_openssl_error(vm, e))?;
+            }
+            if zelf.check_hostname.load() {
+                if let Ok(ip) = ip {
+                    ssl.param_mut()
+                        .set_ip(ip)
+                        .map_err(|e| convert_openssl_error(vm, e))?;
+                } else {
+                    ssl.param_mut()
+                        .set_host(hostname)
+                        .map_err(|e| convert_openssl_error(vm, e))?;
+                }
+            }
+        }
+
         let stream = ssl::SslStream::new(ssl, args.sock.clone())
             .map_err(|e| convert_openssl_error(vm, e))?;
 

vm/src/stdlib/termios.rs

@@ -0,0 +1,122 @@
+pub(crate) use self::termios::make_module;
+
+#[pymodule]
+mod termios {
+    use crate::builtins::{int, PyBytes, PyInt, PyListRef, PyTypeRef};
+    use crate::common::borrow::BorrowValue;
+    use crate::exceptions::PyBaseExceptionRef;
+    use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject};
+    use crate::VirtualMachine;
+    use std::convert::TryFrom;
+    use termios::Termios;
+
+    #[pyattr]
+    use termios::{
+        os::target::NCCS, B0, B110, B1200, B134, B150, B1800, B19200, B200, B2400, B300, B38400,
+        B4800, B50, B600, B75, B9600, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB,
+        ECHO, ECHOE, ECHOK, ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR,
+        INPCK, ISIG, ISTRIP, IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST,
+        PARENB, PARMRK, PARODD, TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON,
+        TCSADRAIN, TCSAFLUSH, TCSANOW, TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT,
+        VSTART, VSTOP, VSUSP, VTIME,
+    };
+
+    #[pyfunction]
+    fn tcgetattr(fd: i32, vm: &VirtualMachine) -> PyResult {
+        let termios = Termios::from_fd(fd).map_err(|e| termios_error(e, vm))?;
+        let noncanon = (termios.c_lflag & termios::ICANON) == 0;
+        let cc = termios
+            .c_cc
+            .iter()
+            .enumerate()
+            .map(|(i, &c)| match i {
+                termios::VMIN | termios::VTIME if noncanon => vm.ctx.new_int(c),
+                _ => vm.ctx.new_bytes(vec![c as u8]),
+            })
+            .collect::<Vec<_>>();
+        let out = vec![
+            termios.c_iflag.into_pyobject(vm),
+            termios.c_oflag.into_pyobject(vm),
+            termios.c_cflag.into_pyobject(vm),
+            termios.c_lflag.into_pyobject(vm),
+            termios::cfgetispeed(&termios).into_pyobject(vm),
+            termios::cfgetospeed(&termios).into_pyobject(vm),
+            vm.ctx.new_list(cc),
+        ];
+        Ok(vm.ctx.new_list(out))
+    }
+
+    #[pyfunction]
+    fn tcsetattr(fd: i32, when: i32, attributes: PyListRef, vm: &VirtualMachine) -> PyResult<()> {
+        let [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] =
+            <&[PyObjectRef; 7]>::try_from(&**attributes.borrow_value())
+                .map_err(|_| {
+                    vm.new_type_error("tcsetattr, arg 3: must be 7 element list".to_owned())
+                })?
+                .clone();
+        let mut termios = Termios::from_fd(fd).map_err(|e| termios_error(e, vm))?;
+        termios.c_iflag = TryFromObject::try_from_object(vm, iflag)?;
+        termios.c_oflag = TryFromObject::try_from_object(vm, oflag)?;
+        termios.c_cflag = TryFromObject::try_from_object(vm, cflag)?;
+        termios.c_lflag = TryFromObject::try_from_object(vm, lflag)?;
+        termios::cfsetispeed(&mut termios, TryFromObject::try_from_object(vm, ispeed)?)
+            .map_err(|e| termios_error(e, vm))?;
+        termios::cfsetospeed(&mut termios, TryFromObject::try_from_object(vm, ospeed)?)
+            .map_err(|e| termios_error(e, vm))?;
+        let cc = PyListRef::try_from_object(vm, cc)?;
+        {
+            let cc = cc.borrow_value();
+            let cc = <&[PyObjectRef; NCCS]>::try_from(&**cc).map_err(|_| {
+                vm.new_type_error(format!(
+                    "tcsetattr: attributes[6] must be {} element list",
+                    NCCS
+                ))
+            })?;
+            for (cc, x) in termios.c_cc.iter_mut().zip(cc.iter()) {
+                *cc = if let Some(c) = x
+                    .payload::<PyBytes>()
+                    .filter(|b| b.borrow_value().len() == 1)
+                {
+                    c.borrow_value()[0] as _
+                } else if let Some(i) = x.payload::<PyInt>() {
+                    int::try_to_primitive(i.borrow_value(), vm)?
+                } else {
+                    return Err(vm.new_type_error(
+                        "tcsetattr: elements of attributes must be characters or integers"
+                            .to_owned(),
+                    ));
+                };
+            }
+        }
+
+        termios::tcsetattr(fd, when, &termios).map_err(|e| termios_error(e, vm))?;
+
+        Ok(())
+    }
+
+    fn termios_error(err: std::io::Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
+        vm.new_exception(
+            get_termios_error(vm),
+            vec![
+                err.raw_os_error().into_pyobject(vm),
+                vm.ctx.new_str(err.to_string()),
+            ],
+        )
+    }
+
+    #[pyattr(name = "error")]
+    fn get_termios_error(vm: &VirtualMachine) -> PyTypeRef {
+        rustpython_common::static_cell! {
+            static TERMIOS_ERROR: PyTypeRef;
+        }
+        TERMIOS_ERROR
+            .get_or_init(|| {
+                vm.ctx.new_class(
+                    "termios.error",
+                    &vm.ctx.exceptions.os_error,
+                    Default::default(),
+                )
+            })
+            .clone()
+    }
+}

vm/src/stdlib/tokenize.rs

@@ -1,26 +0,0 @@
-/*
- * python tokenize module.
- */
-pub(crate) use decl::make_module;
-
-// TODO: create main function when called with -m
-#[pymodule(name = "tokenize")]
-mod decl {
-    use crate::builtins::pystr::PyStrRef;
-    use crate::pyobject::{BorrowValue, PyResult};
-    use crate::vm::VirtualMachine;
-    use rustpython_parser::lexer;
-
-    #[pyfunction]
-    fn tokenize(s: PyStrRef, vm: &VirtualMachine) -> PyResult {
-        let source = s.borrow_value();
-
-        // TODO: implement generator when the time has come.
-        let lexer1 = lexer::make_tokenizer(source);
-
-        let tokens = lexer1
-            .map(|st| vm.ctx.new_str(format!("{:?}", st.unwrap().1)))
-            .collect();
-        Ok(vm.ctx.new_list(tokens))
-    }
-}

vm/src/vm.rs

@@ -991,7 +991,9 @@ impl VirtualMachine {
         PyMethod::get_special(obj, method, self)
     }
 
-    pub(crate) fn call_special_method(
+    /// NOT PUBLIC API
+    #[doc(hidden)]
+    pub fn call_special_method(
         &self,
         obj: PyObjectRef,
         method: &str,