From 97f2da7778422dd7b38d998f4a62008201aefff5 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 12:09:48 -0500 Subject: [PATCH 01/16] Don't panic on invalid utf8 compile(), add __get__/__delete__ to getset/property --- vm/src/builtins/getset.rs | 13 +++++++++++++ vm/src/builtins/make_module.rs | 3 ++- vm/src/builtins/property.rs | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/vm/src/builtins/getset.rs b/vm/src/builtins/getset.rs index 8b91ca62d..dffcbc116 100644 --- a/vm/src/builtins/getset.rs +++ b/vm/src/builtins/getset.rs @@ -336,6 +336,19 @@ impl PyGetSet { } } } + #[pymethod] + fn __set__( + zelf: PyObjectRef, + obj: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::descr_set(zelf, obj, Some(value), vm) + } + #[pymethod] + fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Self::descr_set(zelf, obj, None, vm) + } #[pyproperty(magic)] fn name(&self) -> String { diff --git a/vm/src/builtins/make_module.rs b/vm/src/builtins/make_module.rs index d48e5b9f1..10cdf3079 100644 --- a/vm/src/builtins/make_module.rs +++ b/vm/src/builtins/make_module.rs @@ -157,7 +157,8 @@ mod decl { // TODO: compile::compile should probably get bytes let source = match &source { Either::A(string) => string.borrow_value(), - Either::B(bytes) => std::str::from_utf8(bytes).unwrap(), + Either::B(bytes) => std::str::from_utf8(bytes) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?, }; let flags = args diff --git a/vm/src/builtins/property.rs b/vm/src/builtins/property.rs index 291743481..1c44795b4 100644 --- a/vm/src/builtins/property.rs +++ b/vm/src/builtins/property.rs @@ -136,6 +136,19 @@ impl PyProperty { } } } + #[pymethod] + fn __set__( + zelf: PyObjectRef, + obj: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::descr_set(zelf, obj, Some(value), vm) + } + #[pymethod] + fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Self::descr_set(zelf, obj, None, vm) + } // Access functions From 0056cbc880e83b2411878ac3075f7b45f2127924 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 12:10:10 -0500 Subject: [PATCH 02/16] Remove builtin tokenize --- vm/src/stdlib/mod.rs | 3 --- vm/src/stdlib/tokenize.rs | 26 -------------------------- 2 files changed, 29 deletions(-) delete mode 100644 vm/src/stdlib/tokenize.rs diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 8aa50a383..77865c837 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -37,8 +37,6 @@ mod sysconfigdata; #[cfg(feature = "threading")] mod thread; mod time_module; -#[cfg(feature = "rustpython-parser")] -mod tokenize; mod unicodedata; mod warnings; mod weakref; @@ -118,7 +116,6 @@ pub fn get_module_inits() -> HashMap #[cfg(feature = "rustpython-parser")] { modules.insert("keyword".to_owned(), Box::new(keyword::make_module)); - modules.insert("tokenize".to_owned(), Box::new(tokenize::make_module)); } // Insert compiler related modules: diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs deleted file mode 100644 index 3e1a97a01..000000000 --- a/vm/src/stdlib/tokenize.rs +++ /dev/null @@ -1,26 +0,0 @@ -/* - * python tokenize module. - */ -pub(crate) use decl::make_module; - -// TODO: create main function when called with -m -#[pymodule(name = "tokenize")] -mod decl { - use crate::builtins::pystr::PyStrRef; - use crate::pyobject::{BorrowValue, PyResult}; - use crate::vm::VirtualMachine; - use rustpython_parser::lexer; - - #[pyfunction] - fn tokenize(s: PyStrRef, vm: &VirtualMachine) -> PyResult { - let source = s.borrow_value(); - - // TODO: implement generator when the time has come. - let lexer1 = lexer::make_tokenizer(source); - - let tokens = lexer1 - .map(|st| vm.ctx.new_str(format!("{:?}", st.unwrap().1))) - .collect(); - Ok(vm.ctx.new_list(tokens)) - } -} From 80249debe120eb7d3ce5858de9995197829e2803 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 12:10:37 -0500 Subject: [PATCH 03/16] Add tokenize.py + update token.py to versions from CPython 3.8 --- Lib/token.py | 137 +++++----- Lib/tokenize.py | 682 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 747 insertions(+), 72 deletions(-) create mode 100644 Lib/tokenize.py diff --git a/Lib/token.py b/Lib/token.py index 5fdb22213..493bf0426 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,15 +1,8 @@ -"""Token constants (from "token.h").""" +"""Token constants.""" +# Auto-generated by Tools/scripts/generate_token.py __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] -# This file is automatically generated; please don't muck it up! -# -# To update the symbols in this file, 'cd' to the top directory of -# the python source tree after building the interpreter and run: -# -# ./python Lib/token.py - -#--start constants-- ENDMARKER = 0 NAME = 1 NUMBER = 2 @@ -63,19 +56,76 @@ AT = 49 ATEQUAL = 50 RARROW = 51 ELLIPSIS = 52 -OP = 53 -AWAIT = 54 -ASYNC = 55 -ERRORTOKEN = 56 -N_TOKENS = 57 +COLONEQUAL = 53 +OP = 54 +AWAIT = 55 +ASYNC = 56 +TYPE_IGNORE = 57 +TYPE_COMMENT = 58 +# These aren't used by the C tokenizer but are needed for tokenize.py +ERRORTOKEN = 59 +COMMENT = 60 +NL = 61 +ENCODING = 62 +N_TOKENS = 63 +# Special definitions for cooperation with parser NT_OFFSET = 256 -#--end constants-- tok_name = {value: name for name, value in globals().items() if isinstance(value, int) and not name.startswith('_')} __all__.extend(tok_name.values()) +EXACT_TOKEN_TYPES = { + '!=': NOTEQUAL, + '%': PERCENT, + '%=': PERCENTEQUAL, + '&': AMPER, + '&=': AMPEREQUAL, + '(': LPAR, + ')': RPAR, + '*': STAR, + '**': DOUBLESTAR, + '**=': DOUBLESTAREQUAL, + '*=': STAREQUAL, + '+': PLUS, + '+=': PLUSEQUAL, + ',': COMMA, + '-': MINUS, + '-=': MINEQUAL, + '->': RARROW, + '.': DOT, + '...': ELLIPSIS, + '/': SLASH, + '//': DOUBLESLASH, + '//=': DOUBLESLASHEQUAL, + '/=': SLASHEQUAL, + ':': COLON, + ':=': COLONEQUAL, + ';': SEMI, + '<': LESS, + '<<': LEFTSHIFT, + '<<=': LEFTSHIFTEQUAL, + '<=': LESSEQUAL, + '=': EQUAL, + '==': EQEQUAL, + '>': GREATER, + '>=': GREATEREQUAL, + '>>': RIGHTSHIFT, + '>>=': RIGHTSHIFTEQUAL, + '@': AT, + '@=': ATEQUAL, + '[': LSQB, + ']': RSQB, + '^': CIRCUMFLEX, + '^=': CIRCUMFLEXEQUAL, + '{': LBRACE, + '|': VBAR, + '|=': VBAREQUAL, + '}': RBRACE, + '~': TILDE, +} + def ISTERMINAL(x): return x < NT_OFFSET @@ -84,60 +134,3 @@ def ISNONTERMINAL(x): def ISEOF(x): return x == ENDMARKER - - -def _main(): - import re - import sys - args = sys.argv[1:] - inFileName = args and args[0] or "Include/token.h" - outFileName = "Lib/token.py" - if len(args) > 1: - outFileName = args[1] - try: - fp = open(inFileName) - except OSError as err: - sys.stdout.write("I/O error: %s\n" % str(err)) - sys.exit(1) - with fp: - lines = fp.read().split("\n") - prog = re.compile( - "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)", - re.IGNORECASE) - tokens = {} - for line in lines: - match = prog.match(line) - if match: - name, val = match.group(1, 2) - val = int(val) - tokens[val] = name # reverse so we can sort them... - keys = sorted(tokens.keys()) - # load the output skeleton from the target: - try: - fp = open(outFileName) - except OSError as err: - sys.stderr.write("I/O error: %s\n" % str(err)) - sys.exit(2) - with fp: - format = fp.read().split("\n") - try: - start = format.index("#--start constants--") + 1 - end = format.index("#--end constants--") - except ValueError: - sys.stderr.write("target does not contain format markers") - sys.exit(3) - lines = [] - for val in keys: - lines.append("%s = %d" % (tokens[val], val)) - format[start:end] = lines - try: - fp = open(outFileName, 'w') - except OSError as err: - sys.stderr.write("I/O error: %s\n" % str(err)) - sys.exit(4) - with fp: - fp.write("\n".join(format)) - - -if __name__ == "__main__": - _main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py new file mode 100644 index 000000000..1aee21b5e --- /dev/null +++ b/Lib/tokenize.py @@ -0,0 +1,682 @@ +"""Tokenization help for Python programs. + +tokenize(readline) is a generator that breaks a stream of bytes into +Python tokens. It decodes the bytes according to PEP-0263 for +determining source file encoding. + +It accepts a readline-like method which is called repeatedly to get the +next line of input (or b"" for EOF). It generates 5-tuples with these +members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators. Additionally, all token lists start with an ENCODING token +which tells you which encoding was used to decode the bytes stream. +""" + +__author__ = 'Ka-Ping Yee ' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' + 'Michael Foord') +from builtins import open as _builtin_open +from codecs import lookup, BOM_UTF8 +import collections +from io import TextIOWrapper +import itertools as _itertools +import re +import sys +from token import * +from token import EXACT_TOKEN_TYPES + +cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + +import token +__all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", + "untokenize", "TokenInfo"] +del token + +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): + def __repr__(self): + annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) + return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % + self._replace(type=annotated_type)) + + @property + def exact_type(self): + if self.type == OP and self.string in EXACT_TOKEN_TYPES: + return EXACT_TOKEN_TYPES[self.string] + else: + return self.type + +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO](?:_?[0-7])+' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permutations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {''} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group(StringPrefix + "'''", StringPrefix + '"""') +# Single-line ' or " string. +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Sorting in reverse order puts the long operators before their prefixes. +# Otherwise if = came before ==, == would get recognized as two instances +# of =. +Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) +Funny = group(r'\r?\n', Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) + +tabsize = 8 + +class TokenError(Exception): pass + +class StopTokenizing(Exception): pass + + +class Untokenizer: + + def __init__(self): + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + self.encoding = None + + def add_whitespace(self, start): + row, col = start + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(" " * col_offset) + + def untokenize(self, iterable): + it = iter(iterable) + indents = [] + startline = False + for t in it: + if len(t) == 2: + self.compat(t, it) + break + tok_type, token, start, end, line = t + if tok_type == ENCODING: + self.encoding = token + continue + if tok_type == ENDMARKER: + break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return "".join(self.tokens) + + def compat(self, token, iterable): + indents = [] + toks_append = self.tokens.append + startline = token[0] in (NEWLINE, NL) + prevstring = False + + for tok in _itertools.chain([token], iterable): + toknum, tokval = tok[:2] + if toknum == ENCODING: + self.encoding = tokval + continue + + if toknum in (NAME, NUMBER): + tokval += ' ' + + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + + +def untokenize(iterable): + """Transform tokens back into Python source code. + It returns a bytes object, encoded using the ENCODING + token, which is the first token sequence output by tokenize. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited input: + # Output bytes will tokenize back to the input + t1 = [tok[:2] for tok in tokenize(f.readline)] + newcode = untokenize(t1) + readline = BytesIO(newcode).readline + t2 = [tok[:2] for tok in tokenize(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + out = ut.untokenize(iterable) + if ut.encoding is not None: + out = out.encode(ut.encoding) + return out + + +def _get_normal_name(orig_enc): + """Imitates get_normal_name in tokenizer.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace("_", "-") + if enc == "utf-8" or enc.startswith("utf-8-"): + return "utf-8" + if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ + enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): + return "iso-8859-1" + return orig_enc + +def detect_encoding(readline): + """ + The detect_encoding() function is used to detect the encoding that should + be used to decode a Python source file. It requires one argument, readline, + in the same way as the tokenize() generator. + + It will call readline a maximum of twice, and return the encoding used + (as a string) and a list of any lines (left as bytes) it has read in. + + It detects the encoding from the presence of a utf-8 bom or an encoding + cookie as specified in pep-0263. If both a bom and a cookie are present, + but disagree, a SyntaxError will be raised. If the encoding cookie is an + invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, + 'utf-8-sig' is returned. + + If no encoding is specified, then the default of 'utf-8' will be returned. + """ + try: + filename = readline.__self__.name + except AttributeError: + filename = None + bom_found = False + encoding = None + default = 'utf-8' + def read_or_stop(): + try: + return readline() + except StopIteration: + return b'' + + def find_cookie(line): + try: + # Decode as UTF-8. Either the line is an encoding declaration, + # in which case it should be pure ASCII, or it must be UTF-8 + # per default encoding. + line_string = line.decode('utf-8') + except UnicodeDecodeError: + msg = "invalid or missing encoding declaration" + if filename is not None: + msg = '{} for {!r}'.format(msg, filename) + raise SyntaxError(msg) + + match = cookie_re.match(line_string) + if not match: + return None + encoding = _get_normal_name(match.group(1)) + try: + codec = lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + if filename is None: + msg = "unknown encoding: " + encoding + else: + msg = "unknown encoding for {!r}: {}".format(filename, + encoding) + raise SyntaxError(msg) + + if bom_found: + if encoding != 'utf-8': + # This behaviour mimics the Python interpreter + if filename is None: + msg = 'encoding problem: utf-8' + else: + msg = 'encoding problem for {!r}: utf-8'.format(filename) + raise SyntaxError(msg) + encoding += '-sig' + return encoding + + first = read_or_stop() + if first.startswith(BOM_UTF8): + bom_found = True + first = first[3:] + default = 'utf-8-sig' + if not first: + return default, [] + + encoding = find_cookie(first) + if encoding: + return encoding, [first] + if not blank_re.match(first): + return default, [first] + + second = read_or_stop() + if not second: + return default, [first] + + encoding = find_cookie(second) + if encoding: + return encoding, [first, second] + + return default, [first, second] + + +def open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + buffer = _builtin_open(filename, 'rb') + try: + encoding, lines = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text + except: + buffer.close() + raise + + +def tokenize(readline): + """ + The tokenize() generator requires one argument, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as bytes. Alternatively, readline + can be a callable function terminating with StopIteration: + readline = open(myfile, 'rb').__next__ # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + physical line. + + The first token sequence will always be an ENCODING token + which tells you which encoding was used to decode the bytes stream. + """ + encoding, consumed = detect_encoding(readline) + empty = _itertools.repeat(b"") + rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) + return _tokenize(rl_gen.__next__, encoding) + + +def _tokenize(readline, encoding): + lnum = parenlev = continued = 0 + numchars = '0123456789' + contstr, needcont = '', 0 + contline = None + indents = [0] + + if encoding is not None: + if encoding == "utf-8-sig": + # BOM will already have been stripped. + encoding = "utf-8" + yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') + last_line = b'' + line = b'' + while True: # loop over lines in stream + try: + # We capture the value of the line variable here because + # readline uses the empty string '' to signal end of input, + # hence `line` itself will always be overwritten at the end + # of this loop. + last_line = line + line = readline() + except StopIteration: + line = b'' + + if encoding is not None: + line = line.decode(encoding) + lnum += 1 + pos, max = 0, len(line) + + if contstr: # continued string + if not line: + raise TokenError("EOF in multi-line string", strstart) + endmatch = endprog.match(line) + if endmatch: + pos = end = endmatch.end(0) + yield TokenInfo(STRING, contstr + line[:end], + strstart, (lnum, end), contline + line) + contstr, needcont = '', 0 + contline = None + elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': + yield TokenInfo(ERRORTOKEN, contstr + line, + strstart, (lnum, len(line)), contline) + contstr = '' + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + elif parenlev == 0 and not continued: # new statement + if not line: break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break + + if line[pos] in '#\r\n': # skip comments or blank lines + if line[pos] == '#': + comment_token = line[pos:].rstrip('\r\n') + yield TokenInfo(COMMENT, comment_token, + (lnum, pos), (lnum, pos + len(comment_token)), line) + pos += len(comment_token) + + yield TokenInfo(NL, line[pos:], + (lnum, pos), (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents or dedents + indents.append(column) + yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level", + ("", lnum, pos, line)) + indents = indents[:-1] + + yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) + + else: # continued statement + if not line: + raise TokenError("EOF in multi-line statement", (lnum, 0)) + continued = 0 + + while pos < max: + pseudomatch = _compile(PseudoToken).match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue + token, initial = line[start:end], line[start] + + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): + yield TokenInfo(NUMBER, token, spos, epos, line) + elif initial in '\r\n': + if parenlev > 0: + yield TokenInfo(NL, token, spos, epos, line) + else: + yield TokenInfo(NEWLINE, token, spos, epos, line) + + elif initial == '#': + assert not token.endswith("\n") + yield TokenInfo(COMMENT, token, spos, epos, line) + + elif token in triple_quoted: + endprog = _compile(endpats[token]) + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + yield TokenInfo(STRING, token, spos, (lnum, pos), line) + else: + strstart = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come after + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): + if token[-1] == '\n': # continued string + strstart = (lnum, start) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + yield TokenInfo(STRING, token, spos, epos, line) + + elif initial.isidentifier(): # ordinary name + yield TokenInfo(NAME, token, spos, epos, line) + elif initial == '\\': # continued stmt + continued = 1 + else: + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 + yield TokenInfo(OP, token, spos, epos, line) + else: + yield TokenInfo(ERRORTOKEN, line[pos], + (lnum, pos), (lnum, pos+1), line) + pos += 1 + + # Add an implicit NEWLINE if the input doesn't end in one + if last_line and last_line[-1] not in '\r\n': + yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') + for indent in indents[1:]: # pop remaining indent levels + yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +def generate_tokens(readline): + """Tokenize a source reading Python code as unicode strings. + + This has the same API as tokenize(), except that it expects the *readline* + callable to return str objects instead of bytes. + """ + return _tokenize(readline, None) + +def main(): + import argparse + + # Helper error handling routines + def perror(message): + sys.stderr.write(message) + sys.stderr.write('\n') + + def error(message, filename=None, location=None): + if location: + args = (filename,) + location + (message,) + perror("%s:%d:%d: error: %s" % args) + elif filename: + perror("%s: error: %s" % (filename, message)) + else: + perror("error: %s" % message) + sys.exit(1) + + # Parse the arguments and options + parser = argparse.ArgumentParser(prog='python -m tokenize') + parser.add_argument(dest='filename', nargs='?', + metavar='filename.py', + help='the file to tokenize; defaults to stdin') + parser.add_argument('-e', '--exact', dest='exact', action='store_true', + help='display token names using the exact type') + args = parser.parse_args() + + try: + # Tokenize the input + if args.filename: + filename = args.filename + with _builtin_open(filename, 'rb') as f: + tokens = list(tokenize(f.readline)) + else: + filename = "" + tokens = _tokenize(sys.stdin.readline, None) + + # Output the tokenization + for token in tokens: + token_type = token.type + if args.exact: + token_type = token.exact_type + token_range = "%d,%d-%d,%d:" % (token.start + token.end) + print("%-20s%-15s%-15r" % + (token_range, tok_name[token_type], token.string)) + except IndentationError as err: + line, column = err.args[1][1:3] + error(err.args[0], filename, (line, column)) + except TokenError as err: + line, column = err.args[1] + error(err.args[0], filename, (line, column)) + except SyntaxError as err: + error(err, filename) + except OSError as err: + error(err) + except KeyboardInterrupt: + print("interrupted\n") + except Exception as err: + perror("unexpected error: %s" % err) + raise + +if __name__ == "__main__": + main() From 044f9c1a999103c334865157604dcc0b20abb5cb Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 12:15:22 -0500 Subject: [PATCH 04/16] Add line_buffering arg to TextIOWrapper --- vm/src/stdlib/io.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index c33c3c882..f19fd1338 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1787,6 +1787,8 @@ mod _io { errors: Option, #[pyarg(any, default)] newline: Option, + #[pyarg(any, default = "false")] + line_buffering: bool, } impl TextIOWrapperArgs { @@ -1813,6 +1815,8 @@ mod _io { errors: PyStrRef, // TODO: respect newline newline: Option, + // TODO: respect line_buffering + line_buffering: bool, } #[pyattr] #[pyclass(name = "TextIOWrapper", base = "_TextIOBase")] @@ -1872,6 +1876,7 @@ mod _io { encoding, errors, newline: args.newline, + line_buffering: args.line_buffering, }); Ok(()) From 89ef71a83a87bc07b184203e77edc7ad3382ecd6 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 18:54:47 -0500 Subject: [PATCH 05/16] Fix tab completion for classes --- src/shell/helper.rs | 19 ++++++++++--------- vm/src/vm.rs | 4 +++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 13ec29f60..560441791 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -58,16 +58,16 @@ impl<'vm> ShellHelper<'vm> { fn get_available_completions<'w>( &self, words: &'w [String], - ) -> Option<(&'w str, Box> + 'vm>)> { + ) -> Option<(&'w str, impl Iterator> + 'vm)> { // the very first word and then all the ones after the dot let (first, rest) = words.split_first().unwrap(); let str_iter_method = |obj, name| { - let iter = self.vm.call_method(obj, name, ())?; + let iter = self.vm.call_special_method(obj, name, ())?; PyIterable::::try_from_object(self.vm, iter)?.iter(self.vm) }; - if let Some((last, parents)) = rest.split_last() { + let (word_start, iter1, iter2) = if let Some((last, parents)) = rest.split_last() { // we need to get an attribute based off of the dir() of an object // last: the last word, could be empty if it ends with a dot @@ -82,16 +82,17 @@ impl<'vm> ShellHelper<'vm> { current = self.vm.get_attribute(current.clone(), attr.as_str()).ok()?; } - let current_iter = str_iter_method(¤t, "__dir__").ok()?; + let current_iter = str_iter_method(current, "__dir__").ok()?; - Some((&last, Box::new(current_iter) as _)) + (last, current_iter, None) } else { // we need to get a variable based off of globals/builtins - let globals = str_iter_method(self.globals.as_object(), "keys").ok()?; - let builtins = str_iter_method(&self.vm.builtins, "__dir__").ok()?; - Some((&first, Box::new(Iterator::chain(globals, builtins)) as _)) - } + let globals = str_iter_method(self.globals.as_object().clone(), "keys").ok()?; + let builtins = str_iter_method(self.vm.builtins.clone(), "__dir__").ok()?; + (first, globals, Some(builtins)) + }; + Some((word_start, iter1.chain(iter2.into_iter().flatten()))) } fn complete_opt(&self, line: &str) -> Option<(usize, Vec)> { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 1c6f6365a..8cf9129c9 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -991,7 +991,9 @@ impl VirtualMachine { PyMethod::get_special(obj, method, self) } - pub(crate) fn call_special_method( + /// NOT PUBLIC API + #[doc(hidden)] + pub fn call_special_method( &self, obj: PyObjectRef, method: &str, From c2345698babfe7654077219180ba541b8d84a95e Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 18:55:35 -0500 Subject: [PATCH 06/16] Add fileno() to IOBase --- vm/src/stdlib/io.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index f19fd1338..3541be594 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -314,8 +314,12 @@ mod _io { fn truncate(zelf: PyObjectRef, _pos: OptionalArg, vm: &VirtualMachine) -> PyResult { _unsupported(vm, &zelf, "truncate") } - #[pyattr] + #[pymethod] + fn fileno(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "truncate") + } + #[pyattr] fn __closed(ctx: &PyContext) -> PyObjectRef { ctx.new_bool(false) } From 3db00be5f3f7205c8fe6ad8c10f817c599e9db5e Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 18:55:44 -0500 Subject: [PATCH 07/16] Add termios module --- Cargo.lock | 10 ++++ vm/Cargo.toml | 3 + vm/src/builtins/mod.rs | 2 +- vm/src/stdlib/mod.rs | 5 ++ vm/src/stdlib/termios.rs | 122 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 vm/src/stdlib/termios.rs diff --git a/Cargo.lock b/Cargo.lock index cda08d499..dc131e66d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2091,6 +2091,7 @@ dependencies = [ "socket2", "sre-engine", "static_assertions", + "termios", "thiserror", "thread_local", "timsort", @@ -2427,6 +2428,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index c88cbfb0a..5d476f25f 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -134,6 +134,9 @@ num_cpus = "1" [target.'cfg(not(any(target_arch = "wasm32", target_os = "redox")))'.dependencies] dns-lookup = "1.0" +[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] +termios = "0.3" + [target.'cfg(windows)'.dependencies] winreg = "0.8.0" schannel = "0.1" diff --git a/vm/src/builtins/mod.rs b/vm/src/builtins/mod.rs index d7478b180..f35ba72e0 100644 --- a/vm/src/builtins/mod.rs +++ b/vm/src/builtins/mod.rs @@ -35,7 +35,7 @@ pub use int::{PyInt, PyIntRef}; pub(crate) mod iter; pub use iter::*; pub(crate) mod list; -pub use list::PyList; +pub use list::{PyList, PyListRef}; pub(crate) mod map; pub use map::PyMap; pub(crate) mod mappingproxy; diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 77865c837..ffa90764f 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -62,6 +62,8 @@ mod select; pub mod signal; #[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))] mod ssl; +#[cfg(all(unix, not(target_os = "redox")))] +mod termios; #[cfg(windows)] mod winapi; #[cfg(windows)] @@ -153,6 +155,9 @@ pub fn get_module_inits() -> HashMap modules.insert("pwd".to_owned(), Box::new(pwd::make_module)); } + #[cfg(all(unix, not(target_os = "redox")))] + modules.insert("termios".to_owned(), Box::new(termios::make_module)); + #[cfg(unix)] { modules.insert( diff --git a/vm/src/stdlib/termios.rs b/vm/src/stdlib/termios.rs new file mode 100644 index 000000000..50ebf93ac --- /dev/null +++ b/vm/src/stdlib/termios.rs @@ -0,0 +1,122 @@ +pub(crate) use self::termios::make_module; + +#[pymodule] +mod termios { + use crate::builtins::{int, PyBytes, PyInt, PyListRef, PyTypeRef}; + use crate::common::borrow::BorrowValue; + use crate::exceptions::PyBaseExceptionRef; + use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject}; + use crate::VirtualMachine; + use std::convert::TryFrom; + use termios::Termios; + + #[pyattr] + use termios::{ + os::target::NCCS, B0, B110, B1200, B134, B150, B1800, B19200, B200, B2400, B300, B38400, + B4800, B50, B600, B75, B9600, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB, + ECHO, ECHOE, ECHOK, ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR, + INPCK, ISIG, ISTRIP, IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST, + PARENB, PARMRK, PARODD, TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON, + TCSADRAIN, TCSAFLUSH, TCSANOW, TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT, + VSTART, VSTOP, VSUSP, VTIME, + }; + + #[pyfunction] + fn tcgetattr(fd: i32, vm: &VirtualMachine) -> PyResult { + let termios = Termios::from_fd(fd).map_err(|e| termios_error(e, vm))?; + let noncanon = (termios.c_lflag & termios::ICANON) == 0; + let cc = termios + .c_cc + .iter() + .enumerate() + .map(|(i, &c)| match i { + termios::VMIN | termios::VTIME if noncanon => vm.ctx.new_int(c), + _ => vm.ctx.new_bytes(vec![c as u8]), + }) + .collect::>(); + let out = vec![ + termios.c_iflag.into_pyobject(vm), + termios.c_oflag.into_pyobject(vm), + termios.c_cflag.into_pyobject(vm), + termios.c_lflag.into_pyobject(vm), + termios::cfgetispeed(&termios).into_pyobject(vm), + termios::cfgetospeed(&termios).into_pyobject(vm), + vm.ctx.new_list(cc), + ]; + Ok(vm.ctx.new_list(out)) + } + + #[pyfunction] + fn tcsetattr(fd: i32, when: i32, attributes: PyListRef, vm: &VirtualMachine) -> PyResult<()> { + let [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] = + <&[PyObjectRef; 7]>::try_from(&**attributes.borrow_value()) + .map_err(|_| { + vm.new_type_error("tcsetattr, arg 3: must be 7 element list".to_owned()) + })? + .clone(); + let mut termios = Termios::from_fd(fd).map_err(|e| termios_error(e, vm))?; + termios.c_iflag = TryFromObject::try_from_object(vm, iflag)?; + termios.c_oflag = TryFromObject::try_from_object(vm, oflag)?; + termios.c_cflag = TryFromObject::try_from_object(vm, cflag)?; + termios.c_lflag = TryFromObject::try_from_object(vm, lflag)?; + termios::cfsetispeed(&mut termios, TryFromObject::try_from_object(vm, ispeed)?) + .map_err(|e| termios_error(e, vm))?; + termios::cfsetospeed(&mut termios, TryFromObject::try_from_object(vm, ospeed)?) + .map_err(|e| termios_error(e, vm))?; + let cc = PyListRef::try_from_object(vm, cc)?; + { + let cc = cc.borrow_value(); + let cc = <&[PyObjectRef; NCCS]>::try_from(&**cc).map_err(|_| { + vm.new_type_error(format!( + "tcsetattr: attributes[6] must be {} element list", + NCCS + )) + })?; + for (cc, x) in termios.c_cc.iter_mut().zip(cc.iter()) { + *cc = if let Some(c) = x + .payload::() + .filter(|b| b.borrow_value().len() == 1) + { + c.borrow_value()[0] as _ + } else if let Some(i) = x.payload::() { + int::try_to_primitive(i.borrow_value(), vm)? + } else { + return Err(vm.new_type_error( + "tcsetattr: elements of attributes must be characters or integers" + .to_owned(), + )); + }; + } + } + + termios::tcsetattr(fd, when, &termios).map_err(|e| termios_error(e, vm))?; + + Ok(()) + } + + fn termios_error(err: std::io::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception( + get_termios_error(vm), + vec![ + err.raw_os_error().into_pyobject(vm), + vm.ctx.new_str(err.to_string()), + ], + ) + } + + #[pyattr(name = "error")] + fn get_termios_error(vm: &VirtualMachine) -> PyTypeRef { + rustpython_common::static_cell! { + static TERMIOS_ERROR: PyTypeRef; + } + TERMIOS_ERROR + .get_or_init(|| { + vm.ctx.new_class( + "termios.error", + &vm.ctx.exceptions.os_error, + Default::default(), + ) + }) + .clone() + } +} From 5239e82f0b191d3bafd118202c8647246ffb2ae1 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 18:55:59 -0500 Subject: [PATCH 08/16] Add pty.py and tty.py from CPython 3.8 --- Lib/pty.py | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Lib/tty.py | 36 +++++++++++ 2 files changed, 208 insertions(+) create mode 100644 Lib/pty.py create mode 100644 Lib/tty.py diff --git a/Lib/pty.py b/Lib/pty.py new file mode 100644 index 000000000..a32432041 --- /dev/null +++ b/Lib/pty.py @@ -0,0 +1,172 @@ +"""Pseudo terminal utilities.""" + +# Bugs: No signal handling. Doesn't set slave termios and window size. +# Only tested on Linux. +# See: W. Richard Stevens. 1992. Advanced Programming in the +# UNIX Environment. Chapter 19. +# Author: Steen Lumholt -- with additions by Guido. + +from select import select +import os +import sys +import tty + +__all__ = ["openpty","fork","spawn"] + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +CHILD = 0 + +def openpty(): + """openpty() -> (master_fd, slave_fd) + Open a pty master/slave pair, using os.openpty() if possible.""" + + try: + return os.openpty() + except (AttributeError, OSError): + pass + master_fd, slave_name = _open_terminal() + slave_fd = slave_open(slave_name) + return master_fd, slave_fd + +def master_open(): + """master_open() -> (master_fd, slave_name) + Open a pty master and return the fd, and the filename of the slave end. + Deprecated, use openpty() instead.""" + + try: + master_fd, slave_fd = os.openpty() + except (AttributeError, OSError): + pass + else: + slave_name = os.ttyname(slave_fd) + os.close(slave_fd) + return master_fd, slave_name + + return _open_terminal() + +def _open_terminal(): + """Open pty master and return (master_fd, tty_name).""" + for x in 'pqrstuvwxyzPQRST': + for y in '0123456789abcdef': + pty_name = '/dev/pty' + x + y + try: + fd = os.open(pty_name, os.O_RDWR) + except OSError: + continue + return (fd, '/dev/tty' + x + y) + raise OSError('out of pty devices') + +def slave_open(tty_name): + """slave_open(tty_name) -> slave_fd + Open the pty slave and acquire the controlling terminal, returning + opened filedescriptor. + Deprecated, use openpty() instead.""" + + result = os.open(tty_name, os.O_RDWR) + try: + from fcntl import ioctl, I_PUSH + except ImportError: + return result + try: + ioctl(result, I_PUSH, "ptem") + ioctl(result, I_PUSH, "ldterm") + except OSError: + pass + return result + +def fork(): + """fork() -> (pid, master_fd) + Fork and make the child a session leader with a controlling terminal.""" + + try: + pid, fd = os.forkpty() + except (AttributeError, OSError): + pass + else: + if pid == CHILD: + try: + os.setsid() + except OSError: + # os.forkpty() already set us session leader + pass + return pid, fd + + master_fd, slave_fd = openpty() + pid = os.fork() + if pid == CHILD: + # Establish a new session. + os.setsid() + os.close(master_fd) + + # Slave becomes stdin/stdout/stderr of child. + os.dup2(slave_fd, STDIN_FILENO) + os.dup2(slave_fd, STDOUT_FILENO) + os.dup2(slave_fd, STDERR_FILENO) + if (slave_fd > STDERR_FILENO): + os.close (slave_fd) + + # Explicitly open the tty to make it become a controlling tty. + tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) + os.close(tmp_fd) + else: + os.close(slave_fd) + + # Parent and child process. + return pid, master_fd + +def _writen(fd, data): + """Write all the data to a descriptor.""" + while data: + n = os.write(fd, data) + data = data[n:] + +def _read(fd): + """Default read function.""" + return os.read(fd, 1024) + +def _copy(master_fd, master_read=_read, stdin_read=_read): + """Parent copy loop. + Copies + pty master -> standard output (master_read) + standard input -> pty master (stdin_read)""" + fds = [master_fd, STDIN_FILENO] + while True: + rfds, wfds, xfds = select(fds, [], []) + if master_fd in rfds: + data = master_read(master_fd) + if not data: # Reached EOF. + fds.remove(master_fd) + else: + os.write(STDOUT_FILENO, data) + if STDIN_FILENO in rfds: + data = stdin_read(STDIN_FILENO) + if not data: + fds.remove(STDIN_FILENO) + else: + _writen(master_fd, data) + +def spawn(argv, master_read=_read, stdin_read=_read): + """Create a spawned process.""" + if type(argv) == type(''): + argv = (argv,) + sys.audit('pty.spawn', argv) + pid, master_fd = fork() + if pid == CHILD: + os.execlp(argv[0], *argv) + try: + mode = tty.tcgetattr(STDIN_FILENO) + tty.setraw(STDIN_FILENO) + restore = 1 + except tty.error: # This is the same as termios.error + restore = 0 + try: + _copy(master_fd, master_read, stdin_read) + except OSError: + if restore: + tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode) + + os.close(master_fd) + return os.waitpid(pid, 0)[1] diff --git a/Lib/tty.py b/Lib/tty.py new file mode 100644 index 000000000..a72eb6755 --- /dev/null +++ b/Lib/tty.py @@ -0,0 +1,36 @@ +"""Terminal utilities.""" + +# Author: Steen Lumholt. + +from termios import * + +__all__ = ["setraw", "setcbreak"] + +# Indexes for termios list. +IFLAG = 0 +OFLAG = 1 +CFLAG = 2 +LFLAG = 3 +ISPEED = 4 +OSPEED = 5 +CC = 6 + +def setraw(fd, when=TCSAFLUSH): + """Put terminal into a raw mode.""" + mode = tcgetattr(fd) + mode[IFLAG] = mode[IFLAG] & ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON) + mode[OFLAG] = mode[OFLAG] & ~(OPOST) + mode[CFLAG] = mode[CFLAG] & ~(CSIZE | PARENB) + mode[CFLAG] = mode[CFLAG] | CS8 + mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON | IEXTEN | ISIG) + mode[CC][VMIN] = 1 + mode[CC][VTIME] = 0 + tcsetattr(fd, when, mode) + +def setcbreak(fd, when=TCSAFLUSH): + """Put terminal into a cbreak mode.""" + mode = tcgetattr(fd) + mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON) + mode[CC][VMIN] = 1 + mode[CC][VTIME] = 0 + tcsetattr(fd, when, mode) From 235074faf27f71270005aab84de70b0e4e3b9d55 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 19:37:21 -0500 Subject: [PATCH 09/16] Fully implement SNI in _ssl --- vm/src/stdlib/ssl.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index 180068ce9..95272959e 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -564,6 +564,32 @@ impl PySslContext { SslServerOrClient::Client }; + if let Some(hostname) = &args.server_hostname { + let hostname = hostname.borrow_value(); + if hostname.is_empty() || hostname.starts_with('.') { + return Err(vm.new_value_error( + "server_hostname cannot be an empty string or start with a leading dot." + .to_owned(), + )); + } + let ip = hostname.parse::(); + if ip.is_err() { + ssl.set_hostname(hostname) + .map_err(|e| convert_openssl_error(vm, e))?; + } + if zelf.check_hostname.load() { + if let Ok(ip) = ip { + ssl.param_mut() + .set_ip(ip) + .map_err(|e| convert_openssl_error(vm, e))?; + } else { + ssl.param_mut() + .set_host(hostname) + .map_err(|e| convert_openssl_error(vm, e))?; + } + } + } + let stream = ssl::SslStream::new(ssl, args.sock.clone()) .map_err(|e| convert_openssl_error(vm, e))?; From 4129f0c5c011df49e141fedb4644c8aebe7f620b Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 21:00:06 -0500 Subject: [PATCH 10/16] Fix async for compilation --- bytecode/src/lib.rs | 2 +- compiler/src/compile.rs | 15 ++++----------- vm/src/frame.rs | 1 + 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/bytecode/src/lib.rs b/bytecode/src/lib.rs index 111480054..54db362d6 100644 --- a/bytecode/src/lib.rs +++ b/bytecode/src/lib.rs @@ -978,7 +978,7 @@ impl Instruction { } GetAIter => 0, GetANext => 1, - EndAsyncFor => -1, + EndAsyncFor => -2, } } diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 8b2c7cf3c..0d65bacf3 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -1391,22 +1391,18 @@ impl Compiler { // The thing iterated: self.compile_expression(iter)?; - let check_asynciter_block = if is_async { - let check_asynciter_block = self.new_block(); - + if is_async { self.emit(Instruction::GetAIter); self.switch_to_block(for_block); self.emit(Instruction::SetupExcept { - handler: check_asynciter_block, + handler: else_block, }); self.emit(Instruction::GetANext); self.emit_constant(ConstantData::None); self.emit(Instruction::YieldFrom); self.compile_store(target)?; self.emit(Instruction::PopBlock); - - Some(check_asynciter_block) } else { // Retrieve Iterator self.emit(Instruction::GetIter); @@ -1416,7 +1412,6 @@ impl Compiler { // Start of loop iteration, set targets: self.compile_store(target)?; - None }; let was_in_loop = self.ctx.loop_data; @@ -1425,12 +1420,10 @@ impl Compiler { self.ctx.loop_data = was_in_loop; self.emit(Instruction::Jump { target: for_block }); - if let Some(check_asynciter_block) = check_asynciter_block { - self.switch_to_block(check_asynciter_block); + self.switch_to_block(else_block); + if is_async { self.emit(Instruction::EndAsyncFor); } - - self.switch_to_block(else_block); self.emit(Instruction::PopBlock); self.compile_statements(orelse)?; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2c649034d..e962c28d2 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -904,6 +904,7 @@ impl ExecutingFrame<'_> { } bytecode::Instruction::EndAsyncFor => { let exc = self.pop_value(); + self.pop_value(); // async iterator we were calling __anext__ on if exc.isinstance(&vm.ctx.exceptions.stop_async_iteration) { vm.take_exception().expect("Should have exception in stack"); Ok(None) From 21e6910e1d19616e6eae1689304944388717c14e Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 21:38:47 -0500 Subject: [PATCH 11/16] Misc wrong behavior for function props, module.__init__, type.getattro --- Lib/types.py | 2 +- vm/src/builtins/function.rs | 50 +++++++++++++++++++++-------- vm/src/builtins/function/jitfunc.rs | 6 ++-- vm/src/builtins/module.rs | 16 ++++++--- vm/src/builtins/pytype.rs | 2 -- vm/src/stdlib/signal.rs | 1 + 6 files changed, 54 insertions(+), 23 deletions(-) diff --git a/Lib/types.py b/Lib/types.py index 4567549a9..1f0fa1d54 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -53,7 +53,7 @@ except TypeError: # For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) -# MemberDescriptorType = type(FunctionType.__globals__) +MemberDescriptorType = type(FunctionType.__globals__) del sys, _f, _g, _C # Not for export diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 914d494fd..79d4cd6da 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -38,8 +38,7 @@ pub struct PyFunction { jitted_code: OnceCell, globals: PyDictRef, closure: Option>, - defaults: Option, - kw_only_defaults: Option, + defaults_and_kwdefaults: PyMutex<(Option, Option)>, name: PyMutex, } @@ -58,8 +57,7 @@ impl PyFunction { jitted_code: OnceCell::new(), globals, closure, - defaults, - kw_only_defaults, + defaults_and_kwdefaults: PyMutex::new((defaults, kw_only_defaults)), name, } } @@ -161,10 +159,20 @@ impl PyFunction { ))); } + let mut defaults_and_kwdefaults = None; + // can't be a closure cause it returns a reference to a captured variable :/ + macro_rules! get_defaults { + () => {{ + defaults_and_kwdefaults + .get_or_insert_with(|| self.defaults_and_kwdefaults.lock().clone()) + }}; + } + // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let ndefs = self.defaults.as_ref().map_or(0, |d| d.borrow_value().len()); + let defaults = get_defaults!().0.as_ref().map(|tup| tup.borrow_value()); + let ndefs = defaults.map_or(0, |d| d.len()); let nrequired = code.arg_count - ndefs; @@ -184,9 +192,7 @@ impl PyFunction { ))); } - if let Some(defaults) = &self.defaults { - let defaults = defaults.borrow_value(); - + if let Some(defaults) = defaults { let n = std::cmp::min(nargs, nexpected_args); let i = n.saturating_sub(nrequired); @@ -212,10 +218,8 @@ impl PyFunction { .take(code.kwonlyarg_count) { if slot.is_none() { - if let Some(kw_only_defaults) = &self.kw_only_defaults { - if let Some(default) = - kw_only_defaults.get_item_option(kwarg.clone(), vm)? - { + if let Some(defaults) = &get_defaults!().1 { + if let Some(default) = defaults.get_item_option(kwarg.clone(), vm)? { *slot = Some(default); continue; } @@ -311,12 +315,20 @@ impl PyFunction { #[pyproperty(magic)] fn defaults(&self) -> Option { - self.defaults.clone() + self.defaults_and_kwdefaults.lock().0.clone() + } + #[pyproperty(magic, setter)] + fn set_defaults(&self, defaults: Option) { + self.defaults_and_kwdefaults.lock().0 = defaults } #[pyproperty(magic)] fn kwdefaults(&self) -> Option { - self.kw_only_defaults.clone() + self.defaults_and_kwdefaults.lock().1.clone() + } + #[pyproperty(magic, setter)] + fn set_kwdefaults(&self, kwdefaults: Option) { + self.defaults_and_kwdefaults.lock().1 = kwdefaults } #[pyproperty(magic)] @@ -425,6 +437,16 @@ impl PyBoundMethod { PyBoundMethod { object, function } } + #[pyslot] + fn tp_new( + cls: PyTypeRef, + function: PyObjectRef, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult> { + PyBoundMethod::new(object, function).into_ref_with_type(vm, cls) + } + #[pymethod(magic)] fn repr(&self, vm: &VirtualMachine) -> PyResult { Ok(format!( diff --git a/vm/src/builtins/function/jitfunc.rs b/vm/src/builtins/function/jitfunc.rs index b68cc3baf..3725d08ca 100644 --- a/vm/src/builtins/function/jitfunc.rs +++ b/vm/src/builtins/function/jitfunc.rs @@ -169,8 +169,10 @@ pub(crate) fn get_jit_args<'a>( } } + let (defaults, kwdefaults) = func.defaults_and_kwdefaults.lock().clone(); + // fill in positional defaults - if let Some(defaults) = &func.defaults { + if let Some(defaults) = defaults { let defaults = defaults.borrow_value(); for (i, default) in defaults.iter().enumerate() { let arg_idx = i + func.code.arg_count - defaults.len(); @@ -181,7 +183,7 @@ pub(crate) fn get_jit_args<'a>( } // fill in keyword only defaults - if let Some(kw_only_defaults) = &func.kw_only_defaults { + if let Some(kw_only_defaults) = kwdefaults { for (i, name) in arg_names.kwonlyargs.iter().enumerate() { let arg_idx = i + func.code.arg_count; if !jit_args.is_set(arg_idx) { diff --git a/vm/src/builtins/module.rs b/vm/src/builtins/module.rs index daa36e723..04b63d695 100644 --- a/vm/src/builtins/module.rs +++ b/vm/src/builtins/module.rs @@ -1,7 +1,7 @@ use super::dict::PyDictRef; use super::pystr::{PyStr, PyStrRef}; use super::pytype::PyTypeRef; -use crate::function::{FuncArgs, OptionalOption}; +use crate::function::FuncArgs; use crate::pyobject::{ BorrowValue, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, @@ -43,6 +43,14 @@ pub fn init_module_dict( .expect("Failed to set __spec__ on module"); } +#[derive(FromArgs)] +struct ModuleInitArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(any, default)] + doc: Option, +} + #[pyimpl(with(SlotGetattro), flags(BASETYPE, HAS_DICT))] impl PyModule { #[pyslot] @@ -51,7 +59,7 @@ impl PyModule { } #[pymethod(magic)] - fn init(zelf: PyRef, name: PyStrRef, doc: OptionalOption, vm: &VirtualMachine) { + fn init(zelf: PyRef, args: ModuleInitArgs, vm: &VirtualMachine) { debug_assert!(crate::pyobject::TypeProtocol::class(zelf.as_object()) .slots .flags @@ -59,8 +67,8 @@ impl PyModule { init_module_dict( vm, &zelf.as_object().dict().unwrap(), - name.into_object(), - doc.flatten().into_pyobject(vm), + args.name.into_object(), + args.doc.into_pyobject(vm), ); } diff --git a/vm/src/builtins/pytype.rs b/vm/src/builtins/pytype.rs index adfe80b60..29607e2b8 100644 --- a/vm/src/builtins/pytype.rs +++ b/vm/src/builtins/pytype.rs @@ -536,8 +536,6 @@ impl SlotGetattro for PyType { } else if let Some(attr) = mcl_attr { drop(mcl); vm.call_if_get_descriptor(attr, zelf.into_object()) - } else if let Some(ref getter) = zelf.get_attr("__getattr__") { - vm.invoke(getter, (PyLease::into_pyref(mcl), name_str)) } else { Err(vm.new_attribute_error(format!( "type object '{}' has no attribute '{}'", diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 70f30e2c9..0076db8d4 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -164,6 +164,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "SIGINT" => ctx.new_int(libc::SIGINT as u8), "SIGSEGV" => ctx.new_int(libc::SIGSEGV as u8), "SIGTERM" => ctx.new_int(libc::SIGTERM as u8), + "NSIG" => ctx.new_int(NSIG), "default_int_handler" => int_handler.clone(), }); extend_module_platform_specific(vm, &module); From fd587bd5fe46698819c095c981481e8e79a47304 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 6 Apr 2021 22:03:53 -0500 Subject: [PATCH 12/16] Standardize panicking within frame --- vm/src/frame.rs | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index e962c28d2..321846f0d 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -786,8 +786,8 @@ impl ExecutingFrame<'_> { Ok(None) } } else { - unreachable!( - "Block type must be finally handler when reaching EndFinally instruction!" + self.fatal( + "Block type must be finally handler when reaching EndFinally instruction!", ); } } @@ -820,7 +820,7 @@ impl ExecutingFrame<'_> { let block = self.current_block().unwrap(); let reason = match block.typ { BlockType::FinallyHandler { reason, .. } => reason, - _ => unreachable!("WithCleanupStart expects a FinallyHandler block on stack"), + _ => self.fatal("WithCleanupStart expects a FinallyHandler block on stack"), }; let exc = match reason { Some(UnwindReason::Raising { exception }) => Some(exception), @@ -843,7 +843,7 @@ impl ExecutingFrame<'_> { let block = self.pop_block(); let (reason, prev_exc) = match block.typ { BlockType::FinallyHandler { reason, prev_exc } => (reason, prev_exc), - _ => unreachable!("WithCleanupFinish expects a FinallyHandler block on stack"), + _ => self.fatal("WithCleanupFinish expects a FinallyHandler block on stack"), }; let suppress_exception = pybool::boolval(vm, self.pop_value())?; @@ -1073,7 +1073,7 @@ impl ExecutingFrame<'_> { vm.set_exception(prev_exc); Ok(None) } else { - unreachable!("block type must be ExceptHandler here.") + self.fatal("block type must be ExceptHandler here.") } } bytecode::Instruction::Reverse { amount } => { @@ -1230,7 +1230,7 @@ impl ExecutingFrame<'_> { UnwindReason::Raising { exception } => Err(exception), UnwindReason::Returning { value } => Ok(Some(ExecutionResult::Return(value))), UnwindReason::Break { .. } | UnwindReason::Continue { .. } => { - unreachable!("break or continue must occur within a loop block.") + self.fatal("break or continue must occur within a loop block.") } // UnwindReason::NoWorries => Ok(None), } } @@ -1238,7 +1238,7 @@ impl ExecutingFrame<'_> { fn execute_rotate(&mut self, amount: u32) -> FrameResult { // Shuffles top of stack amount down if amount < 2 { - panic!("Can only rotate two or more values"); + self.fatal("Can only rotate two or more values"); } let mut values = Vec::new(); @@ -1812,17 +1812,14 @@ impl ExecutingFrame<'_> { fn push_value(&mut self, obj: PyObjectRef) { match self.state.stack.try_push(obj) { Ok(()) => {} - Err(_e) => { - dbg!(self); - panic!("tried to push value onto stack but overflowed max_stacksize") - } + Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stacksize"), } } fn pop_value(&mut self) -> PyObjectRef { match self.state.stack.pop() { Some(x) => x, - None => panic!("tried to pop value but there was nothing on the stack"), + None => self.fatal("tried to pop value but there was nothing on the stack"), } } @@ -1839,13 +1836,22 @@ impl ExecutingFrame<'_> { fn last_value_ref(&self) -> &PyObjectRef { match &*self.state.stack { [.., last] => last, - [] => panic!("tried to get top of stack but stack is empty"), + [] => self.fatal("tried to get top of stack but stack is empty"), } } fn nth_value(&self, depth: u32) -> PyObjectRef { self.state.stack[self.state.stack.len() - depth as usize - 1].clone() } + + // redox still has an old nightly, and edition 2021 won't be out for a while + #[allow(non_fmt_panic)] + #[cold] + #[inline(never)] + fn fatal(&self, msg: &'static str) -> ! { + dbg!(self); + panic!(msg) + } } impl fmt::Debug for Frame { From 1b3408f00d7e530e8d76b5d8f1903e5ab9bbeb34 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 7 Apr 2021 12:29:53 -0500 Subject: [PATCH 13/16] Fix defaultdict --- Lib/collections/_defaultdict.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/collections/_defaultdict.py b/Lib/collections/_defaultdict.py index 42635f0d2..f9a79639f 100644 --- a/Lib/collections/_defaultdict.py +++ b/Lib/collections/_defaultdict.py @@ -10,9 +10,11 @@ class defaultdict(dict): def __missing__(self, key): if self.default_factory: - return self.default_factory() + val = self.default_factory() else: raise KeyError(key) + self[key] = val + return val def __repr__(self): return f"defaultdict({self.default_factory}, {dict.__repr__(self)})" From 501c555c547d32e2564e06c142a40cfb032e2f6f Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 7 Apr 2021 16:32:38 -0500 Subject: [PATCH 14/16] Unskip tests --- Lib/test/test_imp.py | 6 ------ Lib/test/test_importlib/import_/test_caching.py | 6 ------ Lib/test/test_importlib/import_/test_meta_path.py | 4 ---- Lib/test/test_posix.py | 2 -- 4 files changed, 18 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 2fd613fea..ead981043 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -103,8 +103,6 @@ class ImportTests(unittest.TestCase): self.assertEqual(fp.readline(), '"""Tokenization help for Python programs.\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue3594(self): temp_mod_name = 'test_imp_helper' sys.path.insert(0, '.') @@ -119,8 +117,6 @@ class ImportTests(unittest.TestCase): support.unlink(temp_mod_name + '.py') support.unlink(temp_mod_name + '.pyc') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue5604(self): # Test cannot cover imp.load_compiled function. # Martin von Loewis note what shared library cannot have non-ascii @@ -215,8 +211,6 @@ class ImportTests(unittest.TestCase): self.assertRaises(SyntaxError, imp.find_module, "badsyntax_pep3120", [path]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_load_from_source(self): # Verify that the imp module can correctly load and find .py files # XXX (ncoghlan): It would be nice to use support.CleanImport diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index 21bdecc6a..8079add5b 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -61,8 +61,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase): # __import__ inconsistent between loaders and built-in import when it comes # to when to use the module in sys.modules and when not to. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_using_cache_after_loader(self): # [from cache on return] with self.create_mock('module') as mock: @@ -71,8 +69,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase): self.assertEqual(id(module), id(sys.modules['module'])) # See test_using_cache_after_loader() for reasoning. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_using_cache_for_assigning_to_attribute(self): # [from cache to attribute] with self.create_mock('pkg.__init__', 'pkg.module') as importer: @@ -83,8 +79,6 @@ class ImportlibUseCache(UseCache, unittest.TestCase): id(sys.modules['pkg.module'])) # See test_using_cache_after_loader() for reasoning. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_using_cache_for_fromlist(self): # [from cache for fromlist] with self.create_mock('pkg.__init__', 'pkg.module') as importer: diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index ad7d2588a..1ee79f3bc 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -67,8 +67,6 @@ class CallSignature: return fxn(*args, **kwargs) return log, wrapper - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_path(self): # [no path] mod_name = 'top_level' @@ -84,8 +82,6 @@ class CallSignature: self.assertEqual(args[0], mod_name) self.assertIsNone(args[1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_with_path(self): # [path set] pkg_name = 'pkg' diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index e4a11b8d1..2782dd6ce 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1769,8 +1769,6 @@ class _PosixSpawnMixin: support.wait_process(pid, exitcode=-signal.SIGUSR1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setsigdef_wrong_type(self): with self.assertRaises(TypeError): self.spawn_func(sys.executable, From 3c3801bbe49036c99b230b2c8de07a03eaa4f5b2 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 7 Apr 2021 20:03:10 -0500 Subject: [PATCH 15/16] Improve defaultdict some more --- Lib/collections/_defaultdict.py | 27 ++++++++++++++++++++++++--- Lib/test/test_copy.py | 2 -- vm/Lib/python_builtins/__reducelib.py | 2 +- vm/src/builtins/function.rs | 15 +++++++++++++-- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/collections/_defaultdict.py b/Lib/collections/_defaultdict.py index f9a79639f..007ca9104 100644 --- a/Lib/collections/_defaultdict.py +++ b/Lib/collections/_defaultdict.py @@ -1,7 +1,11 @@ +from reprlib import recursive_repr as _recursive_repr + class defaultdict(dict): def __init__(self, *args, **kwargs): if len(args) >= 1: default_factory = args[0] + if default_factory is not None and not callable(default_factory): + raise TypeError("first argument must be callable or None") args = args[1:] else: default_factory = None @@ -9,13 +13,30 @@ class defaultdict(dict): self.default_factory = default_factory def __missing__(self, key): - if self.default_factory: + if self.default_factory is not None: val = self.default_factory() else: raise KeyError(key) self[key] = val return val - def __repr__(self): - return f"defaultdict({self.default_factory}, {dict.__repr__(self)})" + @_recursive_repr() + def __repr_factory(factory): + return repr(factory) + def __repr__(self): + return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})" + + def copy(self): + return type(self)(self.default_factory, self) + + __copy__ = copy + + def __reduce__(self): + if self.default_factory is not None: + args = self.default_factory, + else: + args = () + return type(self), args, None, None, iter(self.items()) + +defaultdict.__module__ = 'collections' diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index ae018a394..abc82aea3 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -714,8 +714,6 @@ class TestCopy(unittest.TestCase): self.assertEqual(x.foo, y.foo) self.assertIsNot(x.foo, y.foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deepcopy_dict_subclass(self): class C(dict): def __init__(self, d=None): diff --git a/vm/Lib/python_builtins/__reducelib.py b/vm/Lib/python_builtins/__reducelib.py index 78098617f..0067cd0a8 100644 --- a/vm/Lib/python_builtins/__reducelib.py +++ b/vm/Lib/python_builtins/__reducelib.py @@ -63,7 +63,7 @@ def reduce_2(obj): state = getstate() listitems = iter(obj) if isinstance(obj, list) else None - dictitems = obj.iteritems() if isinstance(obj, dict) else None + dictitems = iter(obj.items()) if isinstance(obj, dict) else None newobj = copyreg.__newobj__ diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 79d4cd6da..f71a36543 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -449,9 +449,20 @@ impl PyBoundMethod { #[pymethod(magic)] fn repr(&self, vm: &VirtualMachine) -> PyResult { + let funcname = if let Some(qname) = + vm.get_attribute_opt(self.function.clone(), "__qualname__")? + { + Some(qname) + } else if let Some(name) = vm.get_attribute_opt(self.function.clone(), "__qualname__")? { + Some(name) + } else { + None + }; + let funcname: Option = funcname.and_then(|o| o.downcast().ok()); Ok(format!( - "", - vm.to_repr(&self.object)?.borrow_value() + "", + funcname.as_ref().map_or("?", |s| s.borrow_value()), + vm.to_repr(&self.object)?.borrow_value(), )) } From c84eb2a8014f39e156a35b344a211993cc276f61 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 7 Apr 2021 20:04:14 -0500 Subject: [PATCH 16/16] Copy test_defaultdict from CPython 3.8 --- Lib/test/test_defaultdict.py | 187 +++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 Lib/test/test_defaultdict.py diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py new file mode 100644 index 000000000..b9f1fb9f2 --- /dev/null +++ b/Lib/test/test_defaultdict.py @@ -0,0 +1,187 @@ +"""Unit tests for collections.defaultdict.""" + +import os +import copy +import pickle +import tempfile +import unittest + +from collections import defaultdict + +def foobar(): + return list + +class TestDefaultDict(unittest.TestCase): + + def test_basic(self): + d1 = defaultdict() + self.assertEqual(d1.default_factory, None) + d1.default_factory = list + d1[12].append(42) + self.assertEqual(d1, {12: [42]}) + d1[12].append(24) + self.assertEqual(d1, {12: [42, 24]}) + d1[13] + d1[14] + self.assertEqual(d1, {12: [42, 24], 13: [], 14: []}) + self.assertTrue(d1[12] is not d1[13] is not d1[14]) + d2 = defaultdict(list, foo=1, bar=2) + self.assertEqual(d2.default_factory, list) + self.assertEqual(d2, {"foo": 1, "bar": 2}) + self.assertEqual(d2["foo"], 1) + self.assertEqual(d2["bar"], 2) + self.assertEqual(d2[42], []) + self.assertIn("foo", d2) + self.assertIn("foo", d2.keys()) + self.assertIn("bar", d2) + self.assertIn("bar", d2.keys()) + self.assertIn(42, d2) + self.assertIn(42, d2.keys()) + self.assertNotIn(12, d2) + self.assertNotIn(12, d2.keys()) + d2.default_factory = None + self.assertEqual(d2.default_factory, None) + try: + d2[15] + except KeyError as err: + self.assertEqual(err.args, (15,)) + else: + self.fail("d2[15] didn't raise KeyError") + self.assertRaises(TypeError, defaultdict, 1) + + def test_missing(self): + d1 = defaultdict() + self.assertRaises(KeyError, d1.__missing__, 42) + d1.default_factory = list + self.assertEqual(d1.__missing__(42), []) + + def test_repr(self): + d1 = defaultdict() + self.assertEqual(d1.default_factory, None) + self.assertEqual(repr(d1), "defaultdict(None, {})") + self.assertEqual(eval(repr(d1)), d1) + d1[11] = 41 + self.assertEqual(repr(d1), "defaultdict(None, {11: 41})") + d2 = defaultdict(int) + self.assertEqual(d2.default_factory, int) + d2[12] = 42 + self.assertEqual(repr(d2), "defaultdict(, {12: 42})") + def foo(): return 43 + d3 = defaultdict(foo) + self.assertTrue(d3.default_factory is foo) + d3[13] + self.assertEqual(repr(d3), "defaultdict(%s, {13: 43})" % repr(foo)) + + def test_print(self): + d1 = defaultdict() + def foo(): return 42 + d2 = defaultdict(foo, {1: 2}) + # NOTE: We can't use tempfile.[Named]TemporaryFile since this + # code must exercise the tp_print C code, which only gets + # invoked for *real* files. + tfn = tempfile.mktemp() + try: + f = open(tfn, "w+") + try: + print(d1, file=f) + print(d2, file=f) + f.seek(0) + self.assertEqual(f.readline(), repr(d1) + "\n") + self.assertEqual(f.readline(), repr(d2) + "\n") + finally: + f.close() + finally: + os.remove(tfn) + + def test_copy(self): + d1 = defaultdict() + d2 = d1.copy() + self.assertEqual(type(d2), defaultdict) + self.assertEqual(d2.default_factory, None) + self.assertEqual(d2, {}) + d1.default_factory = list + d3 = d1.copy() + self.assertEqual(type(d3), defaultdict) + self.assertEqual(d3.default_factory, list) + self.assertEqual(d3, {}) + d1[42] + d4 = d1.copy() + self.assertEqual(type(d4), defaultdict) + self.assertEqual(d4.default_factory, list) + self.assertEqual(d4, {42: []}) + d4[12] + self.assertEqual(d4, {42: [], 12: []}) + + # Issue 6637: Copy fails for empty default dict + d = defaultdict() + d['a'] = 42 + e = d.copy() + self.assertEqual(e['a'], 42) + + def test_shallow_copy(self): + d1 = defaultdict(foobar, {1: 1}) + d2 = copy.copy(d1) + self.assertEqual(d2.default_factory, foobar) + self.assertEqual(d2, d1) + d1.default_factory = list + d2 = copy.copy(d1) + self.assertEqual(d2.default_factory, list) + self.assertEqual(d2, d1) + + def test_deep_copy(self): + d1 = defaultdict(foobar, {1: [1]}) + d2 = copy.deepcopy(d1) + self.assertEqual(d2.default_factory, foobar) + self.assertEqual(d2, d1) + self.assertTrue(d1[1] is not d2[1]) + d1.default_factory = list + d2 = copy.deepcopy(d1) + self.assertEqual(d2.default_factory, list) + self.assertEqual(d2, d1) + + def test_keyerror_without_factory(self): + d1 = defaultdict() + try: + d1[(1,)] + except KeyError as err: + self.assertEqual(err.args[0], (1,)) + else: + self.fail("expected KeyError") + + def test_recursive_repr(self): + # Issue2045: stack overflow when default_factory is a bound method + class sub(defaultdict): + def __init__(self): + self.default_factory = self._factory + def _factory(self): + return [] + d = sub() + self.assertRegex(repr(d), + r"sub\(, \{\}\)") + + # NOTE: printing a subclass of a builtin type does not call its + # tp_print slot. So this part is essentially the same test as above. + tfn = tempfile.mktemp() + try: + f = open(tfn, "w+") + try: + print(d, file=f) + finally: + f.close() + finally: + os.remove(tfn) + + def test_callable_arg(self): + self.assertRaises(TypeError, defaultdict, {}) + + def test_pickling(self): + d = defaultdict(int) + d[1] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(d, proto) + o = pickle.loads(s) + self.assertEqual(d, o) + +if __name__ == "__main__": + unittest.main()