Merge pull request #2252 from RustPython/coolreader18/misc-altair-fixes

Misc fixes found while trying to get Altair to work

Lib/_py_abc.py

@@ -33,7 +33,7 @@ class ABCMeta(type):
     _abc_invalidation_counter = 0
 
     def __new__(mcls, name, bases, namespace, **kwargs):
-        cls = type.__new__(mcls, name, bases, namespace, **kwargs)
+        cls = super().__new__(mcls, name, bases, namespace, **kwargs)
         # Compute set of abstract method names
         abstracts = {name
                      for name, value in namespace.items()

Lib/collections/_defaultdict.py

@@ -1,13 +1,12 @@
 class defaultdict(dict):
-    def __new__(cls, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
         if len(args) >= 1:
             default_factory = args[0]
             args = args[1:]
         else:
             default_factory = None
-        self = dict.__new__(cls, *args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.default_factory = default_factory
-        return self
 
     def __missing__(self, key):
         if self.default_factory:

Lib/importlib/_bootstrap_external.py

@@ -261,11 +261,16 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.7a2  3391 (update GET_AITER #31709)
 #     Python 3.7a4  3392 (PEP 552: Deterministic pycs #31650)
 #     Python 3.7b1  3393 (remove STORE_ANNOTATION opcode #32550)
-#     Python 3.7b5  3394 (restored docstring as the firts stmt in the body;
+#     Python 3.7b5  3394 (restored docstring as the first stmt in the body;
 #                         this might affected the first line number #32911)
 #     Python 3.8a1  3400 (move frame block handling to compiler #17611)
 #     Python 3.8a1  3401 (add END_ASYNC_FOR #33041)
 #     Python 3.8a1  3410 (PEP570 Python Positional-Only Parameters #36540)
+#     Python 3.8b2  3411 (Reverse evaluation order of key: value in dict
+#                         comprehensions #35224)
+#     Python 3.8b2  3412 (Swap the position of positional args and positional
+#                         only args in ast.arguments #37593)
+#     Python 3.8b4  3413 (Fix "break" and "continue" in "finally" #37830)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -963,8 +968,12 @@ class FileLoader:
 
     def get_data(self, path):
         """Return the data from path as raw bytes."""
-        with _io.FileIO(path, 'r') as file:
-            return file.read()
+        if isinstance(self, (SourceLoader, ExtensionFileLoader)):
+            with _io.open_code(str(path)) as file:
+                return file.read()
+        else:
+            with _io.FileIO(path, 'r') as file:
+                return file.read()
 
     # ResourceReader ABC API.
 
@@ -1359,6 +1368,19 @@ class PathFinder:
             return None
         return spec.loader
 
+    @classmethod
+    def find_distributions(cls, *args, **kwargs):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
+        """
+        from importlib.metadata import MetadataPathFinder
+        return MetadataPathFinder.find_distributions(*args, **kwargs)
+
 
 class FileFinder:
 

Lib/importlib/metadata.py

@@ -0,0 +1,566 @@
+import io
+import os
+import re
+import abc
+import csv
+import sys
+import email
+import pathlib
+import zipfile
+import operator
+import functools
+import itertools
+import posixpath
+import collections
+
+from configparser import ConfigParser
+from contextlib import suppress
+from importlib import import_module
+from importlib.abc import MetaPathFinder
+from itertools import starmap
+
+
+__all__ = [
+    'Distribution',
+    'DistributionFinder',
+    'PackageNotFoundError',
+    'distribution',
+    'distributions',
+    'entry_points',
+    'files',
+    'metadata',
+    'requires',
+    'version',
+    ]
+
+
+class PackageNotFoundError(ModuleNotFoundError):
+    """The package was not found."""
+
+
+class EntryPoint(
+        collections.namedtuple('EntryPointBase', 'name value group')):
+    """An entry point as defined by Python packaging conventions.
+
+    See `the packaging docs on entry points
+    <https://packaging.python.org/specifications/entry-points/>`_
+    for more information.
+    """
+
+    pattern = re.compile(
+        r'(?P<module>[\w.]+)\s*'
+        r'(:\s*(?P<attr>[\w.]+))?\s*'
+        r'(?P<extras>\[.*\])?\s*$'
+        )
+    """
+    A regular expression describing the syntax for an entry point,
+    which might look like:
+
+        - module
+        - package.module
+        - package.module:attribute
+        - package.module:object.attribute
+        - package.module:attr [extra1, extra2]
+
+    Other combinations are possible as well.
+
+    The expression is lenient about whitespace around the ':',
+    following the attr, and following any extras.
+    """
+
+    def load(self):
+        """Load the entry point from its definition. If only a module
+        is indicated by the value, return that module. Otherwise,
+        return the named object.
+        """
+        match = self.pattern.match(self.value)
+        module = import_module(match.group('module'))
+        attrs = filter(None, (match.group('attr') or '').split('.'))
+        return functools.reduce(getattr, attrs, module)
+
+    @property
+    def extras(self):
+        match = self.pattern.match(self.value)
+        return list(re.finditer(r'\w+', match.group('extras') or ''))
+
+    @classmethod
+    def _from_config(cls, config):
+        return [
+            cls(name, value, group)
+            for group in config.sections()
+            for name, value in config.items(group)
+            ]
+
+    @classmethod
+    def _from_text(cls, text):
+        config = ConfigParser(delimiters='=')
+        # case sensitive: https://stackoverflow.com/q/1611799/812183
+        config.optionxform = str
+        try:
+            config.read_string(text)
+        except AttributeError:  # pragma: nocover
+            # Python 2 has no read_string
+            config.readfp(io.StringIO(text))
+        return EntryPoint._from_config(config)
+
+    def __iter__(self):
+        """
+        Supply iter so one may construct dicts of EntryPoints easily.
+        """
+        return iter((self.name, self))
+
+    def __reduce__(self):
+        return (
+            self.__class__,
+            (self.name, self.value, self.group),
+            )
+
+
+class PackagePath(pathlib.PurePosixPath):
+    """A reference to a path in a package"""
+
+    def read_text(self, encoding='utf-8'):
+        with self.locate().open(encoding=encoding) as stream:
+            return stream.read()
+
+    def read_binary(self):
+        with self.locate().open('rb') as stream:
+            return stream.read()
+
+    def locate(self):
+        """Return a path-like object for this path"""
+        return self.dist.locate_file(self)
+
+
+class FileHash:
+    def __init__(self, spec):
+        self.mode, _, self.value = spec.partition('=')
+
+    def __repr__(self):
+        return '<FileHash mode: {} value: {}>'.format(self.mode, self.value)
+
+
+class Distribution:
+    """A Python distribution package."""
+
+    @abc.abstractmethod
+    def read_text(self, filename):
+        """Attempt to load metadata file given by the name.
+
+        :param filename: The name of the file in the distribution info.
+        :return: The text if found, otherwise None.
+        """
+
+    @abc.abstractmethod
+    def locate_file(self, path):
+        """
+        Given a path to a file in this distribution, return a path
+        to it.
+        """
+
+    @classmethod
+    def from_name(cls, name):
+        """Return the Distribution for the given package name.
+
+        :param name: The name of the distribution package to search for.
+        :return: The Distribution instance (or subclass thereof) for the named
+            package, if found.
+        :raises PackageNotFoundError: When the named package's distribution
+            metadata cannot be found.
+        """
+        for resolver in cls._discover_resolvers():
+            dists = resolver(DistributionFinder.Context(name=name))
+            dist = next(dists, None)
+            if dist is not None:
+                return dist
+        else:
+            raise PackageNotFoundError(name)
+
+    @classmethod
+    def discover(cls, **kwargs):
+        """Return an iterable of Distribution objects for all packages.
+
+        Pass a ``context`` or pass keyword arguments for constructing
+        a context.
+
+        :context: A ``DistributionFinder.Context`` object.
+        :return: Iterable of Distribution objects for all packages.
+        """
+        context = kwargs.pop('context', None)
+        if context and kwargs:
+            raise ValueError("cannot accept context and kwargs")
+        context = context or DistributionFinder.Context(**kwargs)
+        return itertools.chain.from_iterable(
+            resolver(context)
+            for resolver in cls._discover_resolvers()
+            )
+
+    @staticmethod
+    def at(path):
+        """Return a Distribution for the indicated metadata path
+
+        :param path: a string or path-like object
+        :return: a concrete Distribution instance for the path
+        """
+        return PathDistribution(pathlib.Path(path))
+
+    @staticmethod
+    def _discover_resolvers():
+        """Search the meta_path for resolvers."""
+        declared = (
+            getattr(finder, 'find_distributions', None)
+            for finder in sys.meta_path
+            )
+        return filter(None, declared)
+
+    @property
+    def metadata(self):
+        """Return the parsed metadata for this Distribution.
+
+        The returned object will have keys that name the various bits of
+        metadata.  See PEP 566 for details.
+        """
+        text = (
+            self.read_text('METADATA')
+            or self.read_text('PKG-INFO')
+            # This last clause is here to support old egg-info files.  Its
+            # effect is to just end up using the PathDistribution's self._path
+            # (which points to the egg-info file) attribute unchanged.
+            or self.read_text('')
+            )
+        return email.message_from_string(text)
+
+    @property
+    def version(self):
+        """Return the 'Version' metadata for the distribution package."""
+        return self.metadata['Version']
+
+    @property
+    def entry_points(self):
+        return EntryPoint._from_text(self.read_text('entry_points.txt'))
+
+    @property
+    def files(self):
+        """Files in this distribution.
+
+        :return: List of PackagePath for this distribution or None
+
+        Result is `None` if the metadata file that enumerates files
+        (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
+        missing.
+        Result may be empty if the metadata exists but is empty.
+        """
+        file_lines = self._read_files_distinfo() or self._read_files_egginfo()
+
+        def make_file(name, hash=None, size_str=None):
+            result = PackagePath(name)
+            result.hash = FileHash(hash) if hash else None
+            result.size = int(size_str) if size_str else None
+            result.dist = self
+            return result
+
+        return file_lines and list(starmap(make_file, csv.reader(file_lines)))
+
+    def _read_files_distinfo(self):
+        """
+        Read the lines of RECORD
+        """
+        text = self.read_text('RECORD')
+        return text and text.splitlines()
+
+    def _read_files_egginfo(self):
+        """
+        SOURCES.txt might contain literal commas, so wrap each line
+        in quotes.
+        """
+        text = self.read_text('SOURCES.txt')
+        return text and map('"{}"'.format, text.splitlines())
+
+    @property
+    def requires(self):
+        """Generated requirements specified for this Distribution"""
+        reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs()
+        return reqs and list(reqs)
+
+    def _read_dist_info_reqs(self):
+        return self.metadata.get_all('Requires-Dist')
+
+    def _read_egg_info_reqs(self):
+        source = self.read_text('requires.txt')
+        return source and self._deps_from_requires_text(source)
+
+    @classmethod
+    def _deps_from_requires_text(cls, source):
+        section_pairs = cls._read_sections(source.splitlines())
+        sections = {
+            section: list(map(operator.itemgetter('line'), results))
+            for section, results in
+            itertools.groupby(section_pairs, operator.itemgetter('section'))
+            }
+        return cls._convert_egg_info_reqs_to_simple_reqs(sections)
+
+    @staticmethod
+    def _read_sections(lines):
+        section = None
+        for line in filter(None, lines):
+            section_match = re.match(r'\[(.*)\]$', line)
+            if section_match:
+                section = section_match.group(1)
+                continue
+            yield locals()
+
+    @staticmethod
+    def _convert_egg_info_reqs_to_simple_reqs(sections):
+        """
+        Historically, setuptools would solicit and store 'extra'
+        requirements, including those with environment markers,
+        in separate sections. More modern tools expect each
+        dependency to be defined separately, with any relevant
+        extras and environment markers attached directly to that
+        requirement. This method converts the former to the
+        latter. See _test_deps_from_requires_text for an example.
+        """
+        def make_condition(name):
+            return name and 'extra == "{name}"'.format(name=name)
+
+        def parse_condition(section):
+            section = section or ''
+            extra, sep, markers = section.partition(':')
+            if extra and markers:
+                markers = '({markers})'.format(markers=markers)
+            conditions = list(filter(None, [markers, make_condition(extra)]))
+            return '; ' + ' and '.join(conditions) if conditions else ''
+
+        for section, deps in sections.items():
+            for dep in deps:
+                yield dep + parse_condition(section)
+
+
+class DistributionFinder(MetaPathFinder):
+    """
+    A MetaPathFinder capable of discovering installed distributions.
+    """
+
+    class Context:
+        """
+        Keyword arguments presented by the caller to
+        ``distributions()`` or ``Distribution.discover()``
+        to narrow the scope of a search for distributions
+        in all DistributionFinders.
+
+        Each DistributionFinder may expect any parameters
+        and should attempt to honor the canonical
+        parameters defined below when appropriate.
+        """
+
+        name = None
+        """
+        Specific name for which a distribution finder should match.
+        A name of ``None`` matches all distributions.
+        """
+
+        def __init__(self, **kwargs):
+            vars(self).update(kwargs)
+
+        @property
+        def path(self):
+            """
+            The path that a distribution finder should search.
+
+            Typically refers to Python package paths and defaults
+            to ``sys.path``.
+            """
+            return vars(self).get('path', sys.path)
+
+    @abc.abstractmethod
+    def find_distributions(self, context=Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching the ``context``,
+        a DistributionFinder.Context instance.
+        """
+
+
+class FastPath:
+    """
+    Micro-optimized class for searching a path for
+    children.
+    """
+
+    def __init__(self, root):
+        self.root = root
+        self.base = os.path.basename(root).lower()
+
+    def joinpath(self, child):
+        return pathlib.Path(self.root, child)
+
+    def children(self):
+        with suppress(Exception):
+            return os.listdir(self.root or '')
+        with suppress(Exception):
+            return self.zip_children()
+        return []
+
+    def zip_children(self):
+        zip_path = zipfile.Path(self.root)
+        names = zip_path.root.namelist()
+        self.joinpath = zip_path.joinpath
+
+        return (
+            posixpath.split(child)[0]
+            for child in names
+            )
+
+    def is_egg(self, search):
+        base = self.base
+        return (
+            base == search.versionless_egg_name
+            or base.startswith(search.prefix)
+            and base.endswith('.egg'))
+
+    def search(self, name):
+        for child in self.children():
+            n_low = child.lower()
+            if (n_low in name.exact_matches
+                    or n_low.startswith(name.prefix)
+                    and n_low.endswith(name.suffixes)
+                    # legacy case:
+                    or self.is_egg(name) and n_low == 'egg-info'):
+                yield self.joinpath(child)
+
+
+class Prepared:
+    """
+    A prepared search for metadata on a possibly-named package.
+    """
+    normalized = ''
+    prefix = ''
+    suffixes = '.dist-info', '.egg-info'
+    exact_matches = [''][:0]
+    versionless_egg_name = ''
+
+    def __init__(self, name):
+        self.name = name
+        if name is None:
+            return
+        self.normalized = name.lower().replace('-', '_')
+        self.prefix = self.normalized + '-'
+        self.exact_matches = [
+            self.normalized + suffix for suffix in self.suffixes]
+        self.versionless_egg_name = self.normalized + '.egg'
+
+
+class MetadataPathFinder(DistributionFinder):
+    @classmethod
+    def find_distributions(cls, context=DistributionFinder.Context()):
+        """
+        Find distributions.
+
+        Return an iterable of all Distribution instances capable of
+        loading the metadata for packages matching ``context.name``
+        (or all names if ``None`` indicated) along the paths in the list
+        of directories ``context.path``.
+        """
+        found = cls._search_paths(context.name, context.path)
+        return map(PathDistribution, found)
+
+    @classmethod
+    def _search_paths(cls, name, paths):
+        """Find metadata directories in paths heuristically."""
+        return itertools.chain.from_iterable(
+            path.search(Prepared(name))
+            for path in map(FastPath, paths)
+            )
+
+
+
+class PathDistribution(Distribution):
+    def __init__(self, path):
+        """Construct a distribution from a path to the metadata directory.
+
+        :param path: A pathlib.Path or similar object supporting
+                     .joinpath(), __div__, .parent, and .read_text().
+        """
+        self._path = path
+
+    def read_text(self, filename):
+        with suppress(FileNotFoundError, IsADirectoryError, KeyError,
+                      NotADirectoryError, PermissionError):
+            return self._path.joinpath(filename).read_text(encoding='utf-8')
+    read_text.__doc__ = Distribution.read_text.__doc__
+
+    def locate_file(self, path):
+        return self._path.parent / path
+
+
+def distribution(distribution_name):
+    """Get the ``Distribution`` instance for the named package.
+
+    :param distribution_name: The name of the distribution package as a string.
+    :return: A ``Distribution`` instance (or subclass thereof).
+    """
+    return Distribution.from_name(distribution_name)
+
+
+def distributions(**kwargs):
+    """Get all ``Distribution`` instances in the current environment.
+
+    :return: An iterable of ``Distribution`` instances.
+    """
+    return Distribution.discover(**kwargs)
+
+
+def metadata(distribution_name):
+    """Get the metadata for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: An email.Message containing the parsed metadata.
+    """
+    return Distribution.from_name(distribution_name).metadata
+
+
+def version(distribution_name):
+    """Get the version string for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: The version string for the package as defined in the package's
+        "Version" metadata key.
+    """
+    return distribution(distribution_name).version
+
+
+def entry_points():
+    """Return EntryPoint objects for all installed packages.
+
+    :return: EntryPoint objects for all installed packages.
+    """
+    eps = itertools.chain.from_iterable(
+        dist.entry_points for dist in distributions())
+    by_group = operator.attrgetter('group')
+    ordered = sorted(eps, key=by_group)
+    grouped = itertools.groupby(ordered, by_group)
+    return {
+        group: tuple(eps)
+        for group, eps in grouped
+        }
+
+
+def files(distribution_name):
+    """Return a list of files for the named package.
+
+    :param distribution_name: The name of the distribution package to query.
+    :return: List of files composing the distribution.
+    """
+    return distribution(distribution_name).files
+
+
+def requires(distribution_name):
+    """
+    Return a list of requirements for the named package.
+
+    :return: An iterator of requirements, suitable for
+    packaging.requirement.Requirement.
+    """
+    return distribution(distribution_name).requires

Lib/test/test_subclassinit.py

@@ -112,8 +112,6 @@ class Test(unittest.TestCase):
         self.assertEqual(A.d.name, "d")
         self.assertIs(A.d.owner, A)
 
-    #TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_set_name_metaclass(self):
         class Meta(type):
             def __new__(cls, name, bases, ns):

Lib/test/test_types.py

@@ -789,8 +789,6 @@ class MappingProxyTests(unittest.TestCase):
         self.assertEqual(set(view.values()), set(values))
         self.assertEqual(set(view.items()), set(items))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_copy(self):
         original = {'key1': 27, 'key2': 51, 'key3': 93}
         view = self.mappingproxy(original)

Lib/typing.py

@@ -1144,7 +1144,9 @@ class GenericMeta(TypingMeta, abc.ABCMeta):
 
     def __subclasscheck__(self, cls):
         if self.__origin__ is not None:
-            if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']:
+            # XXX RUSTPYTHON: added _py_abc; I think CPython was fine because abc called
+            # directly into the _abc builtin module, which wasn't in the frame stack
+            if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools', '_py_abc']:
                 raise TypeError("Parameterized generics cannot be used with class "
                                 "or instance checks")
             return False

bytecode/src/bytecode.rs

@@ -435,6 +435,8 @@ impl CodeObject {
             .iter()
             .map(String::as_str)
             .chain(self.kwonlyarg_names.iter().map(String::as_str))
+            .chain(self.varargs_name.as_deref())
+            .chain(self.varkeywords_name.as_deref())
             .chain(
                 self.instructions
                     .iter()

compiler/src/compile.rs

@@ -989,7 +989,13 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
-        self.store_docstring(doc_str);
+
+        self.emit(Instruction::Duplicate);
+        self.load_docstring(doc_str);
+        self.emit(Instruction::Rotate { amount: 2 });
+        self.emit(Instruction::StoreAttr {
+            name: "__doc__".to_owned(),
+        });
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1109,6 +1115,11 @@ impl<O: OutputStream> Compiler<O> {
             name: "__qualname__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
+        self.load_docstring(doc_str);
+        self.emit(Instruction::StoreName {
+            name: "__doc__".to_owned(),
+            scope: bytecode::NameScope::Free,
+        });
         // setup annotations
         if self.find_ann(body) {
             self.emit(Instruction::SetupAnnotation);
@@ -1175,7 +1186,6 @@ impl<O: OutputStream> Compiler<O> {
             });
         }
 
-        self.store_docstring(doc_str);
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1184,10 +1194,9 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn store_docstring(&mut self, doc_str: Option<String>) {
+    fn load_docstring(&mut self, doc_str: Option<String>) {
         // TODO: __doc__ must be default None and no bytecodes unless it is Some
         // Duplicate top of stack (the function or class object)
-        self.emit(Instruction::Duplicate);
 
         // Doc string value:
         self.emit(Instruction::LoadConst {
@@ -1196,11 +1205,6 @@ impl<O: OutputStream> Compiler<O> {
                 None => bytecode::Constant::None, // set docstring None if not declared
             },
         });
-
-        self.emit(Instruction::Rotate { amount: 2 });
-        self.emit(Instruction::StoreAttr {
-            name: "__doc__".to_owned(),
-        });
     }
 
     fn compile_while(

compiler/src/symboltable.rs

@@ -235,15 +235,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
                     if scope_depth > 0 {
                         // check if the name is already defined in any outer scope
                         // therefore
-                        if scope_depth < 2
-                            || !self
-                                .tables
-                                .iter()
-                                .skip(1) // omit the global scope as it is not non-local
-                                .rev() // revert the order for better performance
-                                .any(|t| t.0.contains_key(&symbol.name))
-                        // true when any of symbol tables contains the name -> then negate
-                        {
+                        if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
                                 location: Default::default(),
@@ -274,27 +266,27 @@ impl<'a> SymbolTableAnalyzer<'a> {
         Ok(())
     }
 
+    fn found_in_outer_scope(&self, symbol: &Symbol) -> bool {
+        // Interesting stuff about the __class__ variable:
+        // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
+        symbol.name == "__class__"
+            || self.tables.iter().skip(1).any(|(symbols, typ)| {
+                *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
+            })
+    }
+
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
+        } else if self.found_in_outer_scope(symbol) {
+            // Symbol is in some outer scope.
+            symbol.is_free = true;
+        } else if self.tables.is_empty() {
+            // Don't make assumptions when we don't know.
+            symbol.scope = SymbolScope::Unknown;
         } else {
-            // Interesting stuff about the __class__ variable:
-            // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-            let found_in_outer_scope = symbol.name == "__class__"
-                || self.tables.iter().skip(1).any(|(symbols, typ)| {
-                    *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
-                });
-
-            if found_in_outer_scope {
-                // Symbol is in some outer scope.
-                symbol.is_free = true;
-            } else if self.tables.is_empty() {
-                // Don't make assumptions when we don't know.
-                symbol.scope = SymbolScope::Unknown;
-            } else {
-                // If there are scopes above we can assume global.
-                symbol.scope = SymbolScope::Global;
-            }
+            // If there are scopes above we can assume global.
+            symbol.scope = SymbolScope::Global;
         }
     }
 
@@ -531,6 +523,7 @@ impl SymbolTableBuilder {
                 self.enter_scope(name, SymbolTableType::Class, statement.location.row());
                 self.register_name("__module__", SymbolUsage::Assigned)?;
                 self.register_name("__qualname__", SymbolUsage::Assigned)?;
+                self.register_name("__doc__", SymbolUsage::Assigned)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
                 self.scan_expressions(bases, &ExpressionContext::Load)?;

vm/src/obj/objmappingproxy.rs

@@ -126,6 +126,15 @@ impl PyMappingProxy {
         };
         vm.call_method(&obj, "values", vec![])
     }
+    #[pymethod]
+    pub fn copy(&self, vm: &VirtualMachine) -> PyResult {
+        match &self.mapping {
+            MappingProxyInner::Dict(d) => vm.call_method(d, "copy", vec![]),
+            MappingProxyInner::Class(c) => {
+                Ok(PyDict::from_attributes(c.attributes.read().clone(), vm)?.into_pyobject(vm))
+            }
+        }
+    }
 }
 
 pub fn init(context: &PyContext) {

vm/src/obj/objproperty.rs

@@ -4,6 +4,7 @@
 use crate::common::lock::PyRwLock;
 
 use super::objtype::PyTypeRef;
+use crate::function::PyFuncArgs;
 use crate::pyobject::{
     PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol,
 };
@@ -45,9 +46,9 @@ use crate::vm::VirtualMachine;
 #[pyclass(module = false, name = "property")]
 #[derive(Debug)]
 pub struct PyProperty {
-    getter: Option<PyObjectRef>,
-    setter: Option<PyObjectRef>,
-    deleter: Option<PyObjectRef>,
+    getter: PyRwLock<Option<PyObjectRef>>,
+    setter: PyRwLock<Option<PyObjectRef>>,
+    deleter: PyRwLock<Option<PyObjectRef>>,
     doc: PyRwLock<Option<PyObjectRef>>,
 }
 
@@ -81,7 +82,7 @@ impl SlotDescriptor for PyProperty {
         let (zelf, obj) = Self::_unwrap(zelf, obj, vm)?;
         if vm.is_none(&obj) {
             Ok(zelf.into_object())
-        } else if let Some(getter) = zelf.getter.as_ref() {
+        } else if let Some(getter) = zelf.getter.read().as_ref() {
             vm.invoke(&getter, obj)
         } else {
             Err(vm.new_attribute_error("unreadable attribute".to_string()))
@@ -92,21 +93,29 @@ impl SlotDescriptor for PyProperty {
 #[pyimpl(with(SlotDescriptor), flags(BASETYPE))]
 impl PyProperty {
     #[pyslot]
-    fn tp_new(cls: PyTypeRef, args: PropertyArgs, vm: &VirtualMachine) -> PyResult<PyPropertyRef> {
+    fn tp_new(cls: PyTypeRef, _args: PyFuncArgs, vm: &VirtualMachine) -> PyResult<PyPropertyRef> {
         PyProperty {
-            getter: args.fget,
-            setter: args.fset,
-            deleter: args.fdel,
-            doc: PyRwLock::new(args.doc),
+            getter: PyRwLock::new(None),
+            setter: PyRwLock::new(None),
+            deleter: PyRwLock::new(None),
+            doc: PyRwLock::new(None),
         }
         .into_ref_with_type(vm, cls)
     }
 
+    #[pymethod(magic)]
+    fn init(&self, args: PropertyArgs) {
+        *self.getter.write() = args.fget;
+        *self.setter.write() = args.fset;
+        *self.deleter.write() = args.fdel;
+        *self.doc.write() = args.doc;
+    }
+
     // Descriptor methods
 
     #[pymethod(name = "__set__")]
     fn set(&self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult {
-        if let Some(ref setter) = self.setter.as_ref() {
+        if let Some(ref setter) = self.setter.read().as_ref() {
             vm.invoke(setter, vec![obj, value])
         } else {
             Err(vm.new_attribute_error("can't set attribute".to_owned()))
@@ -115,7 +124,7 @@ impl PyProperty {
 
     #[pymethod(name = "__delete__")]
     fn delete(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
-        if let Some(ref deleter) = self.deleter.as_ref() {
+        if let Some(ref deleter) = self.deleter.read().as_ref() {
             vm.invoke(deleter, obj)
         } else {
             Err(vm.new_attribute_error("can't delete attribute".to_owned()))
@@ -126,25 +135,24 @@ impl PyProperty {
 
     #[pyproperty]
     fn fget(&self) -> Option<PyObjectRef> {
-        self.getter.clone()
+        self.getter.read().clone()
     }
 
     #[pyproperty]
     fn fset(&self) -> Option<PyObjectRef> {
-        self.setter.clone()
+        self.setter.read().clone()
     }
 
     #[pyproperty]
     fn fdel(&self) -> Option<PyObjectRef> {
-        self.deleter.clone()
+        self.deleter.read().clone()
     }
 
     fn doc_getter(&self) -> Option<PyObjectRef> {
         self.doc.read().clone()
     }
-
-    fn doc_setter(&self, value: PyObjectRef, vm: &VirtualMachine) {
-        *self.doc.write() = py_none_to_option(vm, &value);
+    fn doc_setter(&self, value: Option<PyObjectRef>) {
+        *self.doc.write() = value;
     }
 
     // Python builder functions
@@ -156,9 +164,9 @@ impl PyProperty {
         vm: &VirtualMachine,
     ) -> PyResult<PyPropertyRef> {
         PyProperty {
-            getter: getter.or_else(|| zelf.getter.clone()),
-            setter: zelf.setter.clone(),
-            deleter: zelf.deleter.clone(),
+            getter: PyRwLock::new(getter.or_else(|| zelf.fget())),
+            setter: PyRwLock::new(zelf.fset()),
+            deleter: PyRwLock::new(zelf.fdel()),
             doc: PyRwLock::new(None),
         }
         .into_ref_with_type(vm, TypeProtocol::class(&zelf))
@@ -171,9 +179,9 @@ impl PyProperty {
         vm: &VirtualMachine,
     ) -> PyResult<PyPropertyRef> {
         PyProperty {
-            getter: zelf.getter.clone(),
-            setter: setter.or_else(|| zelf.setter.clone()),
-            deleter: zelf.deleter.clone(),
+            getter: PyRwLock::new(zelf.fget()),
+            setter: PyRwLock::new(setter.or_else(|| zelf.fset())),
+            deleter: PyRwLock::new(zelf.fdel()),
             doc: PyRwLock::new(None),
         }
         .into_ref_with_type(vm, TypeProtocol::class(&zelf))
@@ -186,24 +194,15 @@ impl PyProperty {
         vm: &VirtualMachine,
     ) -> PyResult<PyPropertyRef> {
         PyProperty {
-            getter: zelf.getter.clone(),
-            setter: zelf.setter.clone(),
-            deleter: deleter.or_else(|| zelf.deleter.clone()),
+            getter: PyRwLock::new(zelf.fget()),
+            setter: PyRwLock::new(zelf.fset()),
+            deleter: PyRwLock::new(deleter.or_else(|| zelf.fdel())),
             doc: PyRwLock::new(None),
         }
         .into_ref_with_type(vm, TypeProtocol::class(&zelf))
     }
 }
 
-/// Take a python object and turn it into an option object, where python None maps to rust None.
-fn py_none_to_option(vm: &VirtualMachine, value: &PyObjectRef) -> Option<PyObjectRef> {
-    if vm.is_none(value) {
-        None
-    } else {
-        Some(value.clone())
-    }
-}
-
 pub(crate) fn init(context: &PyContext) {
     PyProperty::extend_class(context, &context.types.property_type);
 

vm/src/obj/objsuper.rs

@@ -17,8 +17,6 @@ use crate::scope::NameProtocol;
 use crate::slots::{SlotDescriptor, SlotGetattro};
 use crate::vm::VirtualMachine;
 
-use itertools::Itertools;
-
 pub type PySuperRef = PyRef<PySuper>;
 
 #[pyclass(module = false, name = "super")]
@@ -114,10 +112,12 @@ impl SlotGetattro for PySuper {
             None => return vm.generic_getattribute(zelf.into_object(), name),
         };
         // skip the classes in obj_type.mro up to and including zelf.typ
-        let mut it = obj_type.iter_mro().peekable();
-        for _ in it.peeking_take_while(|cls| !cls.is(&zelf.typ)) {}
-        for cls in it.skip(1) {
-            if let Some(descr) = cls.get_attr(name.borrow_value()) {
+        let it = obj_type
+            .iter_mro()
+            .skip_while(|cls| !cls.is(&zelf.typ))
+            .skip(1);
+        for cls in it {
+            if let Some(descr) = cls.get_direct_attr(name.borrow_value()) {
                 return vm
                     .call_get_descriptor_specific(
                         descr.clone(),

vm/src/obj/objtype.rs

@@ -673,13 +673,14 @@ impl PyType {
     pub fn get_attr(&self, attr_name: &str) -> Option<PyObjectRef> {
         flame_guard!(format!("class_get_attr({:?})", attr_name));
 
-        self.attributes
-            .read()
-            .get(attr_name)
-            .cloned()
+        self.get_direct_attr(attr_name)
             .or_else(|| self.get_super_attr(attr_name))
     }
 
+    pub fn get_direct_attr(&self, attr_name: &str) -> Option<PyObjectRef> {
+        self.attributes.read().get(attr_name).cloned()
+    }
+
     pub fn get_super_attr(&self, attr_name: &str) -> Option<PyObjectRef> {
         self.mro
             .iter()

vm/src/stdlib/io.rs

@@ -1451,7 +1451,7 @@ mod fileio {
     #[derive(FromArgs)]
     struct FileIOArgs {
         #[pyarg(positional)]
-        name: Either<PyStrRef, i64>,
+        name: PyObjectRef,
         #[pyarg(any, default)]
         mode: Option<PyStrRef>,
         #[pyarg(any, default = "true")]
@@ -1477,44 +1477,40 @@ mod fileio {
                 .mode
                 .map(|mode| mode.borrow_value().to_owned())
                 .unwrap_or_else(|| "r".to_owned());
-            let (name, file_no) = match args.name {
-                Either::A(name) => {
-                    if !args.closefd {
-                        return Err(vm.new_value_error(
-                            "Cannot use closefd=False with file name".to_owned(),
-                        ));
-                    }
-                    let mode = compute_c_flag(&mode);
-                    let fd = if let Some(opener) = args.opener {
-                        let fd = vm.invoke(
-                            &opener,
-                            vec![name.clone().into_object(), vm.ctx.new_int(mode)],
-                        )?;
-                        if !vm.isinstance(&fd, &vm.ctx.types.int_type)? {
-                            return Err(
-                                vm.new_type_error("expected integer from opener".to_owned())
-                            );
+            let name = args.name.clone();
+            let fd = if let Some(opener) = args.opener {
+                let mode = compute_c_flag(&mode);
+                let fd = vm.invoke(&opener, vec![name.clone(), vm.ctx.new_int(mode)])?;
+                if !vm.isinstance(&fd, &vm.ctx.types.int_type)? {
+                    return Err(vm.new_type_error("expected integer from opener".to_owned()));
+                }
+                let fd = i64::try_from_object(vm, fd)?;
+                if fd < 0 {
+                    return Err(vm.new_os_error("Negative file descriptor".to_owned()));
+                }
+                fd
+            } else {
+                match Either::<i64, os::PyPathLike>::try_from_object(vm, args.name)? {
+                    Either::A(fno) => fno,
+                    Either::B(path) => {
+                        if !args.closefd {
+                            return Err(vm.new_value_error(
+                                "Cannot use closefd=False with file name".to_owned(),
+                            ));
                         }
-                        let fd = i64::try_from_object(vm, fd)?;
-                        if fd < 0 {
-                            return Err(vm.new_os_error("Negative file descriptor".to_owned()));
-                        }
-                        fd
-                    } else {
+                        let mode = compute_c_flag(&mode);
                         os::open(
-                            os::PyPathLike::new_str(name.borrow_value().to_owned()),
+                            path,
                             mode as _,
                             OptionalArg::Missing,
                             OptionalArg::Missing,
                             vm,
                         )?
-                    };
-                    (name.into_object(), fd)
+                    }
                 }
-                Either::B(fno) => (vm.ctx.new_int(fno), fno),
             };
 
-            zelf.fd.store(file_no);
+            zelf.fd.store(fd);
             zelf.closefd.store(args.closefd);
             vm.set_attr(zelf.as_object(), "name", name)?;
             vm.set_attr(zelf.as_object(), "mode", vm.ctx.new_str(mode))?;

vm/src/stdlib/marshal.rs

@@ -5,7 +5,7 @@ mod decl {
     use crate::bytecode;
     use crate::obj::objbytes::{PyBytes, PyBytesRef};
     use crate::obj::objcode::{PyCode, PyCodeRef};
-    use crate::pyobject::PyResult;
+    use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject};
     use crate::vm::VirtualMachine;
 
     #[pyfunction]
@@ -13,10 +13,23 @@ mod decl {
         PyBytes::from(co.code.to_bytes())
     }
 
+    #[pyfunction]
+    fn dump(co: PyCodeRef, f: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        vm.call_method(&f, "write", vec![dumps(co).into_pyobject(vm)])?;
+        Ok(())
+    }
+
     #[pyfunction]
     fn loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<PyCode> {
         let code = bytecode::CodeObject::from_bytes(&code_bytes)
             .map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?;
         Ok(PyCode { code })
     }
+
+    #[pyfunction]
+    fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyCode> {
+        let read_res = vm.call_method(&f, "read", vec![])?;
+        let bytes = PyBytesRef::try_from_object(vm, read_res)?;
+        loads(bytes, vm)
+    }
 }