diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 07044f2f7..d9c6ab8e0 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -33,7 +33,7 @@ class ABCMeta(type): _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, **kwargs): - cls = type.__new__(mcls, name, bases, namespace, **kwargs) + cls = super().__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() diff --git a/Lib/collections/_defaultdict.py b/Lib/collections/_defaultdict.py index 125a74c13..42635f0d2 100644 --- a/Lib/collections/_defaultdict.py +++ b/Lib/collections/_defaultdict.py @@ -1,13 +1,12 @@ class defaultdict(dict): - def __new__(cls, *args, **kwargs): + def __init__(self, *args, **kwargs): if len(args) >= 1: default_factory = args[0] args = args[1:] else: default_factory = None - self = dict.__new__(cls, *args, **kwargs) + super().__init__(*args, **kwargs) self.default_factory = default_factory - return self def __missing__(self, key): if self.default_factory: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a4e6a7176..2981478d3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -261,11 +261,16 @@ _code_type = type(_write_atomic.__code__) # Python 3.7a2 3391 (update GET_AITER #31709) # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the firts stmt in the body; +# Python 3.7b5 3394 (restored docstring as the first stmt in the body; # this might affected the first line number #32911) # Python 3.8a1 3400 (move frame block handling to compiler #17611) # Python 3.8a1 3401 (add END_ASYNC_FOR #33041) # Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) +# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict +# comprehensions #35224) +# Python 3.8b2 3412 (Swap the position of positional args and positional +# only args in ast.arguments #37593) +# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -963,8 +968,12 @@ class FileLoader: def get_data(self, path): """Return the data from path as raw bytes.""" - with _io.FileIO(path, 'r') as file: - return file.read() + if isinstance(self, (SourceLoader, ExtensionFileLoader)): + with _io.open_code(str(path)) as file: + return file.read() + else: + with _io.FileIO(path, 'r') as file: + return file.read() # ResourceReader ABC API. @@ -1359,6 +1368,19 @@ class PathFinder: return None return spec.loader + @classmethod + def find_distributions(cls, *args, **kwargs): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + from importlib.metadata import MetadataPathFinder + return MetadataPathFinder.find_distributions(*args, **kwargs) + class FileFinder: diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py new file mode 100644 index 000000000..831f59327 --- /dev/null +++ b/Lib/importlib/metadata.py @@ -0,0 +1,566 @@ +import io +import os +import re +import abc +import csv +import sys +import email +import pathlib +import zipfile +import operator +import functools +import itertools +import posixpath +import collections + +from configparser import ConfigParser +from contextlib import suppress +from importlib import import_module +from importlib.abc import MetaPathFinder +from itertools import starmap + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'requires', + 'version', + ] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + +class EntryPoint( + collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + `_ + for more information. + """ + + pattern = re.compile( + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+))?\s*' + r'(?P\[.*\])?\s*$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def extras(self): + match = self.pattern.match(self.value) + return list(re.finditer(r'\w+', match.group('extras') or '')) + + @classmethod + def _from_config(cls, config): + return [ + cls(name, value, group) + for group in config.sections() + for name, value in config.items(group) + ] + + @classmethod + def _from_text(cls, text): + config = ConfigParser(delimiters='=') + # case sensitive: https://stackoverflow.com/q/1611799/812183 + config.optionxform = str + try: + config.read_string(text) + except AttributeError: # pragma: nocover + # Python 2 has no read_string + config.readfp(io.StringIO(text)) + return EntryPoint._from_config(config) + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints easily. + """ + return iter((self.name, self)) + + def __reduce__(self): + return ( + self.__class__, + (self.name, self.value, self.group), + ) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return ''.format(self.mode, self.value) + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(DistributionFinder.Context(name=name)) + dist = next(dists, None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) + for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) + for finder in sys.meta_path + ) + return filter(None, declared) + + @property + def metadata(self): + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + return email.message_from_string(text) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoint._from_text(self.read_text('entry_points.txt')) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is + missing. + Result may be empty if the metadata exists but is empty. + """ + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and list(starmap(make_file, csv.reader(file_lines))) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return source and self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + section_pairs = cls._read_sections(source.splitlines()) + sections = { + section: list(map(operator.itemgetter('line'), results)) + for section, results in + itertools.groupby(section_pairs, operator.itemgetter('section')) + } + return cls._convert_egg_info_reqs_to_simple_reqs(sections) + + @staticmethod + def _read_sections(lines): + section = None + for line in filter(None, lines): + section_match = re.match(r'\[(.*)\]$', line) + if section_match: + section = section_match.group(1) + continue + yield locals() + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + def make_condition(name): + return name and 'extra == "{name}"'.format(name=name) + + def parse_condition(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = '({markers})'.format(markers=markers) + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + for section, deps in sections.items(): + for dep in deps: + yield dep + parse_condition(section) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The path that a distribution finder should search. + + Typically refers to Python package paths and defaults + to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + """ + + def __init__(self, root): + self.root = root + self.base = os.path.basename(root).lower() + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipfile.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return ( + posixpath.split(child)[0] + for child in names + ) + + def is_egg(self, search): + base = self.base + return ( + base == search.versionless_egg_name + or base.startswith(search.prefix) + and base.endswith('.egg')) + + def search(self, name): + for child in self.children(): + n_low = child.lower() + if (n_low in name.exact_matches + or n_low.startswith(name.prefix) + and n_low.endswith(name.suffixes) + # legacy case: + or self.is_egg(name) and n_low == 'egg-info'): + yield self.joinpath(child) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + normalized = '' + prefix = '' + suffixes = '.dist-info', '.egg-info' + exact_matches = [''][:0] + versionless_egg_name = '' + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = name.lower().replace('-', '_') + self.prefix = self.normalized + '-' + self.exact_matches = [ + self.normalized + suffix for suffix in self.suffixes] + self.versionless_egg_name = self.normalized + '.egg' + + +class MetadataPathFinder(DistributionFinder): + @classmethod + def find_distributions(cls, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = cls._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + return itertools.chain.from_iterable( + path.search(Prepared(name)) + for path in map(FastPath, paths) + ) + + + +class PathDistribution(Distribution): + def __init__(self, path): + """Construct a distribution from a path to the metadata directory. + + :param path: A pathlib.Path or similar object supporting + .joinpath(), __div__, .parent, and .read_text(). + """ + self._path = path + + def read_text(self, filename): + with suppress(FileNotFoundError, IsADirectoryError, KeyError, + NotADirectoryError, PermissionError): + return self._path.joinpath(filename).read_text(encoding='utf-8') + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name): + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: An email.Message containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +def entry_points(): + """Return EntryPoint objects for all installed packages. + + :return: EntryPoint objects for all installed packages. + """ + eps = itertools.chain.from_iterable( + dist.entry_points for dist in distributions()) + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return { + group: tuple(eps) + for group, eps in grouped + } + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py index 9579fde93..623790e4e 100644 --- a/Lib/test/test_subclassinit.py +++ b/Lib/test/test_subclassinit.py @@ -112,8 +112,6 @@ class Test(unittest.TestCase): self.assertEqual(A.d.name, "d") self.assertIs(A.d.owner, A) - #TODO: RUSTPYTHON - @unittest.expectedFailure def test_set_name_metaclass(self): class Meta(type): def __new__(cls, name, bases, ns): diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 4a4e6d919..779ee6970 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -789,8 +789,6 @@ class MappingProxyTests(unittest.TestCase): self.assertEqual(set(view.values()), set(values)) self.assertEqual(set(view.items()), set(items)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy(self): original = {'key1': 27, 'key2': 51, 'key3': 93} view = self.mappingproxy(original) diff --git a/Lib/typing.py b/Lib/typing.py index 807cc6125..d34a04f47 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1144,7 +1144,9 @@ class GenericMeta(TypingMeta, abc.ABCMeta): def __subclasscheck__(self, cls): if self.__origin__ is not None: - if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + # XXX RUSTPYTHON: added _py_abc; I think CPython was fine because abc called + # directly into the _abc builtin module, which wasn't in the frame stack + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools', '_py_abc']: raise TypeError("Parameterized generics cannot be used with class " "or instance checks") return False diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 7223d9efc..6e9ef8613 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -435,6 +435,8 @@ impl CodeObject { .iter() .map(String::as_str) .chain(self.kwonlyarg_names.iter().map(String::as_str)) + .chain(self.varargs_name.as_deref()) + .chain(self.varkeywords_name.as_deref()) .chain( self.instructions .iter() diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 2c04ecc9f..f92eadf30 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -989,7 +989,13 @@ impl Compiler { // Turn code object into function object: self.emit(Instruction::MakeFunction); - self.store_docstring(doc_str); + + self.emit(Instruction::Duplicate); + self.load_docstring(doc_str); + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_owned(), + }); self.apply_decorators(decorator_list); self.store_name(name); @@ -1109,6 +1115,11 @@ impl Compiler { name: "__qualname__".to_owned(), scope: bytecode::NameScope::Free, }); + self.load_docstring(doc_str); + self.emit(Instruction::StoreName { + name: "__doc__".to_owned(), + scope: bytecode::NameScope::Free, + }); // setup annotations if self.find_ann(body) { self.emit(Instruction::SetupAnnotation); @@ -1175,7 +1186,6 @@ impl Compiler { }); } - self.store_docstring(doc_str); self.apply_decorators(decorator_list); self.store_name(name); @@ -1184,10 +1194,9 @@ impl Compiler { Ok(()) } - fn store_docstring(&mut self, doc_str: Option) { + fn load_docstring(&mut self, doc_str: Option) { // TODO: __doc__ must be default None and no bytecodes unless it is Some // Duplicate top of stack (the function or class object) - self.emit(Instruction::Duplicate); // Doc string value: self.emit(Instruction::LoadConst { @@ -1196,11 +1205,6 @@ impl Compiler { None => bytecode::Constant::None, // set docstring None if not declared }, }); - - self.emit(Instruction::Rotate { amount: 2 }); - self.emit(Instruction::StoreAttr { - name: "__doc__".to_owned(), - }); } fn compile_while( diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs index 4d759f9a0..b9bcb852c 100644 --- a/compiler/src/symboltable.rs +++ b/compiler/src/symboltable.rs @@ -235,15 +235,7 @@ impl<'a> SymbolTableAnalyzer<'a> { if scope_depth > 0 { // check if the name is already defined in any outer scope // therefore - if scope_depth < 2 - || !self - .tables - .iter() - .skip(1) // omit the global scope as it is not non-local - .rev() // revert the order for better performance - .any(|t| t.0.contains_key(&symbol.name)) - // true when any of symbol tables contains the name -> then negate - { + if scope_depth < 2 || !self.found_in_outer_scope(symbol) { return Err(SymbolTableError { error: format!("no binding for nonlocal '{}' found", symbol.name), location: Default::default(), @@ -274,27 +266,27 @@ impl<'a> SymbolTableAnalyzer<'a> { Ok(()) } + fn found_in_outer_scope(&self, symbol: &Symbol) -> bool { + // Interesting stuff about the __class__ variable: + // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object + symbol.name == "__class__" + || self.tables.iter().skip(1).any(|(symbols, typ)| { + *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name) + }) + } + fn analyze_unknown_symbol(&self, symbol: &mut Symbol) { if symbol.is_assigned || symbol.is_parameter { symbol.scope = SymbolScope::Local; + } else if self.found_in_outer_scope(symbol) { + // Symbol is in some outer scope. + symbol.is_free = true; + } else if self.tables.is_empty() { + // Don't make assumptions when we don't know. + symbol.scope = SymbolScope::Unknown; } else { - // Interesting stuff about the __class__ variable: - // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object - let found_in_outer_scope = symbol.name == "__class__" - || self.tables.iter().skip(1).any(|(symbols, typ)| { - *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name) - }); - - if found_in_outer_scope { - // Symbol is in some outer scope. - symbol.is_free = true; - } else if self.tables.is_empty() { - // Don't make assumptions when we don't know. - symbol.scope = SymbolScope::Unknown; - } else { - // If there are scopes above we can assume global. - symbol.scope = SymbolScope::Global; - } + // If there are scopes above we can assume global. + symbol.scope = SymbolScope::Global; } } @@ -531,6 +523,7 @@ impl SymbolTableBuilder { self.enter_scope(name, SymbolTableType::Class, statement.location.row()); self.register_name("__module__", SymbolUsage::Assigned)?; self.register_name("__qualname__", SymbolUsage::Assigned)?; + self.register_name("__doc__", SymbolUsage::Assigned)?; self.scan_statements(body)?; self.leave_scope(); self.scan_expressions(bases, &ExpressionContext::Load)?; diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs index 2cdac4e87..ae85f5265 100644 --- a/vm/src/obj/objmappingproxy.rs +++ b/vm/src/obj/objmappingproxy.rs @@ -126,6 +126,15 @@ impl PyMappingProxy { }; vm.call_method(&obj, "values", vec![]) } + #[pymethod] + pub fn copy(&self, vm: &VirtualMachine) -> PyResult { + match &self.mapping { + MappingProxyInner::Dict(d) => vm.call_method(d, "copy", vec![]), + MappingProxyInner::Class(c) => { + Ok(PyDict::from_attributes(c.attributes.read().clone(), vm)?.into_pyobject(vm)) + } + } + } } pub fn init(context: &PyContext) { diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index 59894c7bc..4283ce0b9 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -4,6 +4,7 @@ use crate::common::lock::PyRwLock; use super::objtype::PyTypeRef; +use crate::function::PyFuncArgs; use crate::pyobject::{ PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; @@ -45,9 +46,9 @@ use crate::vm::VirtualMachine; #[pyclass(module = false, name = "property")] #[derive(Debug)] pub struct PyProperty { - getter: Option, - setter: Option, - deleter: Option, + getter: PyRwLock>, + setter: PyRwLock>, + deleter: PyRwLock>, doc: PyRwLock>, } @@ -81,7 +82,7 @@ impl SlotDescriptor for PyProperty { let (zelf, obj) = Self::_unwrap(zelf, obj, vm)?; if vm.is_none(&obj) { Ok(zelf.into_object()) - } else if let Some(getter) = zelf.getter.as_ref() { + } else if let Some(getter) = zelf.getter.read().as_ref() { vm.invoke(&getter, obj) } else { Err(vm.new_attribute_error("unreadable attribute".to_string())) @@ -92,21 +93,29 @@ impl SlotDescriptor for PyProperty { #[pyimpl(with(SlotDescriptor), flags(BASETYPE))] impl PyProperty { #[pyslot] - fn tp_new(cls: PyTypeRef, args: PropertyArgs, vm: &VirtualMachine) -> PyResult { + fn tp_new(cls: PyTypeRef, _args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { PyProperty { - getter: args.fget, - setter: args.fset, - deleter: args.fdel, - doc: PyRwLock::new(args.doc), + getter: PyRwLock::new(None), + setter: PyRwLock::new(None), + deleter: PyRwLock::new(None), + doc: PyRwLock::new(None), } .into_ref_with_type(vm, cls) } + #[pymethod(magic)] + fn init(&self, args: PropertyArgs) { + *self.getter.write() = args.fget; + *self.setter.write() = args.fset; + *self.deleter.write() = args.fdel; + *self.doc.write() = args.doc; + } + // Descriptor methods #[pymethod(name = "__set__")] fn set(&self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(ref setter) = self.setter.as_ref() { + if let Some(ref setter) = self.setter.read().as_ref() { vm.invoke(setter, vec![obj, value]) } else { Err(vm.new_attribute_error("can't set attribute".to_owned())) @@ -115,7 +124,7 @@ impl PyProperty { #[pymethod(name = "__delete__")] fn delete(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(ref deleter) = self.deleter.as_ref() { + if let Some(ref deleter) = self.deleter.read().as_ref() { vm.invoke(deleter, obj) } else { Err(vm.new_attribute_error("can't delete attribute".to_owned())) @@ -126,25 +135,24 @@ impl PyProperty { #[pyproperty] fn fget(&self) -> Option { - self.getter.clone() + self.getter.read().clone() } #[pyproperty] fn fset(&self) -> Option { - self.setter.clone() + self.setter.read().clone() } #[pyproperty] fn fdel(&self) -> Option { - self.deleter.clone() + self.deleter.read().clone() } fn doc_getter(&self) -> Option { self.doc.read().clone() } - - fn doc_setter(&self, value: PyObjectRef, vm: &VirtualMachine) { - *self.doc.write() = py_none_to_option(vm, &value); + fn doc_setter(&self, value: Option) { + *self.doc.write() = value; } // Python builder functions @@ -156,9 +164,9 @@ impl PyProperty { vm: &VirtualMachine, ) -> PyResult { PyProperty { - getter: getter.or_else(|| zelf.getter.clone()), - setter: zelf.setter.clone(), - deleter: zelf.deleter.clone(), + getter: PyRwLock::new(getter.or_else(|| zelf.fget())), + setter: PyRwLock::new(zelf.fset()), + deleter: PyRwLock::new(zelf.fdel()), doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) @@ -171,9 +179,9 @@ impl PyProperty { vm: &VirtualMachine, ) -> PyResult { PyProperty { - getter: zelf.getter.clone(), - setter: setter.or_else(|| zelf.setter.clone()), - deleter: zelf.deleter.clone(), + getter: PyRwLock::new(zelf.fget()), + setter: PyRwLock::new(setter.or_else(|| zelf.fset())), + deleter: PyRwLock::new(zelf.fdel()), doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) @@ -186,24 +194,15 @@ impl PyProperty { vm: &VirtualMachine, ) -> PyResult { PyProperty { - getter: zelf.getter.clone(), - setter: zelf.setter.clone(), - deleter: deleter.or_else(|| zelf.deleter.clone()), + getter: PyRwLock::new(zelf.fget()), + setter: PyRwLock::new(zelf.fset()), + deleter: PyRwLock::new(deleter.or_else(|| zelf.fdel())), doc: PyRwLock::new(None), } .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } } -/// Take a python object and turn it into an option object, where python None maps to rust None. -fn py_none_to_option(vm: &VirtualMachine, value: &PyObjectRef) -> Option { - if vm.is_none(value) { - None - } else { - Some(value.clone()) - } -} - pub(crate) fn init(context: &PyContext) { PyProperty::extend_class(context, &context.types.property_type); diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index c35702cb7..0ae0663a6 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -17,8 +17,6 @@ use crate::scope::NameProtocol; use crate::slots::{SlotDescriptor, SlotGetattro}; use crate::vm::VirtualMachine; -use itertools::Itertools; - pub type PySuperRef = PyRef; #[pyclass(module = false, name = "super")] @@ -114,10 +112,12 @@ impl SlotGetattro for PySuper { None => return vm.generic_getattribute(zelf.into_object(), name), }; // skip the classes in obj_type.mro up to and including zelf.typ - let mut it = obj_type.iter_mro().peekable(); - for _ in it.peeking_take_while(|cls| !cls.is(&zelf.typ)) {} - for cls in it.skip(1) { - if let Some(descr) = cls.get_attr(name.borrow_value()) { + let it = obj_type + .iter_mro() + .skip_while(|cls| !cls.is(&zelf.typ)) + .skip(1); + for cls in it { + if let Some(descr) = cls.get_direct_attr(name.borrow_value()) { return vm .call_get_descriptor_specific( descr.clone(), diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 5fd805bf6..e46d7b6f3 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -673,13 +673,14 @@ impl PyType { pub fn get_attr(&self, attr_name: &str) -> Option { flame_guard!(format!("class_get_attr({:?})", attr_name)); - self.attributes - .read() - .get(attr_name) - .cloned() + self.get_direct_attr(attr_name) .or_else(|| self.get_super_attr(attr_name)) } + pub fn get_direct_attr(&self, attr_name: &str) -> Option { + self.attributes.read().get(attr_name).cloned() + } + pub fn get_super_attr(&self, attr_name: &str) -> Option { self.mro .iter() diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index e2ca95a21..62361aa5e 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1451,7 +1451,7 @@ mod fileio { #[derive(FromArgs)] struct FileIOArgs { #[pyarg(positional)] - name: Either, + name: PyObjectRef, #[pyarg(any, default)] mode: Option, #[pyarg(any, default = "true")] @@ -1477,44 +1477,40 @@ mod fileio { .mode .map(|mode| mode.borrow_value().to_owned()) .unwrap_or_else(|| "r".to_owned()); - let (name, file_no) = match args.name { - Either::A(name) => { - if !args.closefd { - return Err(vm.new_value_error( - "Cannot use closefd=False with file name".to_owned(), - )); - } - let mode = compute_c_flag(&mode); - let fd = if let Some(opener) = args.opener { - let fd = vm.invoke( - &opener, - vec![name.clone().into_object(), vm.ctx.new_int(mode)], - )?; - if !vm.isinstance(&fd, &vm.ctx.types.int_type)? { - return Err( - vm.new_type_error("expected integer from opener".to_owned()) - ); + let name = args.name.clone(); + let fd = if let Some(opener) = args.opener { + let mode = compute_c_flag(&mode); + let fd = vm.invoke(&opener, vec![name.clone(), vm.ctx.new_int(mode)])?; + if !vm.isinstance(&fd, &vm.ctx.types.int_type)? { + return Err(vm.new_type_error("expected integer from opener".to_owned())); + } + let fd = i64::try_from_object(vm, fd)?; + if fd < 0 { + return Err(vm.new_os_error("Negative file descriptor".to_owned())); + } + fd + } else { + match Either::::try_from_object(vm, args.name)? { + Either::A(fno) => fno, + Either::B(path) => { + if !args.closefd { + return Err(vm.new_value_error( + "Cannot use closefd=False with file name".to_owned(), + )); } - let fd = i64::try_from_object(vm, fd)?; - if fd < 0 { - return Err(vm.new_os_error("Negative file descriptor".to_owned())); - } - fd - } else { + let mode = compute_c_flag(&mode); os::open( - os::PyPathLike::new_str(name.borrow_value().to_owned()), + path, mode as _, OptionalArg::Missing, OptionalArg::Missing, vm, )? - }; - (name.into_object(), fd) + } } - Either::B(fno) => (vm.ctx.new_int(fno), fno), }; - zelf.fd.store(file_no); + zelf.fd.store(fd); zelf.closefd.store(args.closefd); vm.set_attr(zelf.as_object(), "name", name)?; vm.set_attr(zelf.as_object(), "mode", vm.ctx.new_str(mode))?; diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index c23028b33..1477a26a9 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -5,7 +5,7 @@ mod decl { use crate::bytecode; use crate::obj::objbytes::{PyBytes, PyBytesRef}; use crate::obj::objcode::{PyCode, PyCodeRef}; - use crate::pyobject::PyResult; + use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; #[pyfunction] @@ -13,10 +13,23 @@ mod decl { PyBytes::from(co.code.to_bytes()) } + #[pyfunction] + fn dump(co: PyCodeRef, f: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + vm.call_method(&f, "write", vec![dumps(co).into_pyobject(vm)])?; + Ok(()) + } + #[pyfunction] fn loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { let code = bytecode::CodeObject::from_bytes(&code_bytes) .map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?; Ok(PyCode { code }) } + + #[pyfunction] + fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let read_res = vm.call_method(&f, "read", vec![])?; + let bytes = PyBytesRef::try_from_object(vm, read_res)?; + loads(bytes, vm) + } }