diff --git a/.gitignore b/.gitignore index 967a51a83..e9dd6d220 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode wasm-pack.log .idea/ +tests/snippets/resources \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 36e313e26..864e90ce5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,6 +809,7 @@ dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Lib/PSF-LICENSE b/Lib/PSF-LICENSE new file mode 100644 index 000000000..1afbedba9 --- /dev/null +++ b/Lib/PSF-LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/README.md b/Lib/README.md new file mode 100644 index 000000000..b3aade7bf --- /dev/null +++ b/Lib/README.md @@ -0,0 +1,10 @@ +Standard Library for RustPython +=============================== + +This directory contains all of the Python files that make up the standard library for RustPython. + +Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them +to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. +Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation. + +The first target is to run the ``unittest`` module, so we can leverage the CPython test suite. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py new file mode 100644 index 000000000..c9f7daa16 --- /dev/null +++ b/Lib/_collections_abc.py @@ -0,0 +1,1013 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +from abc import ABCMeta, abstractmethod +import sys + +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", + ] + +# This module has been renamed from collections.abc to _collections_abc to +# speed up interpreter startup. Some of the types such as MutableMapping are +# required early but collections module imports a lot of other modules. +# See issue #19218 +__name__ = "collections.abc" + +# Private list of types that we want to register with the various ABCs +# so that they will pass tests like: +# it = iter(somebytearray) +# assert isinstance(it, Iterable) +# Note: in other implementations, these types might not be distinct +# and they may have their own implementation specific types that +# are not included on this list. +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +# list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +longrange_iterator = type(iter(range(1 << 1000))) +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +# mappingproxy = type(type.__dict__) +# generator = type((lambda: (yield))()) +## coroutine ## +# async def _coro(): pass +# _coro = _coro() +# coroutine = type(_coro) +# _coro.close() # Prevent ResourceWarning +# del _coro +# ## asynchronous generator ## +# async def _ag(): yield +# _ag = _ag() +# async_generator = type(_ag) +# del _ag + + +# ## ONE-TRICK PONIES ### + +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + return _check_methods(C, "__hash__") + return NotImplemented + + +# class Awaitable(metaclass=ABCMeta): + +# __slots__ = () + +# @abstractmethod +# def __await__(self): +# yield + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Awaitable: +# return _check_methods(C, "__await__") +# return NotImplemented + + +# class Coroutine(Awaitable): + +# __slots__ = () + +# @abstractmethod +# def send(self, value): +# """Send a value into the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# raise StopIteration + +# @abstractmethod +# def throw(self, typ, val=None, tb=None): +# """Raise an exception in the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val + +# def close(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# self.throw(GeneratorExit) +# except (GeneratorExit, StopIteration): +# pass +# else: +# raise RuntimeError("coroutine ignored GeneratorExit") + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Coroutine: +# return _check_methods(C, '__await__', 'send', 'throw', 'close') +# return NotImplemented + + +# Coroutine.register(coroutine) + + +# class AsyncIterable(metaclass=ABCMeta): + +# __slots__ = () + +# @abstractmethod +# def __aiter__(self): +# return AsyncIterator() + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterable: +# return _check_methods(C, "__aiter__") +# return NotImplemented + + +# class AsyncIterator(AsyncIterable): + +# __slots__ = () + +# @abstractmethod +# async def __anext__(self): +# """Return the next item or raise StopAsyncIteration when exhausted.""" +# raise StopAsyncIteration + +# def __aiter__(self): +# return self + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterator: +# return _check_methods(C, "__anext__", "__aiter__") +# return NotImplemented + + +# class AsyncGenerator(AsyncIterator): + +# __slots__ = () + +# async def __anext__(self): +# """Return the next item from the asynchronous generator. +# When exhausted, raise StopAsyncIteration. +# """ +# return await self.asend(None) + +# @abstractmethod +# async def asend(self, value): +# """Send a value into the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# raise StopAsyncIteration + +# @abstractmethod +# async def athrow(self, typ, val=None, tb=None): +# """Raise an exception in the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val + +# async def aclose(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# await self.athrow(GeneratorExit) +# except (GeneratorExit, StopAsyncIteration): +# pass +# else: +# raise RuntimeError("asynchronous generator ignored GeneratorExit") + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncGenerator: +# return _check_methods(C, '__aiter__', '__anext__', +# 'asend', 'athrow', 'aclose') +# return NotImplemented + + +# AsyncGenerator.register(async_generator) + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + return _check_methods(C, "__iter__") + return NotImplemented + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + 'Return the next item from the iterator. When exhausted, raise StopIteration' + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + return _check_methods(C, '__iter__', '__next__') + return NotImplemented + +# Iterator.register(bytes_iterator) +# Iterator.register(bytearray_iterator) +# Iterator.register(callable_iterator) +# Iterator.register(dict_keyiterator) +# Iterator.register(dict_valueiterator) +# Iterator.register(dict_itemiterator) +# Iterator.register(list_iterator) +# Iterator.register(list_reverseiterator) +# Iterator.register(range_iterator) +# Iterator.register(longrange_iterator) +# Iterator.register(set_iterator) +# Iterator.register(str_iterator) +# Iterator.register(tuple_iterator) +# Iterator.register(zip_iterator) + + +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + +class Generator(Iterator): + + __slots__ = () + + def __next__(self): + """Return the next item from the generator. + When exhausted, raise StopIteration. + """ + return self.send(None) + + @abstractmethod + def send(self, value): + """Send a value into the generator. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the generator. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside generator. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Generator: + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') + return NotImplemented + +# Generator.register(generator) + + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + return _check_methods(C, "__len__") + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + return _check_methods(C, "__contains__") + return NotImplemented + +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + return _check_methods(C, "__call__") + return NotImplemented + + +### SETS ### + + +class Set(Collection): + + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), redefine __le__ and __ge__, + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) < len(other): + return False + for elem in other: + if elem not in self: + return False + return True + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + __rand__ = __and__ + + def isdisjoint(self, other): + 'Return True if two sets have a null intersection.' + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + __ror__ = __or__ + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __rsub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in other + if value not in self) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + __rxor__ = __xor__ + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + """A mutable set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__, __len__, + add(), and discard(). + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Collection): + + __slots__ = () + + """A Mapping is a generic container for associating key/value + pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __iter__, and __len__. + + """ + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return KeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return ItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + +# Mapping.register(mappingproxy) + + +class MappingView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + +ItemsView.register(dict_items) + + +class ValuesView(MappingView, Collection): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + + __slots__ = () + + """A MutableMapping is a generic container for associating + key/value pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __setitem__, __delitem__, + __iter__, and __len__. + + """ + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + '''D.popitem() -> (k, v), remove and return some (key, value) pair + as a 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + 'D.clear() -> None. Remove all items from D.' + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. + If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v + In either case, this is followed by: for k, v in F.items(): D[k] = v + ''' + if not args: + raise TypeError("descriptor 'update' of 'MutableMapping' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Reversible, Collection): + + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + '''S.index(value, [start, [stop]]) -> integer -- return first index of value. + Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. + ''' + if start is not None and start < 0: + start = max(len(self) + start, 0) + if stop is not None and stop < 0: + stop += len(self) + + i = start + while stop is None or i < stop: + try: + v = self[i] + if v is value or v == value: + return i + except IndexError: + break + i += 1 + raise ValueError + + def count(self, value): + 'S.count(value) -> integer -- return number of occurrences of value' + return sum(1 for v in self if v is value or v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +Sequence.register(memoryview) + + +class ByteString(Sequence): + + """This unifies bytes and bytearray. + + XXX Should add all their methods. + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + + __slots__ = () + + """All the operations on a read-write sequence. + + Concrete subclasses must provide __new__ or __init__, + __getitem__, __setitem__, __delitem__, __len__, and insert(). + + """ + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + 'S.insert(index, value) -- insert value before index' + raise IndexError + + def append(self, value): + 'S.append(value) -- append value to the end of the sequence' + self.insert(len(self), value) + + def clear(self): + 'S.clear() -> None -- remove all items from S' + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + 'S.reverse() -- reverse *IN PLACE*' + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + '''S.pop([index]) -> item -- remove and return item at index (default last). + Raise IndexError if list is empty or index is out of range. + ''' + v = self[index] + del self[index] + return v + + def remove(self, value): + '''S.remove(value) -- remove first occurrence of value. + Raise ValueError if the value is not present. + ''' + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py new file mode 100644 index 000000000..07044f2f7 --- /dev/null +++ b/Lib/_py_abc.py @@ -0,0 +1,147 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, **kwargs): + cls = type.__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = set(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py new file mode 100644 index 000000000..304c66f59 --- /dev/null +++ b/Lib/_weakrefset.py @@ -0,0 +1,196 @@ +# Access WeakSet through the weakref module. +# This code is separated-out because it is needed +# by abc.py to load everything else at startup. + +from _weakref import ref + +__all__ = ['WeakSet'] + + +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + l = self._pending_removals + discard = self.data.discard + while l: + discard(l.pop()) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return (self.__class__, (list(self),), + getattr(self, '__dict__', None)) + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + newset = self.copy() + newset.difference_update(other) + return newset + __sub__ = difference + + def difference_update(self, other): + self.__isub__(other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + __and__ = intersection + + def intersection_update(self, other): + self.__iand__(other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __le__ = issubset + + def __lt__(self, other): + return self.data < set(map(ref, other)) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __ge__ = issuperset + + def __gt__(self, other): + return self.data > set(map(ref, other)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(map(ref, other)) + + def symmetric_difference(self, other): + newset = self.copy() + newset.symmetric_difference_update(other) + return newset + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + self.__ixor__(other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) + return self + + def union(self, other): + return self.__class__(e for s in (self, other) for e in s) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 diff --git a/Lib/abc.py b/Lib/abc.py new file mode 100644 index 000000000..709414127 --- /dev/null +++ b/Lib/abc.py @@ -0,0 +1,170 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) according to PEP 3119.""" + + +def abstractmethod(funcobj): + """A decorator indicating abstract methods. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractmethod + def my_abstract_method(self, ...): + ... + """ + funcobj.__isabstractmethod__ = True + return funcobj + + +class abstractclassmethod(classmethod): + """A decorator indicating abstract classmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractclassmethod + def my_abstract_classmethod(cls, ...): + ... + + 'abstractclassmethod' is deprecated. Use 'classmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractstaticmethod(staticmethod): + """A decorator indicating abstract staticmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractstaticmethod + def my_abstract_staticmethod(...): + ... + + 'abstractstaticmethod' is deprecated. Use 'staticmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractproperty(property): + """A decorator indicating abstract properties. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract properties are overridden. + The abstract properties can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractproperty + def my_abstract_property(self): + ... + + This defines a read-only property; you can also define a read-write + abstract property using the 'long' form of property declaration: + + class C(metaclass=ABCMeta): + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) + + 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' + instead. + """ + + __isabstractmethod__ = True + + +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + from _py_abc import ABCMeta, get_cache_token + ABCMeta.__module__ = 'abc' +else: + class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () diff --git a/Lib/os.py b/Lib/os.py new file mode 100644 index 000000000..8c19a89c6 --- /dev/null +++ b/Lib/os.py @@ -0,0 +1,122 @@ +from _os import * + +# Change environ to automatically call putenv(), unsetenv if they exist. +from _collections_abc import MutableMapping + +class _Environ(MutableMapping): + def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): + self.encodekey = encodekey + self.decodekey = decodekey + self.encodevalue = encodevalue + self.decodevalue = decodevalue + self.putenv = putenv + self.unsetenv = unsetenv + self._data = data + + def __getitem__(self, key): + try: + value = self._data[self.encodekey(key)] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) + + return self.decodevalue(value) + + def __setitem__(self, key, value): + key = self.encodekey(key) + value = self.encodevalue(value) + self.putenv(key, value) + self._data[key] = value + + def __delitem__(self, key): + encodedkey = self.encodekey(key) + self.unsetenv(encodedkey) + try: + del self._data[encodedkey] + except KeyError: + # raise KeyError with the original key value + # raise KeyError(key) from None + raise KeyError(key) + + def __iter__(self): + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: + yield self.decodekey(key) + + def __len__(self): + return len(self._data) + + def __repr__(self): + return 'environ({{{}}})'.format(', '.join( + ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) + for key, value in self._data.items()))) + + def copy(self): + return dict(self) + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None +# else: +# if "putenv" not in __all__: +# __all__.append("putenv") + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") +# else: +# if "unsetenv" not in __all__: +# __all__.append("unsetenv") + +def _createenviron(): + # if name == 'nt': + # # Where Env Var Names Must Be UPPERCASE + # def check_str(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value + # encode = check_str + # decode = str + # def encodekey(key): + # return encode(key).upper() + # data = {} + # for key, value in environ.items(): + # data[encodekey(key)] = value + # else: + # # Where Env Var Names Can Be Mixed Case + # encoding = sys.getfilesystemencoding() + # def encode(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value.encode(encoding, 'surrogateescape') + # def decode(value): + # return value.decode(encoding, 'surrogateescape') + # encodekey = encode + decode = str + encode = str + encodekey = encode + data = environ + return _Environ(data, + encodekey, decode, + encode, decode, + _putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + + +def getenv(key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default. + key, default and the result are str.""" + return environ.get(key, default) diff --git a/Lib/re.py b/Lib/re.py deleted file mode 100644 index 8511222be..000000000 --- a/Lib/re.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" Regular expressions """ - - -def match(pattern, string, flags=0): - return _compile(pattern, flags).match(string) - - -def _compile(pattern, flags): - p = sre_compile.compile(pattern, flags) - return p diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c15d47ad5..91fdc150e 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -8,6 +8,6 @@ edition = "2018" proc-macro = true [dependencies] -syn = "0.15.29" +syn = { version = "0.15.29", features = ["full"] } quote = "0.6.11" proc-macro2 = "0.4.27" diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs new file mode 100644 index 000000000..d017a32e5 --- /dev/null +++ b/derive/src/from_args.rs @@ -0,0 +1,200 @@ +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; + +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +enum ParameterKind { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, +} + +impl ParameterKind { + fn from_ident(ident: &Ident) -> ParameterKind { + if ident == "positional_only" { + ParameterKind::PositionalOnly + } else if ident == "positional_or_keyword" { + ParameterKind::PositionalOrKeyword + } else if ident == "keyword_only" { + ParameterKind::KeywordOnly + } else { + panic!("Unrecognised attribute") + } + } +} + +struct ArgAttribute { + kind: ParameterKind, + default: Option, + optional: bool, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option { + if !attr.path.is_ident("pyarg") { + return None; + } + + match attr.parse_meta().unwrap() { + Meta::List(list) => { + let mut iter = list.nested.iter(); + let first_arg = iter.next().expect("at least one argument in pyarg list"); + let kind = match first_arg { + NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + + let mut attribute = ArgAttribute { + kind, + default: None, + optional: false, + }; + + while let Some(arg) = iter.next() { + attribute.parse_argument(arg); + } + + assert!( + attribute.default.is_none() || !attribute.optional, + "Can't set both a default value and optional" + ); + + Some(attribute) + } + _ => panic!("Bad syntax for pyarg attribute"), + } + } + + fn parse_argument(&mut self, arg: &NestedMeta) { + match arg { + NestedMeta::Meta(Meta::Word(ident)) => { + if ident == "default" { + assert!(self.default.is_none(), "Default already set"); + let expr = syn::parse_str::("Default::default()").unwrap(); + self.default = Some(expr); + } else if ident == "optional" { + self.optional = true; + } else { + panic!("Unrecognised pyarg attribute '{}'", ident); + } + } + NestedMeta::Meta(Meta::NameValue(name_value)) => { + if name_value.ident == "default" { + assert!(self.default.is_none(), "Default already set"); + + match name_value.lit { + Lit::Str(ref val) => { + let expr = val + .parse::() + .expect("a valid expression for default argument"); + self.default = Some(expr); + } + _ => panic!("Expected string value for default argument"), + } + } else if name_value.ident == "optional" { + match name_value.lit { + Lit::Bool(ref val) => { + self.optional = val.value; + } + _ => panic!("Expected boolean value for optional argument"), + } + } else { + panic!("Unrecognised pyarg attribute '{}'", name_value.ident); + } + } + _ => panic!("Bad syntax for first pyarg attribute argument"), + }; + } +} + +fn generate_field(field: &Field) -> TokenStream2 { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(ArgAttribute::from_attribute) + .collect::>(); + let attr = if pyarg_attrs.is_empty() { + ArgAttribute { + kind: ParameterKind::PositionalOrKeyword, + default: None, + optional: false, + } + } else if pyarg_attrs.len() == 1 { + pyarg_attrs.remove(0) + } else { + panic!( + "Multiple pyarg attributes on field '{}'", + field.ident.as_ref().unwrap() + ); + }; + + let name = &field.ident; + let middle = quote! { + .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + }; + let ending = if let Some(default) = attr.default { + quote! { + .unwrap_or_else(|| #default) + } + } else if attr.optional { + quote! { + .map(::rustpython_vm::function::OptionalArg::Present) + .unwrap_or(::rustpython_vm::function::OptionalArg::Missing) + } + } else { + let err = match attr.kind { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { + ::rustpython_vm::function::ArgumentError::TooFewArgs + }, + ParameterKind::KeywordOnly => quote! { + ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + }, + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + match attr.kind { + ParameterKind::PositionalOnly => { + quote! { + #name: args.take_positional()#middle#ending, + } + } + ParameterKind::PositionalOrKeyword => { + quote! { + #name: args.take_positional_keyword(stringify!(#name))#middle#ending, + } + } + ParameterKind::KeywordOnly => { + quote! { + #name: args.take_keyword(stringify!(#name))#middle#ending, + } + } + } +} + +pub fn impl_from_args(input: DeriveInput) -> TokenStream2 { + let fields = match input.data { + Data::Struct(ref data) => { + match data.fields { + Fields::Named(ref fields) => fields.named.iter().map(generate_field), + Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message + } + } + Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message + }; + + let name = &input.ident; + quote! { + impl ::rustpython_vm::function::FromArgs for #name { + fn from_args( + vm: &::rustpython_vm::VirtualMachine, + args: &mut ::rustpython_vm::function::PyFuncArgs + ) -> Result { + Ok(#name { #(#fields)* }) + } + } + } +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 8f15f0e87..26dce935a 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,215 +1,28 @@ extern crate proc_macro; use proc_macro::TokenStream; -use proc_macro2::TokenStream as TokenStream2; -use quote::quote; -use syn::{Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta}; +use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; + +mod from_args; +mod pyclass; #[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { let ast: DeriveInput = syn::parse(input).unwrap(); - let gen = impl_from_args(&ast); - gen.to_string().parse().unwrap() + from_args::impl_from_args(ast).into() } -/// The kind of the python parameter, this corresponds to the value of Parameter.kind -/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) -enum ParameterKind { - PositionalOnly, - PositionalOrKeyword, - KeywordOnly, +#[proc_macro_attribute] +pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + pyclass::impl_pyclass(attr, item).into() } -impl ParameterKind { - fn from_ident(ident: &Ident) -> ParameterKind { - if ident == "positional_only" { - ParameterKind::PositionalOnly - } else if ident == "positional_or_keyword" { - ParameterKind::PositionalOrKeyword - } else if ident == "keyword_only" { - ParameterKind::KeywordOnly - } else { - panic!("Unrecognised attribute") - } - } -} - -struct ArgAttribute { - kind: ParameterKind, - default: Option, - optional: bool, -} - -impl ArgAttribute { - fn from_attribute(attr: &Attribute) -> Option { - if !attr.path.is_ident("pyarg") { - return None; - } - - match attr.parse_meta().unwrap() { - Meta::List(list) => { - let mut iter = list.nested.iter(); - let first_arg = iter.next().expect("at least one argument in pyarg list"); - let kind = match first_arg { - NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - - let mut attribute = ArgAttribute { - kind, - default: None, - optional: false, - }; - - while let Some(arg) = iter.next() { - attribute.parse_argument(arg); - } - - assert!( - attribute.default.is_none() || !attribute.optional, - "Can't set both a default value and optional" - ); - - Some(attribute) - } - _ => panic!("Bad syntax for pyarg attribute"), - } - } - - fn parse_argument(&mut self, arg: &NestedMeta) { - match arg { - NestedMeta::Meta(Meta::Word(ident)) => { - if ident == "default" { - assert!(self.default.is_none(), "Default already set"); - let expr = syn::parse_str::("Default::default()").unwrap(); - self.default = Some(expr); - } else if ident == "optional" { - self.optional = true; - } else { - panic!("Unrecognised pyarg attribute '{}'", ident); - } - } - NestedMeta::Meta(Meta::NameValue(name_value)) => { - if name_value.ident == "default" { - assert!(self.default.is_none(), "Default already set"); - - match name_value.lit { - Lit::Str(ref val) => { - let expr = val - .parse::() - .expect("a valid expression for default argument"); - self.default = Some(expr); - } - _ => panic!("Expected string value for default argument"), - } - } else if name_value.ident == "optional" { - match name_value.lit { - Lit::Bool(ref val) => { - self.optional = val.value; - } - _ => panic!("Expected boolean value for optional argument"), - } - } else { - panic!("Unrecognised pyarg attribute '{}'", name_value.ident); - } - } - _ => panic!("Bad syntax for first pyarg attribute argument"), - }; - } -} - -fn generate_field(field: &Field) -> TokenStream2 { - let mut pyarg_attrs = field - .attrs - .iter() - .filter_map(ArgAttribute::from_attribute) - .collect::>(); - let attr = if pyarg_attrs.is_empty() { - ArgAttribute { - kind: ParameterKind::PositionalOrKeyword, - default: None, - optional: false, - } - } else if pyarg_attrs.len() == 1 { - pyarg_attrs.remove(0) - } else { - panic!( - "Multiple pyarg attributes on field '{}'", - field.ident.as_ref().unwrap() - ); - }; - - let name = &field.ident; - let middle = quote! { - .map(|x| crate::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? - }; - let ending = if let Some(default) = attr.default { - quote! { - .unwrap_or_else(|| #default) - } - } else if attr.optional { - quote! { - .map(crate::function::OptionalArg::Present) - .unwrap_or(crate::function::OptionalArg::Missing) - } - } else { - let err = match attr.kind { - ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - crate::function::ArgumentError::TooFewArgs - }, - ParameterKind::KeywordOnly => quote! { - crate::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) - }, - }; - quote! { - .ok_or_else(|| #err)? - } - }; - - match attr.kind { - ParameterKind::PositionalOnly => { - quote! { - #name: args.take_positional()#middle#ending, - } - } - ParameterKind::PositionalOrKeyword => { - quote! { - #name: args.take_positional_keyword(stringify!(#name))#middle#ending, - } - } - ParameterKind::KeywordOnly => { - quote! { - #name: args.take_keyword(stringify!(#name))#middle#ending, - } - } - } -} - -fn impl_from_args(input: &DeriveInput) -> TokenStream2 { - // FIXME: This references types using `crate` instead of `rustpython_vm` - // so that it can be used in the latter. How can we support both? - // Can use extern crate self as rustpython_vm; once in stable. - // https://github.com/rust-lang/rust/issues/56409 - let fields = match input.data { - Data::Struct(ref data) => { - match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(generate_field), - Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message - } - } - Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message - }; - - let name = &input.ident; - quote! { - impl crate::function::FromArgs for #name { - fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { - Ok(#name { #(#fields)* }) - } - } - } +#[proc_macro_attribute] +pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + pyclass::impl_pyimpl(attr, item).into() } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs new file mode 100644 index 000000000..36ce9df1a --- /dev/null +++ b/derive/src/pyclass.rs @@ -0,0 +1,276 @@ +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use std::collections::HashMap; +use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; + +enum ClassItem { + Method { + item_ident: Ident, + py_name: String, + }, + Property { + item_ident: Ident, + py_name: String, + setter: bool, + }, +} + +fn meta_to_vec(meta: Meta) -> Option> { + match meta { + Meta::Word(_) => Some(Vec::new()), + Meta::List(list) => Some(list.nested.into_iter().collect()), + Meta::NameValue(_) => None, + } +} + +impl ClassItem { + fn extract_from_syn(attrs: &mut Vec, sig: &MethodSig) -> Option { + let mut item = None; + let mut attr_idx = None; + // TODO: better error handling throughout this + for (i, meta) in attrs + .iter() + .filter_map(|attr| attr.parse_meta().ok()) + .enumerate() + { + let name = meta.name(); + if name == "pymethod" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") + } + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => continue, + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pymethod(name = ...)] must be a string"); + } + } + } + _ => {} + } + } + item = Some(ClassItem::Method { + item_ident: sig.ident.clone(), + py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), + }); + attr_idx = Some(i); + } else if name == "pyproperty" { + if item.is_some() { + panic!("You can only have one #[py*] attribute on an impl item") + } + let nesteds = meta_to_vec(meta).expect( + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]", + ); + let mut setter = false; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => continue, + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + panic!("#[pyproperty(name = ...)] must be a string"); + } + } + } + Meta::Word(ident) => { + if ident == "setter" { + setter = true; + } + } + _ => {} + } + } + let py_name = py_name.unwrap_or_else(|| { + let item_ident = sig.ident.to_string(); + if setter { + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; + if name.is_empty() { + panic!( + "A #[pyproperty(setter)] fn with a set_* name have something \ + after \"set_\"" + ) + } else { + name.to_string() + } + } else { + panic!( + "A #[pyproperty(setter)] fn must either have a `name` parameter or a \ + fn name along the lines of \"set_*\"" + ) + } + } else { + item_ident + } + }); + item = Some(ClassItem::Property { + py_name, + item_ident: sig.ident.clone(), + setter, + }); + attr_idx = Some(i); + } + } + if let Some(attr_idx) = attr_idx { + attrs.remove(attr_idx); + } + item + } +} + +pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> TokenStream2 { + let mut imp = if let Item::Impl(imp) = item { + imp + } else { + return quote!(#item); + }; + + let items = imp + .items + .iter_mut() + .filter_map(|item| { + if let ImplItem::Method(meth) = item { + ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig) + } else { + None + } + }) + .collect::>(); + let ty = &imp.self_ty; + let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); + for item in items.iter() { + match item { + ClassItem::Property { + item_ident, + py_name, + setter, + } => { + let entry = properties.entry(py_name).or_default(); + let func = if *setter { &mut entry.1 } else { &mut entry.0 }; + if func.is_some() { + panic!("Multiple property accessors with name {:?}", py_name) + } + *func = Some(item_ident); + } + _ => {} + } + } + let methods = items.iter().filter_map(|item| { + if let ClassItem::Method { + item_ident, + py_name, + } = item + { + Some(quote! { + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident)); + }) + } else { + None + } + }); + let properties = properties.iter().map(|(name, prop)| { + let getter = match prop.0 { + Some(getter) => getter, + None => panic!("Property {:?} is missing a getter", name), + }; + let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); + quote! { + class.set_str_attr( + #name, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(Self::#getter) + #add_setter + .create(), + ); + } + }); + + quote! { + #imp + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#methods)* + #(#properties)* + } + } + } +} + +pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> TokenStream2 { + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), + _ => panic!("#[pyclass] can only be on a struct or enum declaration"), + }; + + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + panic!("#[pyclass(name = ...)] must be a string"); + } + } + } + } + } + let class_name = class_name.unwrap_or_else(|| ident.to_string()); + + let mut doc: Option> = None; + for attr in attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } else { + panic!("expected #[doc = ...] to be a string") + } + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + + quote! { + #item + impl ::rustpython_vm::pyobject::PyClassDef for #ident { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + } + } +} diff --git a/parser/.gitignore b/parser/.gitignore deleted file mode 100644 index 401e8cc22..000000000 --- a/parser/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -src/python.rs -target/ -Cargo.lock - diff --git a/parser/build.rs b/parser/build.rs index 65147ed0e..19f102721 100644 --- a/parser/build.rs +++ b/parser/build.rs @@ -1,8 +1,5 @@ use lalrpop; fn main() { - lalrpop::Configuration::new() - .generate_in_source_tree() - .process() - .unwrap(); + lalrpop::process_root().unwrap() } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 257ceadf2..9fbe8153e 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -16,7 +16,7 @@ pub struct Node { #[derive(Debug, PartialEq)] pub enum Top { Program(Program), - Statement(LocatedStatement), + Statement(Vec), Expression(Expression), } @@ -47,7 +47,7 @@ pub enum Statement { Break, Continue, Return { - value: Option>, + value: Option>, }, Import { import_parts: Vec, @@ -94,7 +94,7 @@ pub enum Statement { }, For { target: Expression, - iter: Vec, + iter: Expression, body: Vec, orelse: Option>, }, @@ -114,12 +114,10 @@ pub enum Statement { bases: Vec, keywords: Vec, decorator_list: Vec, - // TODO: docstring: String, }, FunctionDef { name: String, args: Parameters, - // docstring: String, body: Vec, decorator_list: Vec, returns: Option, @@ -159,9 +157,8 @@ pub enum Expression { value: Box, }, Compare { - a: Box, - op: Comparison, - b: Box, + vals: Vec, + ops: Vec, }, Attribute { value: Box, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index defa5e232..dfc9ecd51 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -466,8 +466,16 @@ where loop { match self.next_char() { Some('\\') => { - if is_raw { + if self.chr0 == Some(quote_char) { + string_content.push(quote_char); + self.next_char(); + } else if is_raw { string_content.push('\\'); + if let Some(c) = self.next_char() { + string_content.push(c) + } else { + return Err(LexicalError::StringError); + } } else { match self.next_char() { Some('\\') => { @@ -532,8 +540,12 @@ where let end_pos = self.get_pos(); let tok = if is_bytes { - Tok::Bytes { - value: string_content.as_bytes().to_vec(), + if string_content.is_ascii() { + Tok::Bytes { + value: string_content.as_bytes().to_vec(), + } + } else { + return Err(LexicalError::StringError); } } else { Tok::String { @@ -711,7 +723,6 @@ where let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); - println!("Emoji: {}", c); return Some(Ok(( tok_start, Tok::Name { @@ -815,7 +826,11 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + tok_end, + ))); } } } @@ -1037,16 +1052,20 @@ where return Some(Ok((tok_start, Tok::Comma, tok_end))); } '.' => { - let tok_start = self.get_pos(); - self.next_char(); - if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { - self.next_char(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + if let Some('0'..='9') = self.chr1 { + return Some(self.lex_number()); } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Dot, tok_end))); + let tok_start = self.get_pos(); + self.next_char(); + if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { + self.next_char(); + self.next_char(); + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); + } else { + let tok_end = self.get_pos(); + return Some(Ok((tok_start, Tok::Dot, tok_end))); + } } } '\n' => { @@ -1438,7 +1457,7 @@ mod tests { #[test] fn test_string() { - let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g'"#); + let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#); let tokens = lex_source(&source); assert_eq!( tokens, @@ -1467,6 +1486,10 @@ mod tests { value: String::from("\\g"), is_fstring: false, }, + Tok::String { + value: String::from("raw\'"), + is_fstring: false, + }, ] ); } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index b10c3d551..5cc8ca3c4 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,11 +1,14 @@ #[macro_use] extern crate log; +use lalrpop_util::lalrpop_mod; pub mod ast; pub mod error; mod fstring; pub mod lexer; pub mod parser; -#[cfg_attr(rustfmt, rustfmt_skip)] -mod python; +lalrpop_mod!( + #[allow(clippy::all)] + python +); pub mod token; diff --git a/parser/src/parser.rs b/parser/src/parser.rs index a119740ec..dd58bd11d 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result { do_lalr_parsing!(source, Program, StartProgram) } -pub fn parse_statement(source: &str) -> Result { +pub fn parse_statement(source: &str) -> Result, ParseError> { do_lalr_parsing!(source, Statement, StartStatement) } @@ -180,7 +180,7 @@ mod tests { let parse_ast = parse_statement(&source).unwrap(); assert_eq!( parse_ast, - ast::LocatedStatement { + vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::If { test: ast::Expression::Number { @@ -229,7 +229,7 @@ mod tests { } },]), } - } + }] ); } @@ -239,7 +239,7 @@ mod tests { let parse_ast = parse_statement(&source); assert_eq!( parse_ast, - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Expression { expression: ast::Expression::Lambda { @@ -271,7 +271,7 @@ mod tests { }) } } - }) + }]) ) } @@ -281,7 +281,7 @@ mod tests { assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::Assign { targets: vec![ast::Expression::Tuple { @@ -309,7 +309,7 @@ mod tests { ] } } - }) + }]) ) } @@ -320,7 +320,7 @@ mod tests { ); assert_eq!( parse_statement(&source), - Ok(ast::LocatedStatement { + Ok(vec![ast::LocatedStatement { location: ast::Location::new(1, 1), node: ast::Statement::ClassDef { name: String::from("Foo"), @@ -393,7 +393,7 @@ mod tests { ], decorator_list: vec![], } - }) + }]) ) } @@ -460,26 +460,30 @@ mod tests { }, ifs: vec![ ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Less, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(5) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(5) + } } - }), + ], + ops: vec![ast::Comparison::Less], }, ast::Expression::Compare { - a: Box::new(ast::Expression::Identifier { - name: "a".to_string() - }), - op: ast::Comparison::Greater, - b: Box::new(ast::Expression::Number { - value: ast::Number::Integer { - value: BigInt::from(10) + vals: vec![ + ast::Expression::Identifier { + name: "a".to_string() + }, + ast::Expression::Number { + value: ast::Number::Integer { + value: BigInt::from(10) + } } - }), + ], + ops: vec![ast::Comparison::Greater], }, ], } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 9e5b309b1..a9663943f 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -2,7 +2,6 @@ // See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4 // See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword -#![allow(unknown_lints,clippy)] use std::iter::FromIterator; @@ -25,29 +24,32 @@ pub Top: ast::Top = { Program: ast::Program = { => ast::Program { - statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) + statements: Vec::from_iter(lines.into_iter().flatten()) }, }; // A file line either has a declaration, or an empty newline: -FileLine: Option = { - => Some(s), - "\n" => None, +FileLine: Vec = { + Statement, + "\n" => vec![], }; Suite: Vec = { - => vec![s], - "\n" indent dedent => s, -}; - -Statement: ast::LocatedStatement = { SimpleStatement, - CompoundStatement, + "\n" indent dedent => s.into_iter().flatten().collect(), }; -SimpleStatement: ast::LocatedStatement = { - "\n" => s, - ";" => s, +Statement: Vec = { + SimpleStatement, + => vec![s], +}; + +SimpleStatement: Vec = { + ";"? "\n" => { + let mut statements = vec![s1]; + statements.extend(s2.into_iter().map(|e| e.1)); + statements + } }; SmallStatement: ast::LocatedStatement = { @@ -103,9 +105,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { - // TODO: this works in most cases: - let rhs = e2.into_iter().next().unwrap(); + => { ast::LocatedStatement { location: loc, node: ast::Statement::AugAssign { @@ -118,31 +118,17 @@ ExpressionStatement: ast::LocatedStatement = { }; AssignSuffix: ast::Expression = { - "=" => { - if e.len() > 1 { - ast::Expression::Tuple { - elements: e - } - } else { - e.into_iter().next().unwrap() - } - }, + "=" => e, "=" => e, }; TestOrStarExprList: ast::Expression = { - => { - let mut res = vec![e]; - res.extend(e2.into_iter().map(|x| x.1)); - - // First build tuple from first item: - let expr = if (res.len() > 1) || comma.is_some() { - ast::Expression::Tuple { elements: res } + > => { + if elements.len() == 1 && comma.is_none() { + elements.into_iter().next().unwrap() } else { - res.into_iter().next().unwrap() - }; - - expr + ast::Expression::Tuple { elements } + } } }; @@ -183,7 +169,7 @@ FlowStatement: ast::LocatedStatement = { "return" => { ast::LocatedStatement { location: loc, - node: ast::Statement::Return { value: t }, + node: ast::Statement::Return { value: t.map(Box::new) }, } }, => { @@ -310,15 +296,11 @@ NonlocalStatement: ast::LocatedStatement = { }; AssertStatement: ast::LocatedStatement = { - "assert" => { + "assert" => { ast::LocatedStatement { location: loc, node: ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, - } + test, msg: msg.map(|e| e.1) } } }, @@ -335,12 +317,9 @@ CompoundStatement: ast::LocatedStatement = { }; IfStatement: ast::LocatedStatement = { - "if" ":" => { + "if" ":" => { // Determine last else: - let mut last = match s3 { - Some(s) => Some(s.2), - None => None, - }; + let mut last = s3.map(|s| s.2); // handle elif: for i in s2.into_iter().rev() { @@ -353,30 +332,24 @@ IfStatement: ast::LocatedStatement = { ast::LocatedStatement { location: loc, - node: ast::Statement::If { test: t, body: s1, orelse: last } + node: ast::Statement::If { test, body: s1, orelse: last } } }, }; WhileStatement: ast::LocatedStatement = { - "while" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + "while" ":" => { + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::While { test: e, body: s, orelse: or_else }, + node: ast::Statement::While { test, body, orelse: or_else }, } }, }; ForStatement: ast::LocatedStatement = { "for" "in" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::For { @@ -389,14 +362,8 @@ ForStatement: ast::LocatedStatement = { TryStatement: ast::LocatedStatement = { "try" ":" => { - let or_else = match else_suite { - Some(s) => Some(s.2), - None => None, - }; - let finalbody = match finally { - Some(s) => Some(s.2), - None => None, - }; + let or_else = else_suite.map(|s| s.2); + let finalbody = finally.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::Try { @@ -437,10 +404,7 @@ WithStatement: ast::LocatedStatement = { WithItem: ast::WithItem = { => { - let optional_vars = match n { - Some(val) => Some(val.1), - None => None, - }; + let optional_vars = n.map(|val| val.1); ast::WithItem { context_expr: t, optional_vars } }, }; @@ -461,27 +425,17 @@ FuncDef: ast::LocatedStatement = { }; Parameters: ast::Parameters = { - "(" )?> ")" => { - match a { - Some(a) => a, - None => Default::default(), - } - }, + "(" )?> ")" => a.unwrap_or_else(Default::default), }; -// parameters are (String, None), kwargs are (String, Some(Test)) where Test is -// the default // Note that this is a macro which is used once for function defs, and // once for lambda defs. -TypedArgsList: ast::Parameters = { - > )?> => { +ParameterList: ast::Parameters = { + > )?> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: - let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 { - Some((_, x)) => x, - None => (None, vec![], vec![], None), - }; + let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); ast::Parameters { args: names, @@ -492,7 +446,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > )> => { + > )> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: @@ -510,7 +464,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { let (vararg, kwonlyargs, kw_defaults, kwarg) = params; ast::Parameters { args: vec![], @@ -521,7 +475,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { ast::Parameters { args: vec![], kwonlyargs: vec![], @@ -535,8 +489,8 @@ TypedArgsList: ast::Parameters = { // Use inline here to make sure the "," is not creating an ambiguity. #[inline] -TypedParameters: (Vec, Vec) = { - > )*> => { +ParameterDefs: (Vec, Vec) = { + > )*> => { // Combine first parameters: let mut args = vec![param1]; args.extend(param2.into_iter().map(|x| x.1)); @@ -564,7 +518,7 @@ TypedParameters: (Vec, Vec) = { } }; -TypedParameterDef: (ast::Parameter, Option) = { +ParameterDef: (ast::Parameter, Option) = { => (i, None), "=" => (i, Some(e)), }; @@ -580,8 +534,11 @@ TypedParameter: ast::Parameter = { }, }; +// Use inline here to make sure the "," is not creating an ambiguity. +// TODO: figure out another grammar that makes this inline no longer required. +#[inline] ParameterListStarArgs: (Option>, Vec, Vec>, Option>) = { - "*" )*> )?> => { + "*" )*> )?> => { // Extract keyword arguments: let mut kwonlyargs = vec![]; let mut kw_defaults = vec![]; @@ -590,10 +547,7 @@ ParameterListStarArgs: (Option>, Vec Some(name), - None => None, - }; + let kwarg = kwarg.map(|n| n.1); (Some(va), kwonlyargs, kw_defaults, kwarg) } @@ -649,66 +603,59 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => { - ast::Expression::Yield { - value: ex.map(|expr| Box::new( - if expr.len() > 1 { - ast::Expression::Tuple { elements: expr } - } else { - expr.into_iter().next().unwrap() - }) - ) - } - }, - "yield" "from" => { - ast::Expression::YieldFrom { - value: Box::new(e), - } - }, + "yield" => ast::Expression::Yield { value: value.map(Box::new) }, + "yield" "from" => ast::Expression::YieldFrom { value: Box::new(e) }, }; Test: ast::Expression = { - => { - match c { - Some(c) => { - ast::Expression::IfExpression { - test: Box::new(c.1), - body: Box::new(e), - orelse: Box::new(c.3), - } - }, - None => e, + => { + if let Some(c) = condition { + ast::Expression::IfExpression { + test: Box::new(c.1), + body: Box::new(expr), + orelse: Box::new(c.3), + } + } else { + expr } }, - => e, + LambdaDef, }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body:Box::new(b) + body: Box::new(body) } } OrTest: ast::Expression = { - => e, + AndTest, "or" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }, }; AndTest: ast::Expression = { - => e, + NotTest, "and" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }, }; NotTest: ast::Expression = { "not" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }, - => e, + Comparison, }; Comparison: ast::Expression = { - => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) }, - => e, + => { + let mut vals = vec![e]; + let mut ops = vec![]; + for x in comparisons { + ops.push(x.0); + vals.push(x.1); + } + ast::Expression::Compare { vals, ops } + }, + Expression, }; CompOp: ast::Comparison = { @@ -726,7 +673,7 @@ CompOp: ast::Comparison = { Expression: ast::Expression = { "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, - => e, + XorExpression, }; XorExpression: ast::Expression = { @@ -776,7 +723,7 @@ Factor: ast::Expression = { "+" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos }, "-" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg }, "~" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv }, - => e, + Power, }; Power: ast::Expression = { @@ -789,7 +736,7 @@ Power: ast::Expression = { }; AtomExpr: ast::Expression = { - => e, + Atom, "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, "." => ast::Expression::Attribute { value: Box::new(e), name: n }, @@ -810,7 +757,7 @@ SubscriptList: ast::Expression = { }; Subscript: ast::Expression = { - => e, + Test, ":" => { let s1 = e1.unwrap_or(ast::Expression::None); let s2 = e2.unwrap_or(ast::Expression::None); @@ -824,30 +771,17 @@ SliceOp: ast::Expression = { } Atom: ast::Expression = { - => ast::Expression::String { value: s }, - => ast::Expression::Bytes { value: b }, - => ast::Expression::Number { value: n }, - => ast::Expression::Identifier { name: i }, + => ast::Expression::String { value }, + => ast::Expression::Bytes { value }, + => ast::Expression::Number { value }, + => ast::Expression::Identifier { name }, "[" "]" => { let elements = e.unwrap_or(Vec::new()); ast::Expression::List { elements } }, - "[" "]" => { - // List comprehension: - e - }, - "(" ")" => { - match e { - None => ast::Expression::Tuple { elements: Vec::new() }, - Some(elements) => { - if elements.len() == 1 && trailing_comma.is_none() { - // This is "(e)", which is equivalent to "e" - elements.into_iter().next().unwrap() - } else { - ast::Expression::Tuple { elements } - } - } - } + "[" "]" => e, + "(" ")" => { + elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() }) }, "(" ")" => { ast::Expression::Comprehension { @@ -866,9 +800,7 @@ Atom: ast::Expression = { }; TestListComp: Vec = { - > <_trailing_comma:","?> => { - e - }, + > <_trailing_comma:","?> => e, }; TestListComp2: ast::Expression = { @@ -881,9 +813,7 @@ TestListComp2: ast::Expression = { }; TestDict: Vec<(ast::Expression, ast::Expression)> = { - > <_trailing_comma:","?> => { - e1 - } + > <_trailing_comma:","?> => elements, }; TestDictComp: ast::Expression = { @@ -900,9 +830,7 @@ DictEntry: (ast::Expression, ast::Expression) = { }; TestSet: Vec = { - > ","? => { - e1 - } + > ","? => e1 }; TestSetComp: ast::Expression = { @@ -914,31 +842,37 @@ TestSetComp: ast::Expression = { } }; +ExpressionOrStarExpression = { + Expression, + StarExpr +}; + ExpressionList: ast::Expression = { - => { - if e.len() == 1 { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements: e } + ast::Expression::Tuple { elements } } }, }; ExpressionList2: Vec = { - ","? => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - }, + > ","? => elements, }; -#[inline] -TestList: Vec = { - => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - } +// A test list is one of: +// - a list of expressions +// - a single expression +// - a single expression followed by a trailing comma +TestList: ast::Expression = { + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() + } else { + ast::Expression::Tuple { elements } + } + } }; // Test @@ -947,27 +881,16 @@ StarExpr: ast::Expression = { }; // Comprehensions: -CompFor: Vec = { - => c, -}; +CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { - ast::Comprehension { - target: e, - iter: i, - ifs: c2, - } + "for" "in" => { + ast::Comprehension { target, iter, ifs: c2 } } }; -ExpressionNoCond: ast::Expression = { - OrTest, -}; - -ComprehensionIf: ast::Expression = { - "if" => c, -}; +ExpressionNoCond: ast::Expression = OrTest; +ComprehensionIf: ast::Expression = "if" => c; ArgumentList: (Vec, Vec) = { > => { @@ -979,8 +902,15 @@ ArgumentList: (Vec, Vec) = { keywords.push(ast::Keyword { name: n, value: value }); }, None => { - if keywords.len() > 0 { - panic!("positional argument follows keyword argument"); + // Allow starred args after keyword arguments. + let is_starred = if let ast::Expression::Starred { .. } = &value { + true + } else { + false + }; + + if keywords.len() > 0 && !is_starred { + panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); }, @@ -1024,8 +954,8 @@ OneOrMore: Vec = { }; Number: ast::Number = { - => { ast::Number::Integer { value: s } }, - => { ast::Number::Float { value: s } }, + => { ast::Number::Integer { value } }, + => { ast::Number::Float { value } }, => { ast::Number::Complex { real: s.0, imag: s.1 } }, }; diff --git a/src/main.rs b/src/main.rs index 750faf98a..2a673736e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,11 +65,8 @@ fn main() { } fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult { - let code_obj = - compile::compile(vm, source, &compile::Mode::Exec, source_path).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })?; + let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path) + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj.borrow()); let vars = vm.ctx.new_scope(); // Keep track of local variables vm.run_code_obj(code_obj, vars) @@ -120,8 +117,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com // Don't inject syntax errors for line continuation Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err), Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - let exc = vm.new_exception(syntax_error, format!("{}", err)); + let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); Err(err) } diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 000000000..05ba24c43 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +snippets/whats_left_to_implement.py diff --git a/tests/README.md b/tests/README.md index a225ba3b6..ba24ab8bc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,3 @@ - # Test snippets This directory contains two sets of test snippets which can be run in @@ -6,6 +5,10 @@ Python. The `snippets/` directory contains functional tests, and the `benchmarks/` directory contains snippets for use in benchmarking RustPython's performance. +## Generates the test for not implemented methods + +run using cpython not_impl_gen.py it automatically generate a +test snippet to check not yet implemented methods ## Running with CPython + RustPython @@ -15,4 +18,3 @@ compilation to bytecode. When this is done, run the bytecode with rustpython. ## Running with RustPython The other option is to run all snippets with RustPython. - diff --git a/tests/generator/not_impl_footer.txt b/tests/generator/not_impl_footer.txt new file mode 100644 index 000000000..5cd500999 --- /dev/null +++ b/tests/generator/not_impl_footer.txt @@ -0,0 +1,12 @@ +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] + +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: + print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_header.txt new file mode 100644 index 000000000..4ceefff4d --- /dev/null +++ b/tests/generator/not_impl_header.txt @@ -0,0 +1,7 @@ +# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE +# EDIT tests/not_impl_gen.py, NOT THIS FILE. +# RESULTS OF THIS TEST DEPEND ON THE CPYTHON +# VERSION USED TO RUN not_impl_gen.py + +import platform + diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py new file mode 100644 index 000000000..2e20f5609 --- /dev/null +++ b/tests/not_impl_gen.py @@ -0,0 +1,33 @@ +objects = [ + bool, + bytearray, + bytes, + complex, + dict, + float, + frozenset, + int, + iter, + list, + memoryview, + range, + set, + str, + tuple, + object +] + +header = open("generator/not_impl_header.txt") +footer = open("generator/not_impl_footer.txt") +output = open("snippets/whats_left_to_implement.py", "w") + +output.write(header.read()) +output.write("expected_methods = {\n") + +for obj in objects: + output.write(f" '{obj.__name__}': ({obj.__name__}, [\n") + output.write("\n".join(f" '{attr}'," for attr in dir(obj))) + output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n")) + +output.write("}\n\n") +output.write(footer.read()) diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index 43bf74756..dad767b45 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -21,3 +21,10 @@ print(foo.body[0].value.func.id) assert foo.body[0].value.func.id == 'print' assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 + +n = ast.parse("3 < 4 > 5\n") +assert n.body[0].value.left.n == 3 +assert 'Lt' in str(n.body[0].value.ops[0]) +assert 'Gt' in str(n.body[0].value.ops[1]) +assert n.body[0].value.comparators[0].n == 4 +assert n.body[0].value.comparators[1].n == 5 diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index a121fb718..b5fbf8d0b 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -8,6 +8,9 @@ a = A() assert "test" in dir(a), "test not in a" assert "test" in dir(A), "test not in A" +a.x = 3 +assert "x" in dir(a), "x not in a" + class B(A): def __dir__(self): return ('q', 'h') diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index c5dbd5d90..46592769e 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -2,9 +2,9 @@ from testutils import assert_raises a = [] assert a[:] == [] -assert a[:2**100] == [] -assert a[-2**100:] == [] -assert a[::2**100] == [] +assert a[: 2 ** 100] == [] +assert a[-2 ** 100 :] == [] +assert a[:: 2 ** 100] == [] assert a[10:20] == [] assert a[-20:-10] == [] @@ -12,15 +12,15 @@ b = [1, 2] assert b[:] == [1, 2] assert b[slice(None)] == [1, 2] -assert b[:2**100] == [1, 2] -assert b[-2**100:] == [1, 2] -assert b[2**100:] == [] -assert b[::2**100] == [1] +assert b[: 2 ** 100] == [1, 2] +assert b[-2 ** 100 :] == [1, 2] +assert b[2 ** 100 :] == [] +assert b[:: 2 ** 100] == [1] assert b[-10:1] == [1] assert b[0:0] == [] assert b[1:0] == [] -assert_raises(ValueError, lambda: b[::0], 'zero step slice') +assert_raises(ValueError, lambda: b[::0], "zero step slice") assert b[::-1] == [2, 1] assert b[1::-1] == [2, 1] @@ -34,7 +34,7 @@ c = list(range(10)) assert c[9:6:-3] == [9] assert c[9::-3] == [9, 6, 3, 0] assert c[9::-4] == [9, 5, 1] -assert c[8::-2**100] == [8] +assert c[8 :: -2 ** 100] == [8] assert c[7:7:-2] == [] assert c[7:8:-2] == [] @@ -44,6 +44,7 @@ d = "123456" assert d[3::-1] == "4321" assert d[4::-3] == "52" +assert [1, 2, 3, 5, 6][-1:-5:-1] == [6, 5, 3, 2] # #746 slice_a = slice(5) assert slice_a.start is None @@ -90,3 +91,41 @@ class CustomIndex: assert c[CustomIndex(1):CustomIndex(3)] == [1, 2] assert d[CustomIndex(1):CustomIndex(3)] == "23" + + +def test_all_slices(): + """ + test all possible slices except big number + """ + + mod = __import__('cpython_generated_slices') + + ll = mod.LL + start = mod.START + end = mod.END + step = mod.STEP + slices_res = mod.SLICES_RES + + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == slices_res[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, slices_res[count] + ) + ) + count += 1 + + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") + + +test_all_slices() diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py new file mode 100644 index 000000000..d2a60943c --- /dev/null +++ b/tests/snippets/bytes.py @@ -0,0 +1,160 @@ +from testutils import assertRaises + +# new +assert bytes([1, 2, 3]) +assert bytes((1, 2, 3)) +assert bytes(range(4)) +assert bytes(3) +assert b"bla" +assert bytes("bla", "utf8") +with assertRaises(TypeError): + bytes("bla") + + +a = b"abcd" +b = b"ab" +c = b"abcd" + +# +# repr +assert repr(bytes([0, 1, 2])) == repr(b"\x00\x01\x02") +assert repr( + bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) + == "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'" +) +assert repr(b"abcd") == "b'abcd'" + +# len +assert len(bytes("abcdé", "utf8")) == 6 + +# comp +assert a == b"abcd" +assert a > b +assert a >= b +assert b < a +assert b <= a + +assert b"foobar".__eq__(2) == NotImplemented +assert b"foobar".__ne__(2) == NotImplemented +assert b"foobar".__gt__(2) == NotImplemented +assert b"foobar".__ge__(2) == NotImplemented +assert b"foobar".__lt__(2) == NotImplemented +assert b"foobar".__le__(2) == NotImplemented + +# hash +hash(a) == hash(b"abcd") + +# iter +[i for i in b"abcd"] == ["a", "b", "c", "d"] +assert list(bytes(3)) == [0, 0, 0] + +# add +assert a + b == b"abcdab" + +# contains +assert b"ab" in b"abcd" +assert b"cd" in b"abcd" +assert b"abcd" in b"abcd" +assert b"a" in b"abcd" +assert b"d" in b"abcd" +assert b"dc" not in b"abcd" +assert 97 in b"abcd" +assert 150 not in b"abcd" +with assertRaises(ValueError): + 350 in b"abcd" + + +# getitem +d = b"abcdefghij" + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == b"cdef" +assert d[-6:] == b"efghij" +assert d[1:8:2] == b"bdfh" +assert d[8:1:-2] == b"igec" + + +# is_xx methods + +assert bytes(b"1a23").isalnum() +assert not bytes(b"1%a23").isalnum() + +assert bytes(b"abc").isalpha() +assert not bytes(b"abc1").isalpha() + +# travis doesn't like this +# assert bytes(b'xyz').isascii() +# assert not bytes([128, 157, 32]).isascii() + +assert bytes(b"1234567890").isdigit() +assert not bytes(b"12ab").isdigit() + +l = bytes(b"lower") +b = bytes(b"UPPER") + +assert l.islower() +assert not l.isupper() +assert b.isupper() +assert not bytes(b"Super Friends").islower() + +assert bytes(b" \n\t").isspace() +assert not bytes(b"\td\n").isspace() + +assert b.isupper() +assert not b.islower() +assert l.islower() +assert not bytes(b"tuPpEr").isupper() + +assert bytes(b"Is Title Case").istitle() +assert not bytes(b"is Not title casE").istitle() + +# upper lower, capitalize +l = bytes(b"lower") +b = bytes(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytes().capitalize() == bytes() + +# hex from hex +assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +try: + bytes.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytes.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# center +assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [b"kok".center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +b"kok".center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".center(2, "a") +with assertRaises(TypeError): + b"b".center(2, b"ba") +b"kok".center(5, bytearray(b"x")) diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 06e125f21..ded02dbfa 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -1,3 +1,6 @@ +__name__ = "class" + + class Foo: def __init__(self, x): assert x == 5 @@ -14,7 +17,12 @@ foo = Foo(5) assert foo.y == Foo.y assert foo.x == 5 assert foo.square() == 25 - +assert Foo.__name__ == "Foo" +assert Foo.__qualname__ == "Foo" +assert Foo.__module__ == "class" +assert Foo.square.__name__ == "square" +assert Foo.square.__qualname__ == "Foo.square" +assert Foo.square.__module__ == "class" class Bar: """ W00t """ @@ -37,8 +45,7 @@ class Bar: assert x == 3 -# TODO: -# assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == " W00t " bar = Bar(42) @@ -117,3 +124,31 @@ assert isinstance(a, super) assert type(a) is super assert a.conjugate() == 1 + +class T1: + "test1" + +assert T1.__doc__ == "test1" + +class T2: + '''test2''' + +assert T2.__doc__ == "test2" + +class T3: + """ + test3 + """ + +assert T3.__doc__ == "\n test3\n " + +class T4: + + """test4""" + + def t1(self): + """t1""" + pass + +assert T4.__doc__ == "test4" +assert T4.t1.__doc__ == "t1" diff --git a/tests/snippets/comparisons.py b/tests/snippets/comparisons.py new file mode 100644 index 000000000..1a4597ef6 --- /dev/null +++ b/tests/snippets/comparisons.py @@ -0,0 +1,13 @@ + +assert 1 < 2 +assert 1 < 2 < 3 +assert 5 == 5 == 5 +assert (5 == 5) == True +assert 5 == 5 != 4 == 4 > 3 > 2 < 3 <= 3 != 0 == 0 + +assert not 1 > 2 +assert not 5 == 5 == True +assert not 5 == 5 != 5 == 5 +assert not 1 < 2 < 3 > 4 +assert not 1 < 2 > 3 < 4 +assert not 1 > 2 < 3 < 4 diff --git a/tests/snippets/comprehensions.py b/tests/snippets/comprehensions.py index 14c545de3..132f3cdcb 100644 --- a/tests/snippets/comprehensions.py +++ b/tests/snippets/comprehensions.py @@ -22,3 +22,8 @@ assert u['1'] == -1 y = [a+2 for a in x if a % 2] print(y) assert y == [3, 5] + +z = [(9,), (10,)] +w = [x for x, in z] +assert w == [9, 10] + diff --git a/tests/snippets/decorators.py b/tests/snippets/decorators.py index fcc196f93..2633308f1 100644 --- a/tests/snippets/decorators.py +++ b/tests/snippets/decorators.py @@ -15,6 +15,16 @@ c = add(10, 3) assert c == 14 +@logged +def add3(a, b, c=2): + return a + b + c + + +d = add3(12, 5) + +assert d == 20 + + def f(func): return lambda: 42 class A: pass a = A() diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 170a6492d..05cdd9928 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,19 +1,26 @@ -def dict_eq(d1, d2): - return (all(k in d2 and d1[k] == d2[k] for k in d1) - and all(k in d1 and d1[k] == d2[k] for k in d2)) +from testutils import assertRaises +assert dict(a=2, b=3) == {'a': 2, 'b': 3} +assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} +assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} -assert dict_eq(dict(a=2, b=3), {'a': 2, 'b': 3}) -assert dict_eq(dict({'a': 2, 'b': 3}, b=4), {'a': 2, 'b': 4}) -assert dict_eq(dict([('a', 2), ('b', 3)]), {'a': 2, 'b': 3}) +assert {} == {} +assert not {'a': 2} == {} +assert not {} == {'a': 2} +assert not {'b': 2} == {'a': 2} +assert not {'a': 4} == {'a': 2} +assert {'a': 2} == {'a': 2} + +nan = float('nan') +assert {'a': nan} == {'a': nan} a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) c['d'] = 3 c['a']['g'] = 2 -assert dict_eq(a, {'g': 2}) -assert dict_eq(b, {'a': a, 'd': 9}) +assert a == {'g': 2} +assert b == {'a': a, 'd': 9} a.clear() assert len(a) == 0 @@ -21,16 +28,141 @@ assert len(a) == 0 a = {'a': 5, 'b': 6} res = set() for value in a.values(): - res.add(value) + res.add(value) assert res == set([5,6]) count = 0 for (key, value) in a.items(): - assert a[key] == value - count += 1 + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) + res.add(key) assert res == set(['a','b']) + +# Deleted values are correctly skipped over: +x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} +del x['c'] +it = iter(x.items()) +assert ('a', 1) == next(it) +assert ('b', 2) == next(it) +assert ('d', 3) == next(it) +with assertRaises(StopIteration): + next(it) + +# Iterating a dictionary is just its keys: +assert ['a', 'b', 'd'] == list(x) + +# Iterating view captures dictionary when iterated. +data = {1: 2, 3: 4} +items = data.items() +assert list(items) == [(1, 2), (3, 4)] +data[5] = 6 +assert list(items) == [(1, 2), (3, 4), (5, 6)] + +# Values can be changed during iteration. +data = {1: 2, 3: 4} +items = iter(data.items()) +assert (1, 2) == next(items) +data[3] = "changed" +assert (3, "changed") == next(items) + +# But we can't add or delete items during iteration. +d = {} +a = iter(d.items()) +d['a'] = 2 +b = iter(d.items()) +assert ('a', 2) == next(b) +with assertRaises(RuntimeError): + next(a) +del d['a'] +with assertRaises(RuntimeError): + next(b) + +# View isn't itself an iterator. +with assertRaises(TypeError): + next(data.keys()) + +assert len(data.keys()) == 2 + +x = {} +x[1] = 1 +assert x[1] == 1 + +x[7] = 7 +x[2] = 2 +x[(5, 6)] = 5 + +with assertRaises(KeyError): + x["not here"] + +with assertRaises(TypeError): + x[[]] # Unhashable type. + +x["here"] = "here" +assert x.get("not here", "default") == "default" +assert x.get("here", "default") == "here" +assert x.get("not here") == None + +class LengthDict(dict): + def __getitem__(self, k): + return len(k) + +x = LengthDict() +assert type(x) == LengthDict +assert x['word'] == 4 +assert x.get('word') is None + +assert 5 == eval("a + word", LengthDict()) + + +class Squares(dict): + def __missing__(self, k): + v = k * k + self[k] = v + return v + +x = Squares() +assert x[-5] == 25 + +# An object that hashes to the same value always, and compares equal if any its values match. +class Hashable(object): + def __init__(self, *args): + self.values = args + def __hash__(self): + return 1 + def __eq__(self, other): + for x in self.values: + for y in other.values: + if x == y: + return True + return False + +x = {} +x[Hashable(1,2)] = 8 + +assert x[Hashable(1,2)] == 8 +assert x[Hashable(3,1)] == 8 + +x[Hashable(8)] = 19 +x[Hashable(19,8)] = 1 +assert x[Hashable(8)] == 1 +assert len(x) == 2 + +assert list({'a': 2, 'b': 10}) == ['a', 'b'] +x = {} +x['a'] = 2 +x['b'] = 10 +assert list(x) == ['a', 'b'] + +y = x.copy() +x['c'] = 12 +assert y == {'a': 2, 'b': 10} + +y.update({'c': 19, "d": -1, 'b': 12}) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} + +y.update(y) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 6675615c5..08e7d4b2b 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -106,3 +106,7 @@ assert_raises(OverflowError, float('inf').as_integer_ratio) assert_raises(OverflowError, float('-inf').as_integer_ratio) assert_raises(ValueError, float('nan').as_integer_ratio) +# Test special case for lexer, float starts with a dot: +a = .5 +assert a == 0.5 + diff --git a/tests/snippets/for.py b/tests/snippets/for.py index 07e138e3c..e913ac907 100644 --- a/tests/snippets/for.py +++ b/tests/snippets/for.py @@ -10,3 +10,15 @@ else: x = 3 assert x == 3 + +y = [] +for x, in [(9,), [2]]: + y.append(x) + +assert y == [9, 2], str(y) + +y = [] +for x, *z in [(9,88,'b'), [2, 'bla'], [None]*4]: + y.append(z) + +assert y == [[88, 'b'], ['bla'], [None]*3], str(y) diff --git a/tests/snippets/function.py b/tests/snippets/function.py index a69178bb7..b499c6776 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,4 +1,74 @@ + +__name__ = "function" + + def foo(): + """test""" return 42 assert foo() == 42 +assert foo.__doc__ == "test" +assert foo.__name__ == "foo" +assert foo.__qualname__ == "foo" +assert foo.__module__ == "function" + +def my_func(a,): + return a+2 + +assert my_func(2) == 4 + +def fubar(): + return 42, + +assert fubar() == (42,) + +def f1(): + + """test1""" + pass + +assert f1.__doc__ == "test1" + +def f2(): + '''test2''' + pass + +assert f2.__doc__ == "test2" + +def f3(): + """ + test3 + """ + pass + +assert f3.__doc__ == "\n test3\n " + +def f4(): + "test4" + pass + +assert f4.__doc__ == "test4" + + +def revdocstr(f): + d = f.__doc__ + d = d + 'w00t' + f.__doc__ = d + return f + +@revdocstr +def f5(): + """abc""" + +assert f5.__doc__ == 'abcw00t', f5.__doc__ + + +def f6(): + def nested(): + pass + + assert nested.__name__ == "nested" + assert nested.__qualname__ == "f6..nested" + + +f6() diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 745648f9d..b9c5a1626 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -1,21 +1,28 @@ +from testutils import assertRaises + + def sum(x, y): return x+y -# def total(a, b, c, d): -# return sum(sum(a,b), sum(c,d)) -# -# assert total(1,1,1,1) == 4 -# assert total(1,2,3,4) == 10 +def total(a, b, c, d): + return sum(sum(a,b), sum(c,d)) + + +assert total(1,1,1,1) == 4 +assert total(1,2,3,4) == 10 assert sum(1, 1) == 2 assert sum(1, 3) == 4 + def sum2y(x, y): return x+y*2 + assert sum2y(1, 1) == 3 assert sum2y(1, 3) == 7 + def va(a, b=2, *c, d, **e): assert a == 1 assert b == 22 @@ -23,22 +30,54 @@ def va(a, b=2, *c, d, **e): assert d == 1337 assert e['f'] == 42 + va(1, 22, 3, 4, d=1337, f=42) +assert va.__defaults__ == (2,) +assert va.__kwdefaults__ is None + + def va2(*args, **kwargs): assert args == (5, 4) assert len(kwargs) == 0 + va2(5, 4) x = (5, 4) va2(*x) va2(5, *x[1:]) -# def va3(x, *, b=2): -# pass -# va3(1, 2, 3, b=10) + +def va3(x, *, a, b=2, c=9): + return x + b + c + + +assert va3(1, a=1, b=10) == 20 + +with assertRaises(TypeError): + va3(1, 2, 3, a=1, b=10) + +with assertRaises(TypeError): + va3(1, b=10) + + +assert va3.__defaults__ is None +kw_defaults = va3.__kwdefaults__ +# assert va3.__kwdefaults__ == {'b': 2, 'c': 9} +assert set(kw_defaults) == {'b', 'c'} +assert kw_defaults['b'] == 2 +assert kw_defaults['c'] == 9 x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) + +# star arg after keyword args: +def fubar(x, y, obj=None): + assert x == 4 + assert y == 5 + assert obj == 6 + +rest = [4, 5] +fubar(obj=6, *rest) diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 5f64c4905..3b7dc77c3 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + r = [] @@ -35,3 +37,30 @@ r = list(g3()) # print(r) assert r == [23, 1, 2, 3, 44] +def g4(): + yield + yield 2, + +r = list(g4()) +assert r == [None, (2,)] + + +def catch_exception(): + try: + yield 1 + except ValueError: + yield 2 + yield 3 + + +g = catch_exception() +assert next(g) == 1 + +assert g.throw(ValueError, ValueError(), None) == 2 +assert next(g) == 3 + +g = catch_exception() +assert next(g) == 1 + +with assertRaises(KeyError): + assert g.throw(KeyError, KeyError(), None) == 2 diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py new file mode 100644 index 000000000..48d124210 --- /dev/null +++ b/tests/snippets/global_nonlocal.py @@ -0,0 +1,13 @@ + +# Test global and nonlocal funkyness + +a = 2 + +def b(): + global a + a = 4 + +assert a == 2 +b() +assert a == 4 + diff --git a/tests/snippets/hash.py b/tests/snippets/hash.py new file mode 100644 index 000000000..b108db705 --- /dev/null +++ b/tests/snippets/hash.py @@ -0,0 +1,23 @@ + +from testutils import assertRaises + + +class A: + pass + + +assert type(hash(None)) is int +assert type(hash(object())) is int +assert type(hash(A())) is int +assert type(hash(1)) is int +assert type(hash(1.1)) is int +assert type(hash("")) is int + +with assertRaises(TypeError): + hash({}) + +with assertRaises(TypeError): + hash(set()) + +with assertRaises(TypeError): + hash([]) diff --git a/tests/snippets/iterable.py b/tests/snippets/iterable.py new file mode 100644 index 000000000..45bb1cf4d --- /dev/null +++ b/tests/snippets/iterable.py @@ -0,0 +1,32 @@ +from testutils import assert_raises + +def test_container(x): + assert 3 in x + assert 4 not in x + assert list(x) == list(iter(x)) + assert list(x) == [0, 1, 2, 3] + assert [*x] == [0, 1, 2, 3] + lst = [] + lst.extend(x) + assert lst == [0, 1, 2, 3] + +class C: + def __iter__(self): + return iter([0, 1, 2, 3]) +test_container(C()) + +class C: + def __getitem__(self, x): + return (0, 1, 2, 3)[x] # raises IndexError on x==4 +test_container(C()) + +class C: + def __getitem__(self, x): + if x > 3: + raise StopIteration + return x +test_container(C()) + +class C: pass +assert_raises(TypeError, lambda: 5 in C()) +assert_raises(TypeError, lambda: iter(C)) diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6b9839321..12d961b61 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -170,3 +170,317 @@ def f(x): return x assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] + +# __delitem__ +x = ['a', 'b', 'c'] +del x[0] +assert x == ['b', 'c'] + +x = ['a', 'b', 'c'] +del x[-1] +assert x == ['a', 'b'] + +x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[2:14:3] +assert x == y +assert x == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] +assert y == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] + +x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[-5:] +assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] + +x = list(range(12)) +del x[10:2:-2] +assert x == [0,1,2,3,5,7,9,11] + +def bad_del_1(): + del ['a', 'b']['a'] +assert_raises(TypeError, bad_del_1) + +def bad_del_2(): + del ['a', 'b'][2] +assert_raises(IndexError, bad_del_2) + +# __setitem__ + +# simple index +x = [1, 2, 3, 4, 5] +x[0] = 'a' +assert x == ['a', 2, 3, 4, 5] +x[-1] = 'b' +assert x == ['a', 2, 3, 4, 'b'] +# make sure refrences are assigned correctly +y = [] +x[1] = y +y.append(100) +assert x[1] == y +assert x[1] == [100] + +#index bounds +def set_index_out_of_bounds_high(): + x = [0, 1, 2, 3, 4] + x[5] = 'a' + +def set_index_out_of_bounds_low(): + x = [0, 1, 2, 3, 4] + x[-6] = 'a' + +assert_raises(IndexError, set_index_out_of_bounds_high) +assert_raises(IndexError, set_index_out_of_bounds_low) + +# non stepped slice index +a = list(range(10)) +x = a[:] +y = a[:] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +# replace whole list +x[:] = ['a', 'b', 'c'] +y[::1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c'] +assert x == y +# splice list start +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[:1] = ['a', 'b', 'c'] +y[0:1] = ['a', 'b', 'c'] +z[:1:1] = ['a', 'b', 'c'] +zz[0:1:1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# splice list end +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[5:] = ['a', 'b', 'c'] +y[5::1] = ['a', 'b', 'c'] +z[5:10] = ['a', 'b', 'c'] +zz[5:10:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +assert x == y +assert x == z +assert x == zz +# insert sec +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[1:1] = ['a', 'b', 'c'] +y[1:0] = ['a', 'b', 'c'] +z[1:1:1] = ['a', 'b', 'c'] +zz[1:0:1] = ['a', 'b', 'c'] +assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# same but negative indexes? +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[-1:-1] = ['a', 'b', 'c'] +y[-1:9] = ['a', 'b', 'c'] +z[-1:-1:1] = ['a', 'b', 'c'] +zz[-1:9:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +assert x == y +assert x == z +assert x == zz +# splice mid +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', 'e'] +y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +assert x == y +x = a[:] +x[3:5] = ['a'] +assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +# assign empty to non stepped empty slice does nothing +x = a[:] +y = a[:] +x[5:2] = [] +y[5:2:1] = [] +assert x == a +assert y == a +# assign empty to non stepped slice removes elems +x = a[:] +y = a[:] +x[2:8] = [] +y[2:8:1] = [] +assert x == [0, 1, 8, 9] +assert x == y +# make sure refrences are assigned correctly +yy = [] +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', yy] +y[3:5:1] = ['a', 'b', 'c', 'd', yy] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +assert x == y +yy.append(100) +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == y +assert x[7] == yy +assert x[7] == [100] +assert y[7] == yy +assert y[7] == [100] + +# no zero step +def no_zero_step_set(): + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] +assert_raises(ValueError, no_zero_step_set) + +# stepped slice index +# forward slice +x = a[:] +x[2:8:2] = ['a', 'b', 'c'] +assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e'] +x[::2] = c +y[-10::2] = c +z[0:10:2] = c +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +assert x == y +assert x == z +assert x == zz +# backward slice +x = a[:] +x[8:2:-2] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +x[::-1] = c +y[9:-11:-1] = c +z[9::-1] = c +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +assert x == y +assert x == z +assert x == zz +# step size bigger than len +x = a[:] +x[::200] = ['a'] +assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x = a[:] +x[5::200] = ['a'] +assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] + +# bad stepped slices +def stepped_slice_assign_too_big(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b', 'c', 'd'] + +assert_raises(ValueError, stepped_slice_assign_too_big) + +def stepped_slice_assign_too_small(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b'] + +assert_raises(ValueError, stepped_slice_assign_too_small) + +# must assign iter t0 slice +def must_assign_iter_to_slice(): + x = [0, 1, 2, 3, 4] + x[::2] = 42 + +assert_raises(TypeError, must_assign_iter_to_slice) + +# other iterables? +a = list(range(10)) + +# string +x = a[:] +x[3:8] = "abcdefghi" +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] + +# tuple +x = a[:] +x[3:8] = (11, 12, 13, 14, 15) +assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + +# class +# __next__ +class CIterNext: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + def __iter__(self): + return self + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + +x = list(range(10)) +x[3:8] = CIterNext() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# __iter__ yield +class CIter: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __iter__(self): + for n in self.sec: + yield n + +x = list(range(10)) +x[3:8] = CIter() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# __getitem but no __iter__ sequence +class CGetItem: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __getitem__(self, sub): + return self.sec[sub] + +x = list(range(10)) +x[3:8] = CGetItem() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# iter raises error +class CIterError: + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + +def bad_iter_assign(): + x = list(range(10)) + x[3:8] = CIterError() + +assert_raises(RuntimeError, bad_iter_assign) + +# slice assign when step or stop is -1 +a = list(range(10)) +x = a[:] +x[-1:-5:-1] = ['a', 'b', 'c', 'd'] +assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x = a[:] +x[-5:-1:-1] = [] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +# is step != 1 and start or stop of slice == -1 +x = list(range(10)) +del x[-1:-5:-1] +assert x == [0, 1, 2, 3, 4, 5] +x = list(range(10)) +del x[-5:-1:-1] diff --git a/tests/snippets/set.py b/tests/snippets/set.py index 9e49dff89..393db0754 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -49,27 +49,35 @@ assert len(a) == 0 assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) +assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) +assert set([1,2,3]).intersection([1,2]) == set([1,2]) assert set([1,2,3]) & set([4,5]) == set([]) assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) +assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) assert set([1,2,3]).difference(set([1,2])) == set([3]) assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) +assert set([1,2,3]).difference([1,2]) == set([3]) assert set([1,2,3]) - set([4,5]) == set([1,2,3]) assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) +assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) +assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -111,6 +119,8 @@ a |= set([3,4,5]) assert a == set([1,2,3,4,5]) with assertRaises(TypeError): a |= 1 +with assertRaises(TypeError): + a |= [1,2,3] a = set([1,2,3]) a.intersection_update([2,3,4,5]) @@ -122,6 +132,8 @@ a &= set([2,3,4,5]) assert a == set([2,3]) with assertRaises(TypeError): a &= 1 +with assertRaises(TypeError): + a &= [1,2,3] a = set([1,2,3]) a.difference_update([3,4,5]) @@ -133,6 +145,8 @@ a -= set([3,4,5]) assert a == set([1,2]) with assertRaises(TypeError): a -= 1 +with assertRaises(TypeError): + a -= [1,2,3] a = set([1,2,3]) a.symmetric_difference_update([3,4,5]) @@ -144,6 +158,8 @@ a ^= set([3,4,5]) assert a == set([1,2,4,5]) with assertRaises(TypeError): a ^= 1 +with assertRaises(TypeError): + a ^= [1,2,3] # frozen set @@ -181,30 +197,45 @@ assert b == a assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) +assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) +assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) +assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) assert_raises(TypeError, lambda: frozenset([[]])) +a = frozenset([1,2,3]) +b = set() +for e in a: + assert e == 1 or e == 2 or e == 3 + b.add(e) +assert a == b + # set and frozen set assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) diff --git a/tests/snippets/statements.py b/tests/snippets/statements.py new file mode 100644 index 000000000..1921363f5 --- /dev/null +++ b/tests/snippets/statements.py @@ -0,0 +1,9 @@ + +# Test several statement types + +# small ones, seperated by ';': + +if True: + 5;4; + b=4 + b; diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ed9e91c70..15fa176d8 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -2,7 +2,36 @@ import os from testutils import assert_raises -assert os.open('README.md', 0) > 0 +fd = os.open('README.md', 0) +assert fd > 0 + +os.close(fd) +assert_raises(OSError, lambda: os.read(fd, 10)) + +FNAME = "test_file_that_no_one_will_have_on_disk" +CONTENT = b"testing" +CONTENT2 = b"rustpython" +CONTENT3 = b"BOYA" + +class TestWithFile(): + def __enter__(self): + open(FNAME, "wb") + return FNAME + + def __exit__(self, exc_type, exc_val, exc_tb): + os.remove(FNAME) + + +with TestWithFile() as fname: + fd = os.open(fname, 1) + assert os.write(fd, CONTENT2) == len(CONTENT2) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) + + fd = os.open(fname, 0) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) @@ -11,3 +40,17 @@ assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) assert os.O_RDONLY == 0 assert os.O_WRONLY == 1 assert os.O_RDWR == 2 + +ENV_KEY = "TEST_ENV_VAR" +ENV_VALUE = "value" + +assert os.getenv(ENV_KEY) == None +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY, 5) == 5 +os.environ[ENV_KEY] = ENV_VALUE +assert ENV_KEY in os.environ +assert os.getenv(ENV_KEY) == ENV_VALUE +del os.environ[ENV_KEY] +os.unsetenv(ENV_KEY) +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY) == None diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 31511db11..adcc106bb 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -55,6 +55,29 @@ assert b.rstrip() == ' hallo' c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' +assert ["koki".center(i, "|") for i in range(3, 10)] == [ + "koki", + "koki", + "|koki", + "|koki|", + "||koki|", + "||koki||", + "|||koki||", +] + + +assert ["kok".center(i, "|") for i in range(2, 10)] == [ + "kok", + "kok", + "kok|", + "|kok|", + "|kok||", + "||kok||", + "||kok|||", + "|||kok|||", +] + + # assert c.isascii() assert c.index('a') == 1 assert c.rindex('l') == 3 diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py new file mode 100644 index 000000000..391932c36 --- /dev/null +++ b/tests/snippets/test_abc.py @@ -0,0 +1,39 @@ +import abc + +from testutils import assertRaises + + +class CustomInterface(abc.ABC): + @abc.abstractmethod + def a(self): + pass + + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + +# TODO raise an error if there are in any abstract methods not fulfilled +# with assertRaises(TypeError): +# CustomInterface() + + +class Concrete: + def a(self): + pass + + +CustomInterface.register(Concrete) + + +class SubConcrete(Concrete): + pass + + +assert issubclass(Concrete, CustomInterface) +assert issubclass(SubConcrete, CustomInterface) +assert not issubclass(tuple, CustomInterface) + +assert isinstance(Concrete(), CustomInterface) +assert isinstance(SubConcrete(), CustomInterface) +assert not isinstance((), CustomInterface) diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 4e3b7269a..83e00c0fe 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -1,4 +1,4 @@ - +from testutils import assertRaises try: raise BaseException() @@ -95,3 +95,75 @@ except TypeError as ex: l.append(3) print('boom', type(ex)) assert l == [1, 3] + +cause = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__cause__ == None + cause = ex + raise NameError from ex +except NameError as ex2: + assert ex2.__cause__ == cause + assert ex2.__context__ == cause + +try: + raise ZeroDivisionError from None +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +with assertRaises(TypeError): + raise ZeroDivisionError from 5 + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + +with assertRaises(NameError): + try: + raise NameError + except: + raise + +with assertRaises(RuntimeError): + raise + +context = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__context__ == None + context = ex + raise NameError +except NameError as ex2: + assert ex2.__context__ == context + assert type(ex2.__context__) == ZeroDivisionError + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__context__ == None + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + assert ex.__context__ == None + +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + pass + finally: + raise NameError +except NameError as ex2: + assert ex2.__context__ == None diff --git a/tests/snippets/tuple.py b/tests/snippets/tuple.py index d50f552dc..b7393fe00 100644 --- a/tests/snippets/tuple.py +++ b/tests/snippets/tuple.py @@ -34,3 +34,7 @@ class Foo(object): foo = Foo() assert (foo,) == (foo,) + +a = (1, 2, 3) +a += 1, +assert a == (1, 2, 3, 1) diff --git a/tests/snippets/weakrefs.py b/tests/snippets/weakrefs.py index ffc61f427..059ddd000 100644 --- a/tests/snippets/weakrefs.py +++ b/tests/snippets/weakrefs.py @@ -1,4 +1,5 @@ -from _weakref import ref +from _weakref import ref, proxy +from testutils import assert_raises class X: @@ -11,3 +12,17 @@ b = ref(a) assert callable(b) assert b() is a + +class G: + def __init__(self, h): + self.h = h + + +g = G(5) +p = proxy(g) + +assert p.h == 5 + +del g + +assert_raises(ReferenceError, lambda: p.h) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py deleted file mode 100644 index b6c8cbe42..000000000 --- a/tests/snippets/whats_left_to_implement.py +++ /dev/null @@ -1,878 +0,0 @@ -expected_methods = [] - -# TODO: using tuple could have been better -expected_methods.append({'name': 'bool', 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -], 'type': bool}) -expected_methods.append({'name': 'bytearray', 'type': bytearray, 'methods': [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'bytes', 'type': bytes, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'complex', 'type': complex, 'methods': [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', -]}) -expected_methods.append({'name': 'dict', 'type': dict, 'methods': [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', -]}) -expected_methods.append({'name': 'float','type':float,'methods':[ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__set_format__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', -]}) -expected_methods.append({'name': 'frozenset','type':frozenset, 'methods': [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', -]}) -expected_methods.append({'name': 'int', 'type':int, 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -]}) -expected_methods.append({'name': 'iter','type':iter,'methods':[ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', -]}) -expected_methods.append({'name': 'list','type':list,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', -]}) -expected_methods.append({'name': 'memoryview','type':memoryview,'methods':[ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', -]}) -expected_methods.append({'name': 'range','type':range,'methods':[ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', -]}) -expected_methods.append({'name': 'set','type':set,'methods':[ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', -]}) -expected_methods.append({'name': 'string','type':str,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' -]}) -expected_methods.append({'name': 'tuple','type':tuple, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', -]}) -expected_methods.append({'name': 'object', 'type':object, 'methods':[ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' -]}) - -not_implemented = [] - -for item in expected_methods: - for method in item['methods']: - try: - if not hasattr(item['type'], method): - not_implemented.append((item['name'], method)) - except NameError: - not_implemented.append((item['name'], method)) - -for r in not_implemented: - print(r[0], ".", r[1]) -else: - print("Not much \\o/") diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 7edbd7b7e..17bcf0dca 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -1,4 +1,3 @@ - # This is a python unittest class automatically populating with all tests # in the tests folder. @@ -11,6 +10,8 @@ import logging import subprocess import contextlib import enum +from pathlib import Path +import shutil import compile_code @@ -19,15 +20,13 @@ class _TestType(enum.Enum): functional = 1 -logger = logging.getLogger('tests') -ROOT_DIR = '..' -TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) -TEST_DIRS = { - _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), -} -CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) +logger = logging.getLogger("tests") +ROOT_DIR = ".." +TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) +TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} +CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) - +RUSTPYTHON_LIB_DIR = os.path.abspath(os.path.join(ROOT_DIR, "Lib")) @contextlib.contextmanager def pushd(path): @@ -38,12 +37,12 @@ def pushd(path): def perform_test(filename, method, test_type): - logger.info('Running %s via %s', filename, method) - if method == 'cpython': + logger.info("Running %s via %s", filename, method) + if method == "cpython": run_via_cpython(filename) - elif method == 'cpython_bytecode': + elif method == "cpython_bytecode": run_via_cpython_bytecode(filename, test_type) - elif method == 'rustpython': + elif method == "rustpython": run_via_rustpython(filename, test_type) else: raise NotImplementedError(method) @@ -57,27 +56,28 @@ def run_via_cpython(filename): def run_via_cpython_bytecode(filename, test_type): # Step1: Create bytecode file: - bytecode_filename = filename + '.bytecode' - with open(bytecode_filename, 'w') as f: + bytecode_filename = filename + ".bytecode" + with open(bytecode_filename, "w") as f: compile_code.compile_to_bytecode(filename, out_file=f) # Step2: run cpython bytecode: env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" with pushd(CPYTHON_RUNNER_DIR): - subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) + subprocess.check_call(["cargo", "run", bytecode_filename], env=env) def run_via_rustpython(filename, test_type): env = os.environ.copy() env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' + env['PYTHONPATH'] = RUSTPYTHON_LIB_DIR - target = 'release' - if env.get('CODE_COVERAGE', 'false') == 'true': - target = 'debug' - binary = os.path.abspath(os.path.join(ROOT_DIR, 'target', target, 'rustpython')) + target = "release" + if env.get("CODE_COVERAGE", "false") == "true": + target = "debug" + binary = os.path.abspath(os.path.join(ROOT_DIR, "target", target, "rustpython")) subprocess.check_call([binary, filename], env=env) @@ -85,15 +85,15 @@ def run_via_rustpython(filename, test_type): def create_test_function(cls, filename, method, test_type): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) - test_function_name = 'test_{}_'.format(method) \ - + os.path.splitext(snippet_filename)[0] \ - .replace('.', '_').replace('-', '_') + test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ + 0 + ].replace(".", "_").replace("-", "_") def test_function(self): perform_test(filename, method, test_type) if hasattr(cls, test_function_name): - raise ValueError('Duplicate test case {}'.format(test_function_name)) + raise ValueError("Duplicate test case {}".format(test_function_name)) setattr(cls, test_function_name, test_function) @@ -103,25 +103,61 @@ def populate(method): for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls + return wrapper def get_test_files(): """ Retrieve test files """ for test_type, test_dir in TEST_DIRS.items(): - for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))): + for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] - if filename.startswith('xfail_'): + if filename.startswith("xfail_"): continue yield test_type, os.path.abspath(filepath) -@populate('cpython') +def generate_slices(path): + # loop used to build slices_res.py with cpython + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + for i in [start, end, step]: + i.append(None) + + slices_res = [] + for s in start: + for e in end: + for t in step: + slices_res.append(ll[s:e:t]) + + path.write_text( + "SLICES_RES={}\nSTART= {}\nEND= {}\nSTEP= {}\nLL={}\n".format( + slices_res, start, end, step, ll + ) + ) + + +@populate("cpython") # @populate('cpython_bytecode') -@populate('rustpython') +@populate("rustpython") class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - subprocess.check_call(['cargo', 'build']) - subprocess.check_call(['cargo', 'build', '--release']) + # Here add resource files + cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + if cls.slices_resource_path.exists(): + cls.slices_resource_path.unlink() + + generate_slices(cls.slices_resource_path) + + # cargo stuff + subprocess.check_call(["cargo", "build"]) + subprocess.check_call(["cargo", "build", "--release"]) + + @classmethod + def tearDownClass(cls): + cls.slices_resource_path.unlink() diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e1b1cc9ff..ef1a9cbc3 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -26,3 +26,4 @@ caseless = "0.2.1" unicode-segmentation = "1.2.1" lazy_static = "^1.0.1" lexical = "2.0.0" +itertools = "^0.8.0" diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index ed656c0fb..f2ca1e4c1 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -18,9 +18,9 @@ use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; use crate::frame::Scope; -use crate::function::{Args, OptionalArg, PyFuncArgs}; +use crate::function::{Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -102,10 +102,8 @@ fn builtin_compile( } }; - compile::compile(vm, &source, &mode, filename.value.to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }) + compile::compile(vm, &source, &mode, filename.value.to_string()) + .map_err(|err| vm.new_syntax_error(&err)) } fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { @@ -151,10 +149,8 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else { return Err(vm.new_type_error("code argument must be str or code object".to_string())); }; @@ -181,10 +177,8 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(vm, &source, &mode, "".to_string()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })? + compile::compile(vm, &source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) { code_obj } else { @@ -667,8 +661,15 @@ fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_vars -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "__builtins__", { +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { + let ctx = &vm.ctx; + + #[cfg(target_arch = "wasm32")] + let open = vm.ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = vm.ctx.new_rustfunc(io_open); + + extend_module!(vm, module, { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -714,6 +715,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "min" => ctx.new_rustfunc(builtin_min), "object" => ctx.object(), "oct" => ctx.new_rustfunc(builtin_oct), + "open" => open, "ord" => ctx.new_rustfunc(builtin_ord), "next" => ctx.new_rustfunc(builtin_next), "pow" => ctx.new_rustfunc(builtin_pow), @@ -748,6 +750,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "NameError" => ctx.exceptions.name_error.clone(), "OverflowError" => ctx.exceptions.overflow_error.clone(), "RuntimeError" => ctx.exceptions.runtime_error.clone(), + "ReferenceError" => ctx.exceptions.reference_error.clone(), "NotImplementedError" => ctx.exceptions.not_implemented_error.clone(), "TypeError" => ctx.exceptions.type_error.clone(), "ValueError" => ctx.exceptions.value_error.clone(), @@ -758,19 +761,33 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), + + // Warnings + "Warning" => ctx.exceptions.warning.clone(), + "BytesWarning" => ctx.exceptions.bytes_warning.clone(), + "UnicodeWarning" => ctx.exceptions.unicode_warning.clone(), + "DeprecationWarning" => ctx.exceptions.deprecation_warning.clone(), + "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.clone(), + "FutureWarning" => ctx.exceptions.future_warning.clone(), + "ImportWarning" => ctx.exceptions.import_warning.clone(), + "SyntaxWarning" => ctx.exceptions.syntax_warning.clone(), + "ResourceWarning" => ctx.exceptions.resource_warning.clone(), + "RuntimeWarning" => ctx.exceptions.runtime_warning.clone(), + "UserWarning" => ctx.exceptions.user_warning.clone(), }); - - #[cfg(not(target_arch = "wasm32"))] - ctx.set_attr(&py_mod, "open", ctx.new_rustfunc(io_open)); - - py_mod } -pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { - let function = args.shift(); - let name_arg = args.shift(); - let bases = args.args.clone(); - let mut metaclass = if let Some(metaclass) = args.get_optional_kwarg("metaclass") { +pub fn builtin_build_class_( + function: PyObjectRef, + qualified_name: PyStringRef, + bases: Args, + mut kwargs: KwArgs, + vm: &VirtualMachine, +) -> PyResult { + let name = qualified_name.value.split('.').next_back().unwrap(); + let name_obj = vm.new_str(name.to_string()); + + let mut metaclass = if let Some(metaclass) = kwargs.pop_kwarg("metaclass") { PyClassRef::try_from_object(vm, metaclass)? } else { vm.get_type() @@ -784,22 +801,26 @@ pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResu } } - let bases = vm.context().new_tuple(bases); + let bases = bases.into_tuple(vm); // Prepare uses full __getattribute__ resolution chain. let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; - let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; + let namespace = vm.invoke(prepare, vec![name_obj.clone(), bases.clone()])?; let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?; let cells = vm.ctx.new_dict(); vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; + + namespace.set_item("__name__", name_obj.clone(), vm)?; + namespace.set_item("__qualname__", qualified_name.into_object(), vm)?; + let class = vm.call_method( metaclass.as_object(), "__call__", - vec![name_arg, bases, namespace.into_object()], + vec![name_obj, bases, namespace.into_object()], )?; - cells.set_item(&vm.ctx, "__class__", class.clone()); + cells.set_item("__class__", class.clone(), vm)?; Ok(class) } diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 2fe485b63..10c667890 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -31,12 +31,19 @@ pub struct CodeObject { bitflags! { pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; + const HAS_KW_ONLY_DEFAULTS = 0x02; const HAS_ANNOTATIONS = 0x04; } } pub type Label = usize; +#[derive(Debug, Clone, PartialEq)] +pub enum NameScope { + Local, + Global, +} + /// A Single bytecode instruction. #[derive(Debug, Clone, PartialEq)] pub enum Instruction { @@ -49,9 +56,11 @@ pub enum Instruction { }, LoadName { name: String, + scope: NameScope, }, StoreName { name: String, + scope: NameScope, }, DeleteName { name: String, @@ -126,6 +135,7 @@ pub enum Instruction { PopBlock, Raise { argc: usize, + in_exc: bool, }, BuildString { size: usize, @@ -321,8 +331,8 @@ impl Instruction { match self { Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)), ImportStar { name } => w!(ImportStar, name), - LoadName { name } => w!(LoadName, name), - StoreName { name } => w!(StoreName, name), + LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)), + StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)), DeleteName { name } => w!(DeleteName, name), StoreSubscript => w!(StoreSubscript), DeleteSubscript => w!(DeleteSubscript), @@ -354,7 +364,7 @@ impl Instruction { SetupWith { end } => w!(SetupWith, end), CleanupWith { end } => w!(CleanupWith, end), PopBlock => w!(PopBlock), - Raise { argc } => w!(Raise, argc), + Raise { argc, in_exc } => w!(Raise, argc, in_exc), BuildString { size } => w!(BuildString, size), BuildTuple { size, unpack } => w!(BuildTuple, size, unpack), BuildList { size, unpack } => w!(BuildList, size, unpack), diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 00bafe727..d28c10feb 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -10,17 +10,21 @@ use crate::error::CompileError; use crate::obj::objcode; use crate::obj::objcode::PyCodeRef; use crate::pyobject::PyValue; +use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope}; use crate::VirtualMachine; use num_complex::Complex64; use rustpython_parser::{ast, parser}; struct Compiler { code_object_stack: Vec, + scope_stack: Vec, nxt_label: usize, source_path: Option, current_source_location: ast::Location, + current_qualified_path: Option, in_loop: bool, in_function_def: bool, + in_exc_handler: bool, } /// Compile a given sourcecode into a bytecode object. @@ -37,15 +41,18 @@ pub fn compile( match mode { Mode::Exec => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_table) } Mode::Eval => { let statement = parser::parse_statement(source).map_err(CompileError::Parse)?; - compiler.compile_statement_eval(&statement) + let symbol_table = statements_to_symbol_table(&statement); + compiler.compile_statement_eval(&statement, symbol_table) } Mode::Single => { let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program_single(&ast) + let symbol_table = make_symbol_table(&ast); + compiler.compile_program_single(&ast, symbol_table) } }?; @@ -72,11 +79,14 @@ impl Compiler { fn new() -> Self { Compiler { code_object_stack: Vec::new(), + scope_stack: Vec::new(), nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), + current_qualified_path: None, in_loop: false, in_function_def: false, + in_exc_handler: false, } } @@ -94,11 +104,17 @@ impl Compiler { } fn pop_code_object(&mut self) -> CodeObject { + // self.scope_stack.pop().unwrap(); self.code_object_stack.pop().unwrap() } - fn compile_program(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { let size_before = self.code_object_stack.len(); + self.scope_stack.push(symbol_scope); self.compile_statements(&program.statements)?; assert!(self.code_object_stack.len() == size_before); @@ -110,7 +126,12 @@ impl Compiler { Ok(()) } - fn compile_program_single(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program_single( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_scope); for statement in &program.statements { if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; @@ -129,15 +150,19 @@ impl Compiler { // Compile statement in eval mode: fn compile_statement_eval( &mut self, - statement: &ast::LocatedStatement, + statements: &[ast::LocatedStatement], + symbol_table: SymbolScope, ) -> Result<(), CompileError> { - if let ast::Statement::Expression { ref expression } = statement.node { - self.compile_expression(expression)?; - self.emit(Instruction::ReturnValue); - Ok(()) - } else { - Err(CompileError::ExpectExpr) + self.scope_stack.push(symbol_table); + for statement in statements { + if let ast::Statement::Expression { ref expression } = statement.node { + self.compile_expression(expression)?; + } else { + return Err(CompileError::ExpectExpr); + } } + self.emit(Instruction::ReturnValue); + Ok(()) } fn compile_statements( @@ -150,6 +175,31 @@ impl Compiler { Ok(()) } + fn scope_for_name(&self, name: &str) -> bytecode::NameScope { + let role = self.lookup_name(name); + match role { + SymbolRole::Global => bytecode::NameScope::Global, + _ => bytecode::NameScope::Local, + } + } + + fn load_name(&mut self, name: &str) { + // TODO: if global, do something else! + let scope = self.scope_for_name(name); + self.emit(Instruction::LoadName { + name: name.to_string(), + scope, + }); + } + + fn store_name(&mut self, name: &str) { + let scope = self.scope_for_name(name); + self.emit(Instruction::StoreName { + name: name.to_string(), + scope, + }); + } + fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> { trace!("Compiling {:?}", statement); self.set_source_location(&statement.location); @@ -173,15 +223,14 @@ impl Compiler { name: module.clone(), symbol: symbol.clone(), }); - self.emit(Instruction::StoreName { - name: match alias { - Some(alias) => alias.clone(), - None => match symbol { - Some(symbol) => symbol.clone(), - None => module.clone(), - }, + let name = match alias { + Some(alias) => alias.clone(), + None => match symbol { + Some(symbol) => symbol.clone(), + None => module.clone(), }, - }); + }; + self.store_name(&name); } } } @@ -192,11 +241,8 @@ impl Compiler { // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::Global { names } => { - unimplemented!("global {:?}", names); - } - ast::Statement::Nonlocal { names } => { - unimplemented!("nonlocal {:?}", names); + ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => { + // Handled during symbol table construction. } ast::Statement::If { test, body, orelse } => { let end_label = self.new_label(); @@ -281,15 +327,24 @@ impl Compiler { match cause { Some(cause) => { self.compile_expression(cause)?; - self.emit(Instruction::Raise { argc: 2 }); + self.emit(Instruction::Raise { + argc: 2, + in_exc: self.in_exc_handler, + }); } None => { - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); } } } None => { - self.emit(Instruction::Raise { argc: 0 }); + self.emit(Instruction::Raise { + argc: 0, + in_exc: self.in_exc_handler, + }); } }, ast::Statement::Try { @@ -319,6 +374,7 @@ impl Compiler { self.compile_test(test, Some(end_label), None, EvalContext::Statement)?; self.emit(Instruction::LoadName { name: String::from("AssertionError"), + scope: bytecode::NameScope::Local, }); match msg { Some(e) => { @@ -333,7 +389,10 @@ impl Compiler { }); } } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 1, + in_exc: self.in_exc_handler, + }); self.set_label(end_label); } ast::Statement::Break => { @@ -353,19 +412,8 @@ impl Compiler { return Err(CompileError::InvalidReturn); } match value { - Some(e) => { - let size = e.len(); - for v in e { - self.compile_expression(v)?; - } - - // If we have more than 1 return value, make it a tuple: - if size > 1 { - self.emit(Instruction::BuildTuple { - size, - unpack: false, - }); - } + Some(v) => { + self.compile_expression(v)?; } None => { self.emit(Instruction::LoadConst { @@ -431,8 +479,8 @@ impl Compiler { name: &str, args: &ast::Parameters, ) -> Result { - let have_kwargs = !args.defaults.is_empty(); - if have_kwargs { + let have_defaults = !args.defaults.is_empty(); + if have_defaults { // Construct a tuple: let size = args.defaults.len(); for element in &args.defaults { @@ -444,6 +492,25 @@ impl Compiler { }); } + let mut num_kw_only_defaults = 0; + for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) { + if let Some(default) = default { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: kw.arg.clone(), + }, + }); + self.compile_expression(default)?; + num_kw_only_defaults += 1; + } + } + if num_kw_only_defaults > 0 { + self.emit(Instruction::BuildMap { + size: num_kw_only_defaults, + unpack: false, + }); + } + let line_number = self.get_source_line_number(); self.code_object_stack.push(CodeObject::new( args.args.iter().map(|a| a.arg.clone()).collect(), @@ -454,11 +521,15 @@ impl Compiler { line_number, name.to_string(), )); + self.enter_scope(); let mut flags = bytecode::FunctionOpArg::empty(); - if have_kwargs { + if have_defaults { flags |= bytecode::FunctionOpArg::HAS_DEFAULTS; } + if num_kw_only_defaults > 0 { + flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS; + } Ok(flags) } @@ -501,6 +572,8 @@ impl Compiler { self.emit(Instruction::Jump { target: else_label }); // except handlers: + let was_in_exc_handler = self.in_exc_handler; + self.in_exc_handler = true; self.set_label(handler_label); // Exception is on top of stack now handler_label = self.new_label(); @@ -514,6 +587,7 @@ impl Compiler { // Check exception type: self.emit(Instruction::LoadName { name: String::from("isinstance"), + scope: bytecode::NameScope::Local, }); self.emit(Instruction::Rotate { amount: 2 }); self.compile_expression(exc_type)?; @@ -528,21 +602,16 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { - self.emit(Instruction::StoreName { - name: alias.clone(), - }); - } else { - // Drop exception from top of stack: - self.emit(Instruction::Pop); + // Duplicate exception for context: + self.emit(Instruction::Duplicate); + self.store_name(alias); } - } else { - // Catch all! - // Drop exception from top of stack: - self.emit(Instruction::Pop); } // Handler code: self.compile_statements(&handler.body)?; + // Drop exception from top of stack: + self.emit(Instruction::Pop); self.emit(Instruction::Jump { target: finally_label, }); @@ -563,7 +632,12 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { + argc: 0, + in_exc: true, + }); + + self.in_exc_handler = was_in_exc_handler; // We successfully ran the try block: // else: @@ -577,7 +651,6 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - // unimplemented!(); Ok(()) } @@ -596,8 +669,18 @@ impl Compiler { let was_in_function_def = self.in_function_def; self.in_loop = false; self.in_function_def = true; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(self.create_qualified_name(name, ".")); + + self.prepare_decorators(decorator_list)?; + let mut flags = self.enter_function(name, args)?; - self.compile_statements(body)?; + + let (new_body, doc_str) = get_doc(body); + + self.compile_statements(new_body)?; // Emit None at end: self.emit(Instruction::LoadConst { @@ -605,8 +688,7 @@ impl Compiler { }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - - self.prepare_decorators(decorator_list)?; + self.leave_scope(); // Prepare type annotations: let mut num_annotations = 0; @@ -651,17 +733,18 @@ impl Compiler { }); self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); // Turn code object into function object: self.emit(Instruction::MakeFunction { flags }); + self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); + + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -677,6 +760,11 @@ impl Compiler { ) -> Result<(), CompileError> { let was_in_loop = self.in_loop; self.in_loop = false; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(qualified_name.clone()); + self.prepare_decorators(decorator_list)?; self.emit(Instruction::LoadBuildClass); let line_number = self.get_source_line_number(); @@ -689,13 +777,27 @@ impl Compiler { line_number, name.to_string(), )); - self.compile_statements(body)?; + self.enter_scope(); + + let (new_body, doc_str) = get_doc(body); + + self.emit(Instruction::LoadName { + name: "__name__".to_string(), + scope: bytecode::NameScope::Local, + }); + self.emit(Instruction::StoreName { + name: "__module__".to_string(), + scope: bytecode::NameScope::Local, + }); + self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -714,7 +816,7 @@ impl Compiler { self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -750,19 +852,38 @@ impl Compiler { }); } + self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; Ok(()) } + fn store_docstring(&mut self, doc_str: Option) { + if let Some(doc_string) = doc_str { + // Duplicate top of stack (the function or class object) + self.emit(Instruction::Duplicate); + + // Doc string value: + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } + } + fn compile_for( &mut self, target: &ast::Expression, - iter: &[ast::Expression], + iter: &ast::Expression, body: &[ast::LocatedStatement], orelse: &Option>, ) -> Result<(), CompileError> { @@ -776,9 +897,7 @@ impl Compiler { }); // The thing iterated: - for i in iter { - self.compile_expression(i)?; - } + self.compile_expression(iter)?; // Retrieve Iterator self.emit(Instruction::GetIter); @@ -806,12 +925,83 @@ impl Compiler { Ok(()) } + fn compile_chained_comparison( + &mut self, + vals: &[ast::Expression], + ops: &[ast::Comparison], + ) -> Result<(), CompileError> { + assert!(!ops.is_empty()); + assert_eq!(vals.len(), ops.len() + 1); + + let to_operator = |op: &ast::Comparison| match op { + ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, + ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, + ast::Comparison::Less => bytecode::ComparisonOperator::Less, + ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, + ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, + ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, + ast::Comparison::In => bytecode::ComparisonOperator::In, + ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, + ast::Comparison::Is => bytecode::ComparisonOperator::Is, + ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, + }; + + // a == b == c == d + // compile into (pseudocode): + // result = a == b + // if result: + // result = b == c + // if result: + // result = c == d + + // initialize lhs outside of loop + self.compile_expression(&vals[0])?; + + let break_label = self.new_label(); + let last_label = self.new_label(); + + // for all comparisons except the last (as the last one doesn't need a conditional jump) + let ops_slice = &ops[0..ops.len()]; + let vals_slice = &vals[1..ops.len()]; + for (op, val) in ops_slice.iter().zip(vals_slice.iter()) { + self.compile_expression(val)?; + // store rhs for the next comparison in chain + self.emit(Instruction::Duplicate); + self.emit(Instruction::Rotate { amount: 3 }); + + self.emit(Instruction::CompareOperation { + op: to_operator(op), + }); + + // if comparison result is false, we break with this value; if true, try the next one. + // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP) + self.emit(Instruction::Duplicate); + self.emit(Instruction::JumpIfFalse { + target: break_label, + }); + self.emit(Instruction::Pop); + } + + // handle the last comparison + self.compile_expression(vals.last().unwrap())?; + self.emit(Instruction::CompareOperation { + op: to_operator(ops.last().unwrap()), + }); + self.emit(Instruction::Jump { target: last_label }); + + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.set_label(break_label); + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::Pop); + + self.set_label(last_label); + Ok(()) + } + fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); } ast::Expression::Subscript { a, b } => { self.compile_expression(a)?; @@ -988,24 +1178,8 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Compare { a, op, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - - let i = match op { - ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, - ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, - ast::Comparison::Less => bytecode::ComparisonOperator::Less, - ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, - ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, - ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, - ast::Comparison::In => bytecode::ComparisonOperator::In, - ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, - ast::Comparison::Is => bytecode::ComparisonOperator::Is, - ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, - }; - let i = Instruction::CompareOperation { op: i }; - self.emit(i); + ast::Expression::Compare { vals, ops } => { + self.compile_chained_comparison(vals, ops)?; } ast::Expression::Number { value } => { let const_value = match value { @@ -1114,9 +1288,7 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + self.load_name(name); } ast::Expression::Lambda { args, body } => { let name = "".to_string(); @@ -1125,6 +1297,7 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1332,6 +1505,7 @@ impl Compiler { // Load iterator onto stack (passed as first argument): self.emit(Instruction::LoadName { name: String::from(".0"), + scope: bytecode::NameScope::Local, }); } else { // Evaluate iterated item: @@ -1471,6 +1645,26 @@ impl Compiler { Ok(()) } + // Scope helpers: + fn enter_scope(&mut self) { + // println!("Enter scope {:?}", self.scope_stack); + // Enter first subscope! + let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0); + self.scope_stack.push(scope); + } + + fn leave_scope(&mut self) { + // println!("Leave scope {:?}", self.scope_stack); + let scope = self.scope_stack.pop().unwrap(); + assert!(scope.sub_scopes.is_empty()); + } + + fn lookup_name(&self, name: &str) -> &SymbolRole { + // println!("Looking up {:?}", name); + let scope = self.scope_stack.last().unwrap(); + scope.lookup(name).unwrap() + } + // Low level helper functions: fn emit(&mut self, instruction: Instruction) { let location = self.current_source_location.clone(); @@ -1506,17 +1700,41 @@ impl Compiler { self.current_source_location.get_row() } + fn create_qualified_name(&self, name: &str, suffix: &str) -> String { + if let Some(ref qualified_path) = self.current_qualified_path { + format!("{}.{}{}", qualified_path, name, suffix) + } else { + format!("{}{}", name, suffix) + } + } + fn mark_generator(&mut self) { self.current_code_object().is_generator = true; } } +fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option) { + if let Some(val) = body.get(0) { + if let ast::Statement::Expression { ref expression } = val.node { + if let ast::Expression::String { ref value } = expression { + if let ast::StringGroup::Constant { ref value } = value { + if let Some((_, body_rest)) = body.split_first() { + return (body_rest, Some(value.to_string())); + } + } + } + } + } + (body, None) +} + #[cfg(test)] mod tests { use super::Compiler; use crate::bytecode::CodeObject; use crate::bytecode::Constant::*; use crate::bytecode::Instruction::*; + use crate::symboltable::make_symbol_table; use rustpython_parser::parser; fn compile_exec(source: &str) -> CodeObject { @@ -1524,7 +1742,8 @@ mod tests { compiler.source_path = Some("source_path".to_string()); compiler.push_new_code_object("".to_string()); let ast = parser::parse_program(&source.to_string()).unwrap(); - compiler.compile_program(&ast).unwrap(); + let symbol_scope = make_symbol_table(&ast); + compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 436923ea9..1f4b4eecd 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -9,19 +9,37 @@ use num_traits::ToPrimitive; /// And: http://code.activestate.com/recipes/578375/ use std::collections::HashMap; -pub struct Dict { +#[derive(Clone)] +pub struct Dict { size: usize, indices: HashMap, - entries: Vec>, + entries: Vec>>, } -struct DictEntry { +impl Default for Dict { + fn default() -> Self { + Dict { + size: 0, + indices: HashMap::new(), + entries: Vec::new(), + } + } +} + +#[derive(Clone)] +struct DictEntry { hash: usize, key: PyObjectRef, - value: PyObjectRef, + value: T, } -impl Dict { +#[derive(Debug)] +pub struct DictSize { + size: usize, + entries_size: usize, +} + +impl Dict { pub fn new() -> Self { Dict { size: 0, @@ -31,12 +49,7 @@ impl Dict { } /// Store a key - pub fn insert( - &mut self, - vm: &VirtualMachine, - key: &PyObjectRef, - value: PyObjectRef, - ) -> PyResult<()> { + pub fn insert(&mut self, vm: &VirtualMachine, key: &PyObjectRef, value: T) -> PyResult<()> { match self.lookup(vm, key)? { LookupResult::Existing(index) => { // Update existing key @@ -75,19 +88,24 @@ impl Dict { } /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { if let Some(entry) = &self.entries[index] { - Ok(entry.value.clone()) + Ok(Some(entry.value.clone())) } else { panic!("Lookup returned invalid index into entries!"); } } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Ok(None) } } + pub fn clear(&mut self) { + self.entries.clear(); + self.indices.clear(); + self.size = 0 + } + /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { @@ -96,7 +114,7 @@ impl Dict { Ok(()) } else { let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } @@ -108,13 +126,26 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - self.entries - .iter() - .filter(|e| e.is_some()) - .map(|e| e.as_ref().unwrap()) - .map(|e| (e.key.clone(), e.value.clone())) - .collect() + pub fn size(&self) -> DictSize { + DictSize { + size: self.size, + entries_size: self.entries.len(), + } + } + + pub fn next_entry(&self, position: &mut usize) -> Option<(&PyObjectRef, &T)> { + while *position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { + *position += 1; + return Some((key, value)); + } + *position += 1; + } + None + } + + pub fn has_changed_size(&self, position: &DictSize) -> bool { + position.size != self.size || self.entries.len() != position.entries_size } /// Lookup the index for the given key. @@ -175,12 +206,8 @@ fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { } /// Invoke __eq__ on two keys -fn do_eq( - vm: &VirtualMachine, - key1: &PyObjectRef, - key2: &PyObjectRef, -) -> Result { - let result = vm._eq(key1, key2.clone())?; +fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { + let result = vm._eq(key1.clone(), key2.clone())?; Ok(objbool::get_value(&result)) } diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 6c7b3ca54..21bc9dadf 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,7 +1,5 @@ extern crate rustpython_parser; -use std::error::Error; - use crate::compile; use crate::frame::Scope; use crate::pyobject::PyResult; @@ -13,10 +11,7 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) debug!("Code object: {:?}", bytecode); vm.run_code_obj(bytecode, scope) } - Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - Err(vm.new_exception(syntax_error, err.description().to_string())) - } + Err(err) => Err(vm.new_syntax_error(&err)), } } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index fb757cd0f..58a0f929d 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -2,7 +2,7 @@ use crate::function::PyFuncArgs; use crate::obj::objsequence; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{create_type, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -13,13 +13,34 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { vm.new_str("No msg".to_string()) }; let traceback = vm.ctx.new_list(Vec::new()); - vm.ctx.set_attr(&zelf, "msg", msg); - vm.ctx.set_attr(&zelf, "__traceback__", traceback); + vm.set_attr(&zelf, "msg", msg)?; + vm.set_attr(&zelf, "__traceback__", traceback)?; Ok(vm.get_none()) } -// Print exception including traceback: +// print excption chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + let mut had_cause = false; + if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { + if !vm.get_none().is(&cause) { + had_cause = true; + print_exception(vm, &cause); + println!("\nThe above exception was the direct cause of the following exception:\n"); + } + } + if !had_cause { + if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { + if !vm.get_none().is(&context) { + print_exception(vm, &context); + println!("\nDuring handling of the above exception, another exception occurred:\n"); + } + } + } + print_exception_inner(vm, exc) +} + +// Print exception including traceback: +pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { @@ -97,12 +118,25 @@ pub struct ExceptionZoo { pub os_error: PyClassRef, pub overflow_error: PyClassRef, pub permission_error: PyClassRef, + pub reference_error: PyClassRef, pub runtime_error: PyClassRef, pub stop_iteration: PyClassRef, pub syntax_error: PyClassRef, pub type_error: PyClassRef, pub value_error: PyClassRef, pub zero_division_error: PyClassRef, + + pub warning: PyClassRef, + pub bytes_warning: PyClassRef, + pub unicode_warning: PyClassRef, + pub deprecation_warning: PyClassRef, + pub pending_deprecation_warning: PyClassRef, + pub future_warning: PyClassRef, + pub import_warning: PyClassRef, + pub syntax_warning: PyClassRef, + pub resource_warning: PyClassRef, + pub runtime_warning: PyClassRef, + pub user_warning: PyClassRef, } impl ExceptionZoo { @@ -119,6 +153,7 @@ impl ExceptionZoo { let name_error = create_type("NameError", &type_type, &exception_type); let os_error = create_type("OSError", &type_type, &exception_type); let runtime_error = create_type("RuntimeError", &type_type, &exception_type); + let reference_error = create_type("ReferenceError", &type_type, &exception_type); let stop_iteration = create_type("StopIteration", &type_type, &exception_type); let syntax_error = create_type("SyntaxError", &type_type, &exception_type); let type_error = create_type("TypeError", &type_type, &exception_type); @@ -130,6 +165,19 @@ impl ExceptionZoo { let file_not_found_error = create_type("FileNotFoundError", &type_type, &os_error); let permission_error = create_type("PermissionError", &type_type, &os_error); + let warning = create_type("Warning", &type_type, &exception_type); + let bytes_warning = create_type("BytesWarning", &type_type, &warning); + let unicode_warning = create_type("UnicodeWarning", &type_type, &warning); + let deprecation_warning = create_type("DeprecationWarning", &type_type, &warning); + let pending_deprecation_warning = + create_type("PendingDeprecationWarning", &type_type, &warning); + let future_warning = create_type("FutureWarning", &type_type, &warning); + let import_warning = create_type("ImportWarning", &type_type, &warning); + let syntax_warning = create_type("SyntaxWarning", &type_type, &warning); + let resource_warning = create_type("ResourceWarning", &type_type, &warning); + let runtime_warning = create_type("RuntimeWarning", &type_type, &warning); + let user_warning = create_type("UserWarning", &type_type, &warning); + ExceptionZoo { arithmetic_error, assertion_error, @@ -152,6 +200,18 @@ impl ExceptionZoo { type_error, value_error, zero_division_error, + warning, + bytes_warning, + unicode_warning, + deprecation_warning, + pending_deprecation_warning, + future_warning, + import_warning, + syntax_warning, + resource_warning, + runtime_warning, + reference_error, + user_warning, } } } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 2cd080824..668558713 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -8,20 +8,22 @@ use crate::builtins; use crate::bytecode; use crate::function::PyFuncArgs; use crate::obj::objbool; -use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; -use crate::obj::objdict::{self, PyDictRef}; +use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objstr::PyString; +use crate::obj::objtuple::PyTuple; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use itertools::Itertools; /* * So a scope is a linked list of scopes. @@ -124,26 +126,28 @@ pub trait NameProtocol { fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); fn delete_name(&self, vm: &VirtualMachine, name: &str); fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); } impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } - if let Some(value) = self.globals.get_item(name) { + if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { return Some(value); } vm.get_attribute(vm.builtins.clone(), name).ok() } - fn load_cell(&self, _vm: &VirtualMachine, name: &str) -> Option { + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } @@ -151,11 +155,19 @@ impl NameProtocol for Scope { } fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(&vm.ctx, key, value) + self.get_locals().set_item(key, value, vm).unwrap(); } - fn delete_name(&self, _vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key) + fn delete_name(&self, vm: &VirtualMachine, key: &str) { + self.get_locals().del_item(key, vm).unwrap(); + } + + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { + self.globals.get_item_option(name, vm).unwrap() + } + + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.globals.set_item(name, value, vm).unwrap(); } } @@ -278,6 +290,17 @@ impl Frame { } } + pub fn throw( + &self, + vm: &VirtualMachine, + exception: PyObjectRef, + ) -> Result { + match self.unwind_exception(vm, exception) { + None => self.run(vm), + Some(exception) => Err(exception), + } + } + pub fn fetch_instruction(&self) -> &bytecode::Instruction { let ins2 = &self.code.instructions[*self.lasti.borrow()]; *self.lasti.borrow_mut() += 1; @@ -310,8 +333,14 @@ impl Frame { ref symbol, } => self.import(vm, name, symbol), bytecode::Instruction::ImportStar { ref name } => self.import_star(vm, name), - bytecode::Instruction::LoadName { ref name } => self.load_name(vm, name), - bytecode::Instruction::StoreName { ref name } => self.store_name(vm, name), + bytecode::Instruction::LoadName { + ref name, + ref scope, + } => self.load_name(vm, name, scope), + bytecode::Instruction::StoreName { + ref name, + ref scope, + } => self.store_name(vm, name, scope), bytecode::Instruction::DeleteName { ref name } => self.delete_name(vm, name), bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), @@ -382,21 +411,23 @@ impl Frame { Ok(None) } bytecode::Instruction::BuildMap { size, unpack } => { - let map_obj = vm.ctx.new_dict().into_object(); - for _x in 0..*size { - let obj = self.pop_value(); - if *unpack { + let map_obj = vm.ctx.new_dict(); + if *unpack { + for obj in self.pop_multiple(*size) { // Take all key-value pairs from the dict: - let dict_elements = objdict::get_key_value_pairs(&obj); - for (key, value) in dict_elements.iter() { - objdict::set_item(&map_obj, vm, key, value); + let dict: PyDictRef = + obj.downcast().expect("Need a dictionary to build a map."); + for (key, value) in dict { + map_obj.set_item(key, value, vm).unwrap(); } - } else { - let key = self.pop_value(); - objdict::set_item(&map_obj, vm, &key, &obj); + } + } else { + for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { + map_obj.set_item(key, value, vm).unwrap(); } } - self.push_value(map_obj); + + self.push_value(map_obj.into_object()); Ok(None) } bytecode::Instruction::BuildSlice { size } => { @@ -554,7 +585,10 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let _qualified_name = self.pop_value(); + let qualified_name = self + .pop_value() + .downcast::() + .expect("qualified name to be a string"); let code_obj = self .pop_value() .downcast() @@ -566,18 +600,44 @@ impl Frame { vm.ctx.new_dict().into_object() }; + let kw_only_defaults = + if flags.contains(bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS) { + Some( + self.pop_value().downcast::().expect( + "Stack value for keyword only defaults expected to be a dict", + ), + ) + } else { + None + }; + let defaults = if flags.contains(bytecode::FunctionOpArg::HAS_DEFAULTS) { - self.pop_value() + Some( + self.pop_value() + .downcast::() + .expect("Stack value for defaults expected to be a tuple"), + ) } else { - vm.get_none() + None }; // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm.ctx.new_function(code_obj, scope, defaults); + let obj = vm + .ctx + .new_function(code_obj, scope, defaults, kw_only_defaults); - vm.ctx.set_attr(&obj, "__annotations__", annotations); + let name = qualified_name.value.split('.').next_back().unwrap(); + vm.set_attr(&obj, "__name__", vm.new_str(name.to_string()))?; + vm.set_attr(&obj, "__qualname__", qualified_name)?; + let module = self + .scope + .globals + .get_item_option("__name__", vm)? + .unwrap_or_else(|| vm.get_none()); + vm.set_attr(&obj, "__module__", module)?; + vm.set_attr(&obj, "__annotations__", annotations)?; self.push_value(obj); Ok(None) @@ -604,11 +664,11 @@ impl Frame { } bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { - let kw_dict = self.pop_value(); - let dict_elements = objdict::get_elements(&kw_dict).clone(); - dict_elements + let kw_dict: PyDictRef = + self.pop_value().downcast().expect("Kwargs must be a dict."); + kw_dict .into_iter() - .map(|elem| (elem.0, (elem.1).1)) + .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { vec![] @@ -647,32 +707,39 @@ impl Frame { Ok(None) } - bytecode::Instruction::Raise { argc } => { + bytecode::Instruction::Raise { argc, in_exc } => { + let cause = match argc { + 2 => self.get_exception(vm, true)?, + _ => vm.get_none(), + }; let exception = match argc { - 1 => self.pop_value(), - 0 | 2 | 3 => panic!("Not implemented!"), + 0 => match in_exc { + true => self.get_exception(vm, false)?, + false => { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } + }, + 1 | 2 => self.get_exception(vm, false)?, + 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { - info!("Exception raised: {:?}", exception); - Err(exception) - } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { - if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { - let exception = vm.new_empty_exception(exception)?; - info!("Exception raised: {:?}", exception); - Err(exception) - } else { - let msg = format!( - "Can only raise BaseException derived types, not {}", - exception - ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) - } - } else { - Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) - } + let context = match argc { + 0 => vm.get_none(), // We have already got the exception, + _ => match in_exc { + true => self.get_exception(vm, false)?, + false => vm.get_none(), + }, + }; + info!( + "Exception raised: {:?} with cause: {:?} and context: {:?}", + exception, cause, context + ); + vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + vm.set_attr(&exception, vm.new_str("__context__".to_string()), context)?; + Err(exception) } bytecode::Instruction::Break => { @@ -710,9 +777,7 @@ impl Frame { Ok(None) } bytecode::Instruction::LoadBuildClass => { - let rustfunc = - PyBuiltinFunction::new(Box::new(builtins::builtin_build_class_)).into_ref(vm); - self.push_value(rustfunc.into_object()); + self.push_value(vm.ctx.new_rustfunc(builtins::builtin_build_class_)); Ok(None) } bytecode::Instruction::UnpackSequence { size } => { @@ -831,8 +896,8 @@ impl Frame { // Grab all the names from the module and put them in the context if let Some(dict) = &module.dict { - for (k, v) in dict.get_key_value_pairs().iter() { - self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + for (k, v) in dict { + self.scope.store_name(&vm, &objstr::get_value(&k), v); } } Ok(None) @@ -953,9 +1018,21 @@ impl Frame { vm.call_method(context_manager, "__exit__", args) } - fn store_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { + fn store_name( + &self, + vm: &VirtualMachine, + name: &str, + name_scope: &bytecode::NameScope, + ) -> FrameResult { let obj = self.pop_value(); - self.scope.store_name(&vm, name, obj); + match name_scope { + bytecode::NameScope::Global => { + self.scope.store_global(vm, name, obj); + } + bytecode::NameScope::Local => { + self.scope.store_name(&vm, name, obj); + } + } Ok(None) } @@ -964,37 +1041,43 @@ impl Frame { Ok(None) } - fn load_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - match self.scope.load_name(&vm, name) { - Some(value) => { - self.push_value(value); - Ok(None) - } + fn load_name( + &self, + vm: &VirtualMachine, + name: &str, + name_scope: &bytecode::NameScope, + ) -> FrameResult { + let optional_value = match name_scope { + bytecode::NameScope::Global => self.scope.load_global(vm, name), + bytecode::NameScope::Local => self.scope.load_name(&vm, name), + }; + + let value = match optional_value { + Some(value) => value, None => { let name_error_type = vm.ctx.exceptions.name_error.clone(); let msg = format!("name '{}' is not defined", name); let name_error = vm.new_exception(name_error_type, msg); - Err(name_error) + return Err(name_error); } - } - } + }; - fn subscript(&self, vm: &VirtualMachine, a: PyObjectRef, b: PyObjectRef) -> PyResult { - vm.call_method(&a, "__getitem__", vec![b]) + self.push_value(value); + Ok(None) } fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); let value = self.pop_value(); - vm.call_method(&obj, "__setitem__", vec![idx, value])?; + obj.set_item(idx, value, vm)?; Ok(None) } fn execute_delete_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); - vm.call_method(&obj, "__delitem__", vec![idx])?; + obj.del_item(idx, vm)?; Ok(None) } @@ -1029,7 +1112,7 @@ impl Frame { bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), // TODO: Subscript should probably have its own op bytecode::BinaryOperator::Subscript if inplace => unreachable!(), - bytecode::BinaryOperator::Subscript => self.subscript(vm, a_ref, b_ref), + bytecode::BinaryOperator::Subscript => a_ref.get_item(b_ref, vm), bytecode::BinaryOperator::Modulo if inplace => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::Modulo => vm._mod(a_ref, b_ref), bytecode::BinaryOperator::Lshift if inplace => vm._ilshift(a_ref, b_ref), @@ -1067,36 +1150,14 @@ impl Frame { a.get_id() } - // https://docs.python.org/3/reference/expressions.html#membership-test-operations - fn _membership( - &self, - vm: &VirtualMachine, - needle: PyObjectRef, - haystack: &PyObjectRef, - ) -> PyResult { - vm.call_method(&haystack, "__contains__", vec![needle]) - // TODO: implement __iter__ and __getitem__ cases when __contains__ is - // not implemented. - } - fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(found), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(objbool::boolval(vm, found)?)) } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - haystack.class().name - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(!objbool::boolval(vm, found)?)) } fn _is(&self, a: PyObjectRef, b: PyObjectRef) -> bool { @@ -1201,6 +1262,28 @@ impl Frame { let stack = self.stack.borrow_mut(); stack[stack.len() - depth - 1].clone() } + + fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { + let exception = self.pop_value(); + if none_allowed && vm.get_none().is(&exception) + || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) + { + Ok(exception) + } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exception)?; + Ok(exception) + } else { + let msg = format!( + "Can only raise BaseException derived types, not {}", + exception + ); + Err(vm.new_type_error(msg)) + } + } else { + Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) + } + } } impl fmt::Debug for Frame { @@ -1224,8 +1307,8 @@ impl fmt::Debug for Frame { .map(|elem| format!("\n > {:?}", elem)) .collect::(); let dict = self.scope.get_locals(); - let local_str = objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) - .iter() + let local_str = dict + .into_iter() .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) .collect::(); write!( diff --git a/vm/src/function.rs b/vm/src/function.rs index 921534549..a72ead8f4 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -3,7 +3,9 @@ use std::mem; use std::ops::RangeInclusive; use crate::obj::objtype::{isinstance, PyClassRef}; -use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use crate::pyobject::{ + IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use self::OptionalArg::*; @@ -249,6 +251,12 @@ pub trait FromArgs: Sized { /// an appropriate FromArgs implementation must be created. pub struct KwArgs(HashMap); +impl KwArgs { + pub fn pop_kwarg(&mut self, name: &str) -> Option { + self.0.remove(name) + } +} + impl FromArgs for KwArgs where T: TryFromObject, @@ -279,8 +287,16 @@ impl IntoIterator for KwArgs { /// /// `Args` optionally accepts a generic type parameter to allow type checks /// or conversions of each argument. +#[derive(Clone)] pub struct Args(Vec); +impl Args> { + pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_tuple(self.0.into_iter().map(PyRef::into_object).collect()) + } +} + impl FromArgs for Args where T: TryFromObject, @@ -410,6 +426,7 @@ tuple_from_py_func_args!(A, B); tuple_from_py_func_args!(A, B, C); tuple_from_py_func_args!(A, B, C, D); tuple_from_py_func_args!(A, B, C, D, E); +tuple_from_py_func_args!(A, B, C, D, E, F); /// A built-in Python function. pub type PyNativeFunc = Box PyResult + 'static>; @@ -441,11 +458,8 @@ where } } -impl IntoPyNativeFunc for PyNativeFunc { - fn into_func(self) -> PyNativeFunc { - self - } -} +pub struct OwnedParam(std::marker::PhantomData); +pub struct RefParam(std::marker::PhantomData); // This is the "magic" that allows rust functions of varying signatures to // generate native python functions. @@ -453,11 +467,10 @@ impl IntoPyNativeFunc for PyNativeFunc { // Note that this could be done without a macro - it is simply to avoid repetition. macro_rules! into_py_native_func_tuple { ($(($n:tt, $T:ident)),*) => { - impl IntoPyNativeFunc<($($T,)*), R> for F + impl IntoPyNativeFunc<($(OwnedParam<$T>,)*), R> for F where F: Fn($($T,)* &VirtualMachine) -> R + 'static, $($T: FromArgs,)* - ($($T,)*): FromArgs, R: IntoPyObject, { fn into_func(self) -> PyNativeFunc { @@ -468,6 +481,22 @@ macro_rules! into_py_native_func_tuple { }) } } + + impl IntoPyNativeFunc<(RefParam, $(OwnedParam<$T>,)*), R> for F + where + F: Fn(&S, $($T,)* &VirtualMachine) -> R + 'static, + S: PyValue, + $($T: FromArgs,)* + R: IntoPyObject, + { + fn into_func(self) -> PyNativeFunc { + Box::new(move |vm, args| { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)* vm).into_pyobject(vm) + }) + } + } }; } diff --git a/vm/src/import.rs b/vm/src/import.rs index 936b89680..b59ce6426 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -2,20 +2,19 @@ * Import mechanics */ -use std::error::Error; use std::path::PathBuf; use crate::compile; use crate::frame::Scope; use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, PyResult}; +use crate::pyobject::{ItemProtocol, PyResult}; use crate::util; use crate::vm::VirtualMachine; fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &str) -> PyResult { // Check for Rust-native modules if let Some(module) = vm.stdlib_inits.borrow().get(module) { - return Ok(module(&vm.ctx).clone()); + return Ok(module(vm).clone()); } let notfound_error = vm.context().exceptions.module_not_found_error.clone(); @@ -25,21 +24,18 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s let file_path = find_source(vm, current_path, module) .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), e.description().to_string()))?; + .map_err(|e| vm.new_exception(import_error.clone(), e.to_string()))?; let code_obj = compile::compile( vm, &source, &compile::Mode::Exec, file_path.to_str().unwrap().to_string(), ) - .map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.description().to_string()) - })?; + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj); let attrs = vm.ctx.new_dict(); - attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); + attrs.set_item("__name__", vm.new_str(module.to_string()), vm)?; vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; Ok(vm.ctx.new_module(module, attrs)) } @@ -47,11 +43,11 @@ fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &s pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { // First, see if we already loaded the module: let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; - if let Some(module) = sys_modules.get_item(module_name) { + if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { return Ok(module); } let module = import_uncached_module(vm, current_path, module_name)?; - sys_modules.set_item(&vm.ctx, module_name, module.clone()); + sys_modules.set_item(module_name, module.clone(), vm)?; Ok(module) } @@ -64,12 +60,13 @@ fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result paths.insert(0, current_path); + let rel_name = name.replace('.', "/"); let suffixes = [".py", "/__init__.py"]; let mut file_paths = vec![]; for path in paths { for suffix in suffixes.iter() { let mut file_path = path.clone(); - file_path.push(format!("{}{}", name, suffix)); + file_path.push(format!("{}{}", rel_name, suffix)); file_paths.push(file_path); } } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 3892fe2a4..8b14454fe 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -32,6 +32,10 @@ extern crate rustpython_parser; #[macro_use] extern crate rustpython_derive; +extern crate self as rustpython_vm; + +pub use rustpython_derive::*; + //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; @@ -42,6 +46,7 @@ pub mod macros; mod builtins; pub mod bytecode; pub mod compile; +mod dictdatatype; pub mod error; pub mod eval; mod exceptions; @@ -52,6 +57,7 @@ pub mod import; pub mod obj; pub mod pyobject; pub mod stdlib; +mod symboltable; mod sysmodule; mod traceback; pub mod util; diff --git a/vm/src/macros.rs b/vm/src/macros.rs index a63186797..333673985 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -9,7 +9,7 @@ macro_rules! replace_expr { #[macro_export] macro_rules! count_tts { - ($($tts:tt)*) => {0usize $(+ replace_expr!($tts 1usize))*}; + ($($tts:tt)*) => {0usize $(+ $crate::replace_expr!($tts 1usize))*}; } #[macro_export] @@ -20,6 +20,8 @@ macro_rules! type_check { let arg = &$args.args[$arg_count]; if !$crate::obj::objtype::isinstance(arg, &expected_type) { + use $crate::pyobject::TypeProtocol; + let arg_typ = arg.class(); let expected_type_name = $vm.to_pystr(&expected_type)?; let actual_type = $vm.to_pystr(&arg_typ)?; @@ -45,14 +47,14 @@ macro_rules! arg_check { } }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*] ) => { - arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); + $crate::arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*], optional=[$( ($optional_arg_name:ident, $optional_arg_type:expr) ),*] ) => { let mut arg_count = 0; // use macro magic to compile-time count number of required and optional arguments - let minimum_arg_count = count_tts!($($arg_name)*); - let maximum_arg_count = minimum_arg_count + count_tts!($($optional_arg_name)*); + let minimum_arg_count = $crate::count_tts!($($arg_name)*); + let maximum_arg_count = minimum_arg_count + $crate::count_tts!($($optional_arg_name)*); // verify that the number of given arguments is right if $args.args.len() < minimum_arg_count || $args.args.len() > maximum_arg_count { @@ -72,7 +74,7 @@ macro_rules! arg_check { // check if the type matches. If not, return with error // assign the arg to a variable $( - type_check!($vm, $args, arg_count, $arg_name, $arg_type); + $crate::type_check!($vm, $args, arg_count, $arg_name, $arg_type); let $arg_name = &$args.args[arg_count]; #[allow(unused_assignments)] { @@ -85,7 +87,7 @@ macro_rules! arg_check { // assign the arg to a variable $( let $optional_arg_name = if arg_count < $args.args.len() { - type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); + $crate::type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); let ret = Some(&$args.args[arg_count]); #[allow(unused_assignments)] { @@ -114,14 +116,21 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => { - { - let py_mod = $ctx.new_module($module_name, $ctx.new_dict()); - $( - $ctx.set_attr(&py_mod, $name, $value); - )* - py_mod - } + ( $vm:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let module = $vm.ctx.new_module($module_name, $vm.ctx.new_dict()); + $( + $vm.set_attr(&module, $name, $value).unwrap(); + )* + module + }}; +} + +#[macro_export] +macro_rules! extend_module { + ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)* }) => { + $( + $vm.set_attr(&$module, $name, $value).unwrap(); + )* } } @@ -131,7 +140,7 @@ macro_rules! py_class { { let py_class = $ctx.new_class($class_name, $class_base); $( - $ctx.set_attr(&py_class, $name, $value); + py_class.set_str_attr($name, $value); )* py_class } @@ -143,7 +152,93 @@ macro_rules! extend_class { ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { let class = $class; $( - $ctx.set_attr(class, $name, $value); + class.set_str_attr($name, $value); )* } } + +/// Macro to match on the built-in class of a Python object. +/// +/// Like `match`, `match_class!` must be exhaustive, so a default arm with +/// the uncasted object is required. +/// +/// # Examples +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// assert_eq!( +/// "int", +/// match_class!(obj.clone(), +/// PyInt => "int", +/// PyFloat => "float", +/// _ => "neither", +/// ) +/// ); +/// +/// ``` +/// +/// With a binding to the downcasted type: +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// +/// let int_value = match_class!(obj, +/// i @ PyInt => i.as_bigint().clone(), +/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), +/// obj => panic!("non-numeric object {}", obj), +/// ); +/// +/// assert!(int_value.is_zero()); +/// ``` +#[macro_export] +macro_rules! match_class { + // The default arm. + ($obj:expr, _ => $default:expr $(,)?) => { + $default + }; + + // The default arm, binding the original object to the specified identifier. + ($obj:expr, $binding:ident => $default:expr $(,)?) => {{ + let $binding = $obj; + $default + }}; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier. + ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { + match $obj.downcast::<$class>() { + Ok($binding) => $expr, + Err(_obj) => $crate::match_class!(_obj, $($rest)*), + } + }; + + // An arm taken when the object is an instance of the specified built-in + // class. + ($obj:expr, $class:ty => $expr:expr, $($rest:tt)*) => { + if $obj.payload_is::<$class>() { + $expr + } else { + $crate::match_class!($obj, $($rest)*) + } + }; +} diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index 0419d827f..bfe89fd15 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -3,6 +3,7 @@ pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; +pub mod objbyteinner; pub mod objbytes; pub mod objclassmethod; pub mod objcode; @@ -33,5 +34,6 @@ pub mod objstr; pub mod objsuper; pub mod objtuple; pub mod objtype; +pub mod objweakproxy; pub mod objweakref; pub mod objzip; diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index d4ef4b0fb..724c8bc5a 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -1,9 +1,7 @@ use num_traits::Zero; use crate::function::PyFuncArgs; -use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; use super::objint::PyInt; diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index c609fdb2f..f6244399b 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,6 +1,6 @@ //! Implementation of the python bytearray object. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::fmt::Write; use std::ops::{Deref, DerefMut}; @@ -11,6 +11,7 @@ use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint; +use super::objiter; use super::objtype::PyClassRef; #[derive(Debug)] @@ -67,6 +68,7 @@ pub fn init(context: &PyContext) { "__eq__" => context.new_rustfunc(PyByteArrayRef::eq), "__len__" => context.new_rustfunc(PyByteArrayRef::len), "__repr__" => context.new_rustfunc(PyByteArrayRef::repr), + "__iter__" => context.new_rustfunc(PyByteArrayRef::iter), "clear" => context.new_rustfunc(PyByteArrayRef::clear), "isalnum" => context.new_rustfunc(PyByteArrayRef::isalnum), "isalpha" => context.new_rustfunc(PyByteArrayRef::isalpha), @@ -80,6 +82,12 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyByteArrayRef::pop), "upper" => context.new_rustfunc(PyByteArrayRef::upper) }); + + let bytearrayiterator_type = &context.bytearrayiterator_type; + extend_class!(context, bytearrayiterator_type, { + "__next__" => context.new_rustfunc(PyByteArrayIteratorRef::next), + "__iter__" => context.new_rustfunc(PyByteArrayIteratorRef::iter), + }); } fn bytearray_new( @@ -225,6 +233,13 @@ impl PyByteArrayRef { value: RefCell::new(bytes), } } + + fn iter(self, _vm: &VirtualMachine) -> PyByteArrayIterator { + PyByteArrayIterator { + position: Cell::new(0), + bytearray: self, + } + } } // helper function for istitle @@ -266,3 +281,33 @@ mod tests { assert_eq!(&to_hex(&[11u8, 222u8]), "\\x0b\\xde"); } } + +#[derive(Debug)] +pub struct PyByteArrayIterator { + position: Cell, + bytearray: PyByteArrayRef, +} + +impl PyValue for PyByteArrayIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytearrayiterator_type() + } +} + +type PyByteArrayIteratorRef = PyRef; + +impl PyByteArrayIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytearray.value.borrow().len() { + let ret = self.bytearray.value.borrow()[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs new file mode 100644 index 000000000..d0a63633a --- /dev/null +++ b/vm/src/obj/objbyteinner.rs @@ -0,0 +1,397 @@ +use crate::pyobject::PyObjectRef; +use num_bigint::BigInt; + +use crate::function::OptionalArg; + +use crate::vm::VirtualMachine; + +use crate::pyobject::{PyResult, TypeProtocol}; + +use crate::obj::objstr::PyString; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::objint; +use super::objsequence::PySliceableSequence; +use crate::obj::objint::PyInt; +use num_integer::Integer; +use num_traits::ToPrimitive; + +use super::objbytearray::{get_value as get_value_bytearray, PyByteArray}; +use super::objbytes::PyBytes; + +#[derive(Debug, Default, Clone)] +pub struct PyByteInner { + pub elements: Vec, +} + +impl PyByteInner { + pub fn new( + val_option: OptionalArg, + enc_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // First handle bytes(string, encoding[, errors]) + if let OptionalArg::Present(enc) = enc_option { + if let OptionalArg::Present(eval) = val_option { + if let Ok(input) = eval.downcast::() { + if let Ok(encoding) = enc.clone().downcast::() { + if &encoding.value.to_lowercase() == "utf8" + || &encoding.value.to_lowercase() == "utf-8" + // TODO: different encoding + { + return Ok(PyByteInner { + elements: input.value.as_bytes().to_vec(), + }); + } else { + return Err( + vm.new_value_error(format!("unknown encoding: {}", encoding.value)), //should be lookup error + ); + } + } else { + return Err(vm.new_type_error(format!( + "bytes() argument 2 must be str, not {}", + enc.class().name + ))); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + // Only one argument + } else { + let value = if let OptionalArg::Present(ival) = val_option { + match_class!(ival.clone(), + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); + Ok(vec![0; size])}, + _l @ PyString=> {return Err(vm.new_type_error("string argument without an encoding".to_string()));}, + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)))}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => Ok(PyByteInner { elements: val }), + Err(err) => Err(err), + } + } + } + + pub fn repr(&self) -> PyResult { + let mut res = String::with_capacity(self.elements.len()); + for i in self.elements.iter() { + match i { + 0..=8 => res.push_str(&format!("\\x0{}", i)), + 9 => res.push_str("\\t"), + 10 => res.push_str("\\n"), + 13 => res.push_str("\\r"), + 32..=126 => res.push(*(i) as char), + _ => res.push_str(&format!("\\x{:x}", i)), + } + } + Ok(res) + } + + pub fn len(&self) -> usize { + self.elements.len() + } + + pub fn is_empty(&self) -> bool { + self.elements.len() == 0 + } + + pub fn eq(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements == other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn ge(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements >= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn le(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements <= other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn gt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements > other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn lt(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + if self.elements < other.elements { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + + pub fn hash(&self) -> usize { + let mut hasher = DefaultHasher::new(); + self.elements.hash(&mut hasher); + hasher.finish() as usize + } + + pub fn add(&self, other: &PyByteInner, _vm: &VirtualMachine) -> Vec { + let elements: Vec = self + .elements + .iter() + .chain(other.elements.iter()) + .cloned() + .collect(); + elements + } + + pub fn contains_bytes(&self, other: &PyByteInner, vm: &VirtualMachine) -> PyResult { + for (n, i) in self.elements.iter().enumerate() { + if n + other.len() <= self.len() + && *i == other.elements[0] + && &self.elements[n..n + other.len()] == other.elements.as_slice() + { + return Ok(vm.new_bool(true)); + } + } + Ok(vm.new_bool(false)) + } + + pub fn contains_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(int) = int.as_bigint().to_u8() { + if self.elements.contains(&int) { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } else { + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } + } + + pub fn getitem_int(&self, int: &PyInt, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + Ok(vm.new_int(self.elements[idx])) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + pub fn getitem_slice(&self, slice: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice).unwrap())) + } + + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_alphanumeric()), + )) + } + + pub fn isalpha(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_alphabetic()), + )) + } + + pub fn isascii(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_ascii()), + )) + } + + pub fn isdigit(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_digit(10)), + )) + } + + pub fn islower(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_lowercase()), + )) + } + + pub fn isspace(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_whitespace()), + )) + } + + pub fn isupper(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_uppercase()), + )) + } + + pub fn istitle(&self, vm: &VirtualMachine) -> PyResult { + if self.elements.is_empty() { + return Ok(vm.new_bool(false)); + } + + let mut iter = self.elements.iter().peekable(); + let mut prev_cased = false; + + while let Some(c) = iter.next() { + let current = char::from(*c); + let next = if let Some(k) = iter.peek() { + char::from(**k) + } else if current.is_uppercase() { + return Ok(vm.new_bool(!prev_cased)); + } else { + return Ok(vm.new_bool(prev_cased)); + }; + + let is_cased = current.to_uppercase().next().unwrap() != current + || current.to_lowercase().next().unwrap() != current; + if (is_cased && next.is_uppercase() && !prev_cased) + || (!is_cased && next.is_lowercase()) + { + return Ok(vm.new_bool(false)); + } + + prev_cased = is_cased; + } + + Ok(vm.new_bool(true)) + } + + pub fn lower(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_lowercase() + } + + pub fn upper(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_uppercase() + } + + pub fn capitalize(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::new(); + if let Some((first, second)) = self.elements.split_first() { + new.push(first.to_ascii_uppercase()); + second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); + } + new + } + + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { + let bla = self + .elements + .iter() + .map(|x| format!("{:02x}", x)) + .collect::(); + Ok(vm.ctx.new_str(bla)) + } + + pub fn fromhex(string: String, vm: &VirtualMachine) -> Result, PyObjectRef> { + // first check for invalid character + for (i, c) in string.char_indices() { + if !c.is_digit(16) && !c.is_whitespace() { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + i + ))); + } + } + + // strip white spaces + let stripped = string.split_whitespace().collect::(); + + // Hex is evaluated on 2 digits + if stripped.len() % 2 != 0 { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + stripped.len() - 1 + ))); + } + + // parse even string + Ok(stripped + .chars() + .collect::>() + .chunks(2) + .map(|x| x.to_vec().iter().collect::()) + .map(|x| u8::from_str_radix(&x, 16).unwrap()) + .collect::>()) + } + + pub fn center(&self, width: &BigInt, fillbyte: u8, _vm: &VirtualMachine) -> Vec { + let width = width.to_usize().unwrap(); + + // adjust right et left side + if width <= self.len() { + return self.elements.clone(); + } + let diff: usize = width - self.len(); + let mut ln: usize = diff / 2; + let mut rn: usize = ln; + + if diff.is_odd() && self.len() % 2 == 0 { + ln += 1 + } + + if diff.is_odd() && self.len() % 2 != 0 { + rn += 1 + } + + // merge all + let mut res = vec![fillbyte; ln]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; rn][..]); + + res + } +} + +pub fn is_byte(obj: &PyObjectRef) -> Option> { + match_class!(obj.clone(), + + i @ PyBytes => Some(i.get_value().to_vec()), + j @ PyByteArray => Some(get_value_bytearray(&j.as_object()).to_vec()), + _ => None) +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 8d64cc507..0556de053 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,27 +1,42 @@ -use std::cell::Cell; -use std::hash::{Hash, Hasher}; +use crate::obj::objint::PyInt; +use crate::obj::objstr::PyString; +use crate::vm::VirtualMachine; +use core::cell::Cell; use std::ops::Deref; -use num_traits::ToPrimitive; +use crate::function::OptionalArg; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; -use crate::vm::VirtualMachine; +use super::objbyteinner::{is_byte, PyByteInner}; +use super::objiter; +use super::objslice::PySlice; +use super::objtype::PyClassRef; -use super::objint; -use super::objtype::{self, PyClassRef}; - -#[derive(Debug)] +/// "bytes(iterable_of_ints) -> bytes\n\ +/// bytes(string, encoding[, errors]) -> bytes\n\ +/// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ +/// bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ +/// bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytes")] +#[derive(Clone, Debug)] pub struct PyBytes { - value: Vec, + inner: PyByteInner, } -type PyBytesRef = PyRef; +pub type PyBytesRef = PyRef; impl PyBytes { - pub fn new(data: Vec) -> Self { - PyBytes { value: data } + pub fn new(elements: Vec) -> Self { + PyBytes { + inner: PyByteInner { elements }, + } + } + + pub fn get_value(&self) -> &[u8] { + &self.inner.elements } } @@ -29,7 +44,7 @@ impl Deref for PyBytes { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.value + &self.inner.elements } } @@ -39,165 +54,249 @@ impl PyValue for PyBytes { } } -// Binary data support +pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { + &obj.payload::().unwrap().inner.elements +} -// Fill bytes class methods: pub fn init(context: &PyContext) { - let bytes_type = context.bytes_type.as_object(); - - let bytes_doc = - "bytes(iterable_of_ints) -> bytes\n\ - bytes(string, encoding[, errors]) -> bytes\n\ - bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ - bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ - bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - any object implementing the buffer API.\n \ - - an integer"; - + PyBytesRef::extend_class(context, &context.bytes_type); + let bytes_type = &context.bytes_type; extend_class!(context, bytes_type, { - "__eq__" => context.new_rustfunc(bytes_eq), - "__lt__" => context.new_rustfunc(bytes_lt), - "__le__" => context.new_rustfunc(bytes_le), - "__gt__" => context.new_rustfunc(bytes_gt), - "__ge__" => context.new_rustfunc(bytes_ge), - "__hash__" => context.new_rustfunc(bytes_hash), - "__new__" => context.new_rustfunc(bytes_new), - "__repr__" => context.new_rustfunc(bytes_repr), - "__len__" => context.new_rustfunc(bytes_len), - "__iter__" => context.new_rustfunc(bytes_iter), - "__doc__" => context.new_str(bytes_doc.to_string()) + "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), + }); + let bytesiterator_type = &context.bytesiterator_type; + extend_class!(context, bytesiterator_type, { + "__next__" => context.new_rustfunc(PyBytesIteratorRef::next), + "__iter__" => context.new_rustfunc(PyBytesIteratorRef::iter), }); } -fn bytes_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); +#[pyimpl] +impl PyBytesRef { + #[pymethod(name = "__new__")] + fn bytes_new( + cls: PyClassRef, + val_option: OptionalArg, + enc_option: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyBytes { + inner: PyByteInner::new(val_option, enc_option, vm)?, } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; + .into_ref_with_type(vm, cls) + } - PyBytes::new(value).into_ref_with_type(vm, cls) -} + #[pymethod(name = "__repr__")] + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("b'{}'", self.inner.repr()?))) + } -fn bytes_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + #[pymethod(name = "__len__")] + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.eq(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } -fn bytes_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.ge(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.le(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.gt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => self.inner.lt(&bytes.inner, vm), + _ => Ok(vm.ctx.not_implemented())) + } + #[pymethod(name = "__hash__")] + fn hash(self, _vm: &VirtualMachine) -> usize { + self.inner.hash() + } - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__iter__")] + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { + position: Cell::new(0), + bytes: self, + } + } -fn bytes_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + bytes @ PyBytes => Ok(vm.ctx.new_bytes(self.inner.add(&bytes.inner, vm))), + _ => Ok(vm.ctx.not_implemented())) + } - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__contains__")] + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(needle, + bytes @ PyBytes => self.inner.contains_bytes(&bytes.inner, vm), + int @ PyInt => self.inner.contains_int(&int, vm), + obj => Err(vm.new_type_error(format!("a bytes-like object is required, not {}", obj)))) + } -fn bytes_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(needle, + int @ PyInt => self.inner.getitem_int(&int, vm), + slice @ PySlice => self.inner.getitem_slice(slice.as_object(), vm), + obj => Err(vm.new_type_error(format!("byte indices must be integers or slices, not {}", obj)))) + } - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalnum(vm) + } -fn bytes_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalpha(vm) + } - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.isascii(vm) + } -fn bytes_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.isdigit(vm) + } - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.islower(vm) + } -fn bytes_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) -} + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.isspace(vm) + } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - &obj.payload::().unwrap().value -} + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.isupper(vm) + } -fn bytes_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) -} + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.istitle(vm) + } -fn bytes_iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { - position: Cell::new(0), - iterated_obj: obj.into_object(), + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.upper(vm))) + } + + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.capitalize(vm))) + } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.hex(vm) + } + + // #[pymethod(name = "fromhex")] + fn fromhex(string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(string, + s @ PyString => { + match PyByteInner::fromhex(s.to_string(), vm) { + Ok(x) => Ok(vm.ctx.new_bytes(x)), + Err(y) => Err(y)}}, + obj => Err(vm.new_type_error(format!("fromhex() argument must be str, not {}", obj ))) + ) + } + + #[pymethod(name = "center")] + fn center( + self, + width: PyObjectRef, + fillbyte: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let sym = if let OptionalArg::Present(v) = fillbyte { + match is_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "center() argument 2 must be a byte string of length 1, not {}", + &v + ))); + } + } + } else { + 32 // default is space + }; + + match_class!(width, + i @PyInt => Ok(vm.ctx.new_bytes(self.inner.center(i.as_bigint(), sym, vm))), + obj => {Err(vm.new_type_error(format!("{} cannot be interpreted as an integer", obj)))} + ) + } +} + +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, +} + +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } +} + +type PyBytesIteratorRef = PyRef; + +impl PyBytesIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.inner.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self } } diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index ffacfdc07..7a8d90d1c 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -9,6 +9,8 @@ pub struct PyClassMethod { pub type PyClassMethodRef = PyRef; impl PyValue for PyClassMethod { + const HAVE_DICT: bool = true; + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.classmethod_type() } diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index a26645b91..60549cd3a 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -80,8 +80,7 @@ impl PyCodeRef { } pub fn init(context: &PyContext) { - let code_type = context.code_type.as_object(); - extend_class!(context, code_type, { + extend_class!(context, &context.code_type, { "__new__" => context.new_rustfunc(PyCodeRef::new), "__repr__" => context.new_rustfunc(PyCodeRef::repr), diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 296830e43..9f88dd6a4 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,11 +1,11 @@ use num_complex::Complex64; use num_traits::ToPrimitive; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -use super::objfloat; +use super::objfloat::{self, PyFloat}; use super::objint; use super::objtype::{self, PyClassRef}; @@ -35,17 +35,17 @@ pub fn init(context: &PyContext) { This is equivalent to (real + imag*1j) where imag defaults to 0."; extend_class!(context, complex_type, { - "__abs__" => context.new_rustfunc(complex_abs), - "__add__" => context.new_rustfunc(complex_add), "__doc__" => context.new_str(complex_doc.to_string()), - "__eq__" => context.new_rustfunc(complex_eq), - "__neg__" => context.new_rustfunc(complex_neg), - "__new__" => context.new_rustfunc(complex_new), - "__radd__" => context.new_rustfunc(complex_radd), - "__repr__" => context.new_rustfunc(complex_repr), - "conjugate" => context.new_rustfunc(complex_conjugate), - "imag" => context.new_property(complex_imag), - "real" => context.new_property(complex_real) + "__abs__" => context.new_rustfunc(PyComplexRef::abs), + "__add__" => context.new_rustfunc(PyComplexRef::add), + "__eq__" => context.new_rustfunc(PyComplexRef::eq), + "__neg__" => context.new_rustfunc(PyComplexRef::neg), + "__new__" => context.new_rustfunc(PyComplexRef::new), + "__radd__" => context.new_rustfunc(PyComplexRef::radd), + "__repr__" => context.new_rustfunc(PyComplexRef::repr), + "conjugate" => context.new_rustfunc(PyComplexRef::conjugate), + "imag" => context.new_property(PyComplexRef::imag), + "real" => context.new_property(PyComplexRef::real) }); } @@ -53,128 +53,95 @@ pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -fn complex_new( - cls: PyClassRef, - real: OptionalArg, - imag: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let real = match real { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; +impl PyComplexRef { + fn new( + cls: PyClassRef, + real: OptionalArg, + imag: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let real = match real { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let imag = match imag { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; + let imag = match imag { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; - let value = Complex64::new(real, imag); - PyComplex { value }.into_ref_with_type(vm, cls) -} - -fn complex_real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) -} - -fn complex_imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) -} - -fn complex_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) -} - -fn complex_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + let value = Complex64::new(real, imag); + PyComplex { value }.into_ref_with_type(vm, cls) } -} -fn complex_radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + fn real(self, _vm: &VirtualMachine) -> PyFloat { + self.value.re.into() } -} -fn complex_conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + fn imag(self, _vm: &VirtualMachine) -> PyFloat { + self.value.im.into() + } - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) -} + fn abs(self, _vm: &VirtualMachine) -> PyFloat { + let Complex64 { im, re } = self.value; + re.hypot(im).into() + } -fn complex_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); - - let z = get_value(zelf); - - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, - None => false, + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.complex_type()) { + vm.ctx.new_complex(self.value + get_value(&other)) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) + } else { + vm.ctx.not_implemented() } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) - } else { - return Ok(vm.ctx.not_implemented()); - }; + } - Ok(vm.ctx.new_bool(result)) -} + fn radd(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + vm.ctx.new_complex(Complex64::new( + self.value.re + objint::get_value(&other).to_f64().unwrap(), + self.value.im, + )) + } else { + vm.ctx.not_implemented() + } + } -fn complex_neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) -} + fn conjugate(self, _vm: &VirtualMachine) -> PyComplex { + self.value.conj().into() + } -fn complex_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) - } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { + self.value == get_value(&other) + } else if objtype::isinstance(&other, &vm.ctx.int_type()) { + match objint::get_value(&other).to_f64() { + Some(f) => self.value.im == 0.0f64 && self.value.re == f, + None => false, + } + } else if objtype::isinstance(&other, &vm.ctx.float_type()) { + self.value.im == 0.0 && self.value.re == objfloat::get_value(&other) + } else { + return vm.ctx.not_implemented(); + }; + + vm.ctx.new_bool(result) + } + + fn neg(self, _vm: &VirtualMachine) -> PyComplex { + PyComplex::from(-self.value) + } + + fn repr(self, _vm: &VirtualMachine) -> String { + let Complex64 { re, im } = self.value; + if re == 0.0 { + format!("{}j", im) + } else { + format!("({}+{}j)", re, im) + } + } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 061c3477e..e5419e932 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,25 +1,25 @@ use std::cell::{Cell, RefCell}; -use std::collections::HashMap; use std::fmt; -use std::ops::{Deref, DerefMut}; use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - DictProtocol, PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyObjectRef, PyRef, PyResult, + PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; +use super::objbool; use super::objiter; -use super::objstr::{self, PyStringRef}; -use super::objtype; +use super::objstr; +use crate::dictdatatype; use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyClassImpl; -pub type DictContentType = HashMap; +pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { - // TODO: should be private - pub entries: RefCell, + entries: RefCell, } pub type PyDictRef = PyRef; @@ -36,107 +36,35 @@ impl PyValue for PyDict { } } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap().entries.borrow() -} - -pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut + 'a { - obj.payload::().unwrap().entries.borrow_mut() -} - -pub fn set_item( - dict: &PyObjectRef, - _vm: &VirtualMachine, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // TODO: use vm to call eventual __hash__ and __eq__methods. - let mut elements = get_mut_elements(dict); - set_item_in_content(&mut elements, needle, value); -} - -pub fn set_item_in_content( - elements: &mut DictContentType, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // XXX: Currently, we only support String keys, so we have to unwrap the - // PyObject (and ensure it is a String). - - // TODO: invoke __hash__ function here! - let needle_str = objstr::get_value(needle); - elements.insert(needle_str, (needle.clone(), value.clone())); -} - -pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = get_elements(dict); - get_key_value_pairs_from_content(&dict_elements) -} - -pub fn get_key_value_pairs_from_content( - dict_content: &DictContentType, -) -> Vec<(PyObjectRef, PyObjectRef)> { - let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_content.iter() { - let (key, obj) = pair; - pairs.push((key.clone(), obj.clone())); - } - pairs -} - -pub fn get_item(dict: &PyObjectRef, key: &PyObjectRef) -> Option { - let needle_str = objstr::get_value(key); - get_key_str(dict, &needle_str) -} - -// Special case for the case when requesting a str key from a dict: -pub fn get_key_str(dict: &PyObjectRef, key: &str) -> Option { - let elements = get_elements(dict); - content_get_key_str(&elements, key) -} - -/// Retrieve a key from dict contents: -pub fn content_get_key_str(elements: &DictContentType, key: &str) -> Option { - // TODO: let hash: usize = key; - match elements.get(key) { - Some(v) => Some(v.1.clone()), - None => None, - } -} - -pub fn contains_key_str(dict: &PyObjectRef, key: &str) -> bool { - let elements = get_elements(dict); - content_contains_key_str(&elements, key) -} - -pub fn content_contains_key_str(elements: &DictContentType, key: &str) -> bool { - // TODO: let hash: usize = key; - elements.get(key).is_some() -} - -/// Take a python dictionary and convert it to attributes. -pub fn py_dict_to_attributes(dict: &PyObjectRef) -> PyAttributes { - let mut attrs = PyAttributes::new(); - for (key, value) in get_key_value_pairs(dict) { - let key = objstr::get_value(&key); - attrs.insert(key, value); - } - attrs -} - // Python dict methods: impl PyDictRef { fn new( - _class: PyClassRef, // TODO Support subclasses of int. + class: PyClassRef, dict_obj: OptionalArg, kwargs: KwArgs, vm: &VirtualMachine, ) -> PyResult { - let dict = vm.ctx.new_dict(); + let dict = DictContentType::default(); + + let entries = RefCell::new(dict); + // it's unfortunate that we can't abstract over RefCall, as we should be able to use dict + // directly here, but that would require generic associated types + PyDictRef::merge(&entries, dict_obj, kwargs, vm)?; + + PyDict { entries }.into_ref_with_type(vm, class) + } + + fn merge( + dict: &RefCell, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { if let OptionalArg::Present(dict_obj) = dict_obj { - if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { - for (needle, value) in get_key_value_pairs(&dict_obj) { - set_item(dict.as_object(), vm, &needle, &value); + let dicted: PyResult = dict_obj.clone().downcast(); + if let Ok(dict_obj) = dicted { + for (key, value) in dict_obj { + dict.borrow_mut().insert(vm, &key, value)?; } } else { let iter = objiter::get_iter(vm, &dict_obj)?; @@ -149,36 +77,66 @@ impl PyDictRef { None => break, }; let elem_iter = objiter::get_iter(vm, &element)?; - let needle = - objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + let key = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; if objiter::get_next_object(vm, &elem_iter)?.is_some() { return Err(err(vm)); } - set_item(dict.as_object(), vm, &needle, &value); + dict.borrow_mut().insert(vm, &key, value)?; } } } - for (needle, value) in kwargs.into_iter() { - let py_needle = vm.new_str(needle); - set_item(&dict.as_object(), vm, &py_needle, &value); + let mut dict_borrowed = dict.borrow_mut(); + for (key, value) in kwargs.into_iter() { + dict_borrowed.insert(vm, &vm.new_str(key), value)?; } - Ok(dict) + Ok(()) } fn bool(self, _vm: &VirtualMachine) -> bool { !self.entries.borrow().is_empty() } + fn inner_eq(self, other: &PyDict, vm: &VirtualMachine) -> PyResult { + if other.entries.borrow().len() != self.entries.borrow().len() { + return Ok(false); + } + for (k, v1) in self { + match other.entries.borrow().get(vm, &k)? { + Some(v2) => { + if v1.is(&v2) { + continue; + } + let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; + if !value { + return Ok(false); + } + } + None => { + return Ok(false); + } + } + } + Ok(true) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + let eq = self.inner_eq(other, vm)?; + Ok(vm.ctx.new_bool(eq)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + fn len(self, _vm: &VirtualMachine) -> usize { self.entries.borrow().len() } - fn repr(self, vm: &VirtualMachine) -> PyResult { + fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { - let elements = get_key_value_pairs(self.as_object()); let mut str_parts = vec![]; - for (key, value) in elements { + for (key, value) in self { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -188,134 +146,296 @@ impl PyDictRef { } else { "{...}".to_string() }; - Ok(vm.new_str(s)) + Ok(s) } - fn contains(self, key: PyStringRef, _vm: &VirtualMachine) -> bool { - self.entries.borrow().contains_key(&key.value) + fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let key = &key.value; - // Delete the item: - let mut elements = self.entries.borrow_mut(); - match elements.remove(key) { - Some(_) => Ok(()), - None => Err(vm.new_value_error(format!("Key not found: {}", key))), - } + fn inner_delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().delete(vm, &key) } fn clear(self, _vm: &VirtualMachine) { self.entries.borrow_mut().clear() } - /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyIteratorValue { - let keys = self - .entries - .borrow() - .values() - .map(|(k, _v)| k.clone()) - .collect(); - let key_list = vm.ctx.new_list(keys); - - PyIteratorValue { - position: Cell::new(0), - iterated_obj: key_list, - } + fn iter(self, _vm: &VirtualMachine) -> PyDictKeyIterator { + PyDictKeyIterator::new(self) } - fn values(self, vm: &VirtualMachine) -> PyIteratorValue { - let values = self - .entries - .borrow() - .values() - .map(|(_k, v)| v.clone()) - .collect(); - let values_list = vm.ctx.new_list(values); - - PyIteratorValue { - position: Cell::new(0), - iterated_obj: values_list, - } + fn keys(self, _vm: &VirtualMachine) -> PyDictKeys { + PyDictKeys::new(self) } - fn items(self, vm: &VirtualMachine) -> PyIteratorValue { - let items = self - .entries - .borrow() - .values() - .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) - .collect(); - let items_list = vm.ctx.new_list(items); - - PyIteratorValue { - position: Cell::new(0), - iterated_obj: items_list, - } + fn values(self, _vm: &VirtualMachine) -> PyDictValues { + PyDictValues::new(self) } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, _vm: &VirtualMachine) { - let mut elements = self.entries.borrow_mut(); - set_item_in_content(&mut elements, &needle, &value) + fn items(self, _vm: &VirtualMachine) -> PyDictItems { + PyDictItems::new(self) } - fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { - let key = &key.value; + fn inner_setitem( + self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.entries.borrow_mut().insert(vm, &key, value) + } - // What we are looking for: - let elements = self.entries.borrow(); - if elements.contains_key(key) { - Ok(elements[key].1.clone()) - } else { - Err(vm.new_value_error(format!("Key not found: {}", key))) + fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(value) = self.entries.borrow().get(vm, &key)? { + return Ok(value); } + + if let Ok(method) = vm.get_method(self.clone().into_object(), "__missing__") { + return vm.invoke(method, vec![key]); + } + + Err(vm.new_key_error(format!("Key not found: {}", vm.to_pystr(&key)?))) } fn get( self, - key: PyStringRef, + key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine, - ) -> PyObjectRef { - // What we are looking for: - let key = &key.value; + ) -> PyResult { + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => match default { + OptionalArg::Present(value) => Ok(value), + OptionalArg::Missing => Ok(vm.ctx.none()), + }, + } + } - let elements = self.entries.borrow(); - if elements.contains_key(key) { - elements[key].1.clone() - } else { - match default { - OptionalArg::Present(value) => value, - OptionalArg::Missing => vm.ctx.none(), - } + fn copy(self, _vm: &VirtualMachine) -> PyDict { + PyDict { + entries: self.entries.clone(), + } + } + + fn update( + self, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) + } + + /// Take a python dictionary and convert it to attributes. + pub fn to_attributes(self) -> PyAttributes { + let mut attrs = PyAttributes::new(); + for (key, value) in self { + let key = objstr::get_value(&key); + attrs.insert(key, value); + } + attrs + } + + pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult { + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + + for (key, value) in attrs { + entries + .borrow_mut() + .insert(vm, &vm.ctx.new_str(key), value)?; + } + + Ok(PyDict { entries }.into_ref(vm)) + } + + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } + + pub fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().contains(vm, &key).unwrap() + } + + pub fn size(&self) -> dictdatatype::DictSize { + self.entries.borrow().size() + } +} + +impl ItemProtocol for PyDictRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + self.as_object().get_item(key, vm) + } + + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.as_object().set_item(key, value, vm) + } + + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + self.as_object().del_item(key, vm) + } +} + +// Implement IntoIterator so that we can easily iterate dictionaries from rust code. +impl IntoIterator for PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self) + } +} + +impl IntoIterator for &PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self.clone()) + } +} + +pub struct DictIter { + dict: PyDictRef, + position: usize, +} + +impl DictIter { + pub fn new(dict: PyDictRef) -> DictIter { + DictIter { dict, position: 0 } + } +} + +impl Iterator for DictIter { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option { + match self.dict.entries.borrow().next_entry(&mut self.position) { + Some((key, value)) => Some((key.clone(), value.clone())), + None => None, } } } -impl DictProtocol for PyDictRef { - fn contains_key(&self, k: &str) -> bool { - content_contains_key_str(&self.entries.borrow(), k) - } +macro_rules! dict_iterator { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { + #[pyclass(name = $class_name)] + #[derive(Debug)] + struct $name { + pub dict: PyDictRef, + } - fn get_item(&self, k: &str) -> Option { - content_get_key_str(&self.entries.borrow(), k) - } + #[pyimpl] + impl $name { + fn new(dict: PyDictRef) -> Self { + $name { dict: dict } + } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - get_key_value_pairs(self.as_object()) - } + #[pymethod(name = "__iter__")] + fn iter(&self, _vm: &VirtualMachine) -> $iter_name { + $iter_name::new(self.dict.clone()) + } - // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - let key = ctx.new_str(key.to_string()); - set_item_in_content(&mut self.entries.borrow_mut(), &key, &v); - } + #[pymethod(name = "__len__")] + fn len(&self, vm: &VirtualMachine) -> usize { + self.dict.clone().len(vm) + } + } - fn del_item(&self, key: &str) { - let mut elements = get_mut_elements(self.as_object()); - elements.remove(key).unwrap(); - } + impl PyValue for $name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$class.clone() + } + } + + #[pyclass(name = $iter_class_name)] + #[derive(Debug)] + struct $iter_name { + pub dict: PyDictRef, + pub size: dictdatatype::DictSize, + pub position: Cell, + } + + #[pyimpl] + impl $iter_name { + fn new(dict: PyDictRef) -> Self { + $iter_name { + position: Cell::new(0), + size: dict.size(), + dict, + } + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let mut position = self.position.get(); + let dict = self.dict.entries.borrow(); + if dict.has_changed_size(&self.size) { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "dictionary changed size during iteration".to_string(), + )); + } + match dict.next_entry(&mut position) { + Some((key, value)) => { + self.position.set(position); + Ok($result_fn(vm, key, value)) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + } + + impl PyValue for $iter_name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$iter_class.clone() + } + } + }; +} + +dict_iterator! { + PyDictKeys, + PyDictKeyIterator, + dictkeys_type, + dictkeyiterator_type, + "dictkeys", + "dictkeyiterator", + |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() +} + +dict_iterator! { + PyDictValues, + PyDictValueIterator, + dictvalues_type, + dictvalueiterator_type, + "dictvalues", + "dictvalueiterator", + |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() +} + +dict_iterator! { + PyDictItems, + PyDictItemIterator, + dictitems_type, + dictitemiterator_type, + "dictitems", + "dictitemiterator", + |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| + vm.ctx.new_tuple(vec![key.clone(), value.clone()]) } pub fn init(context: &PyContext) { @@ -323,16 +443,27 @@ pub fn init(context: &PyContext) { "__bool__" => context.new_rustfunc(PyDictRef::bool), "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), - "__delitem__" => context.new_rustfunc(PyDictRef::delitem), - "__getitem__" => context.new_rustfunc(PyDictRef::getitem), + "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__eq__" => context.new_rustfunc(PyDictRef::eq), + "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), - "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__setitem__" => context.new_rustfunc(PyDictRef::inner_setitem), + "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), - "keys" => context.new_rustfunc(PyDictRef::iter), + "keys" => context.new_rustfunc(PyDictRef::keys), "get" => context.new_rustfunc(PyDictRef::get), + "copy" => context.new_rustfunc(PyDictRef::copy), + "update" => context.new_rustfunc(PyDictRef::update), }); + + PyDictKeys::extend_class(context, &context.dictkeys_type); + PyDictKeyIterator::extend_class(context, &context.dictkeyiterator_type); + PyDictValues::extend_class(context, &context.dictvalues_type); + PyDictValueIterator::extend_class(context, &context.dictvalueiterator_type); + PyDictItems::extend_class(context, &context.dictitems_type); + PyDictItemIterator::extend_class(context, &context.dictitemiterator_type); } diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 8ce843352..cb188e1f1 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -1,10 +1,9 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let ellipsis_type = context.ellipsis_type.as_object(); - extend_class!(context, ellipsis_type, { + extend_class!(context, &context.ellipsis_type, { "__new__" => context.new_rustfunc(ellipsis_new), "__repr__" => context.new_rustfunc(ellipsis_repr) }); diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 85781bb7a..c94aeabbd 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -57,13 +57,17 @@ impl PyEnumerateRef { Ok(result) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let enumerate_type = &context.enumerate_type; - objiter::iter_type_init(context, enumerate_type); extend_class!(context, enumerate_type, { "__new__" => context.new_rustfunc(enumerate_new), - "__next__" => context.new_rustfunc(PyEnumerateRef::next) + "__next__" => context.new_rustfunc(PyEnumerateRef::next), + "__iter__" => context.new_rustfunc(PyEnumerateRef::iter), }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index 07c512f6d..858a0531a 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -52,13 +52,15 @@ impl PyFilterRef { } } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let filter_type = &context.filter_type; - objiter::iter_type_init(context, filter_type); - let filter_doc = "filter(function or None, iterable) --> filter object\n\n\ Return an iterator yielding those items of iterable for which function(item)\n\ @@ -67,6 +69,7 @@ pub fn init(context: &PyContext) { extend_class!(context, filter_type, { "__new__" => context.new_rustfunc(filter_new), "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(PyFilterRef::next) + "__next__" => context.new_rustfunc(PyFilterRef::next), + "__iter__" => context.new_rustfunc(PyFilterRef::iter), }); } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index ca6bad9d2..e16edf5ce 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -16,6 +16,12 @@ pub struct PyFloat { value: f64, } +impl PyFloat { + pub fn to_f64(&self) -> f64 { + self.value + } +} + impl PyValue for PyFloat { fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.float_type() @@ -34,10 +40,8 @@ impl From for PyFloat { } } -pub type PyFloatRef = PyRef; - -impl PyFloatRef { - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +impl PyFloat { + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let value = self.value; let result = if objtype::isinstance(&other, &vm.ctx.float_type()) { let other = get_value(&other); @@ -56,7 +60,7 @@ impl PyFloatRef { vm.ctx.new_bool(result) } - fn lt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn lt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 < get_value(&i2)) @@ -68,7 +72,7 @@ impl PyFloatRef { } } - fn le(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn le(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 <= get_value(&i2)) @@ -80,7 +84,7 @@ impl PyFloatRef { } } - fn gt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn gt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 > get_value(&i2)) @@ -92,7 +96,7 @@ impl PyFloatRef { } } - fn ge(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn ge(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 >= get_value(&i2)) @@ -104,11 +108,11 @@ impl PyFloatRef { } } - fn abs(self, _vm: &VirtualMachine) -> f64 { + fn abs(&self, _vm: &VirtualMachine) -> f64 { self.value.abs() } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1 + get_value(&other)) @@ -120,23 +124,23 @@ impl PyFloatRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { self.value != 0.0 } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.float_type()) || objtype::isinstance(&other, &vm.ctx.int_type()) { - let r1 = PyFloatRef::floordiv(self.clone(), other.clone(), vm)?; - let r2 = PyFloatRef::mod_(self, other, vm)?; + let r1 = self.floordiv(other.clone(), vm)?; + let r2 = self.mod_(other, vm)?; Ok(vm.ctx.new_tuple(vec![r1, r2])) } else { Ok(vm.ctx.not_implemented()) } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -195,7 +199,7 @@ impl PyFloatRef { PyFloat { value }.into_ref_with_type(vm, cls) } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -214,11 +218,11 @@ impl PyFloatRef { } } - fn neg(self, _vm: &VirtualMachine) -> f64 { + fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { vm.ctx.new_float(v1.powf(get_value(&other))) @@ -230,7 +234,7 @@ impl PyFloatRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(v1 - get_value(&other))) @@ -243,7 +247,7 @@ impl PyFloatRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type()) { Ok(vm.ctx.new_float(get_value(&other) - v1)) @@ -256,11 +260,11 @@ impl PyFloatRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -279,7 +283,7 @@ impl PyFloatRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { get_value(&other) @@ -298,7 +302,7 @@ impl PyFloatRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { let v1 = self.value; if objtype::isinstance(&other, &vm.ctx.float_type) { Ok(vm.ctx.new_float(v1 * get_value(&other))) @@ -311,16 +315,16 @@ impl PyFloatRef { } } - fn is_integer(self, _vm: &VirtualMachine) -> bool { + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf + } + + fn is_integer(&self, _vm: &VirtualMachine) -> bool { let v = self.value; (v - v.round()).abs() < std::f64::EPSILON } - fn real(self, _vm: &VirtualMachine) -> Self { - self - } - - fn as_integer_ratio(self, vm: &VirtualMachine) -> PyResult { + fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult { let value = self.value; if value.is_infinite() { return Err( @@ -338,6 +342,8 @@ impl PyFloatRef { } } +pub type PyFloatRef = PyRef; + // Retrieve inner float value: pub fn get_value(obj: &PyObjectRef) -> f64 { obj.payload::().unwrap().value @@ -361,31 +367,31 @@ pub fn init(context: &PyContext) { let float_doc = "Convert a string or number to a floating point number, if possible."; extend_class!(context, float_type, { - "__eq__" => context.new_rustfunc(PyFloatRef::eq), - "__lt__" => context.new_rustfunc(PyFloatRef::lt), - "__le__" => context.new_rustfunc(PyFloatRef::le), - "__gt__" => context.new_rustfunc(PyFloatRef::gt), - "__ge__" => context.new_rustfunc(PyFloatRef::ge), - "__abs__" => context.new_rustfunc(PyFloatRef::abs), - "__add__" => context.new_rustfunc(PyFloatRef::add), - "__radd__" => context.new_rustfunc(PyFloatRef::add), - "__bool__" => context.new_rustfunc(PyFloatRef::bool), - "__divmod__" => context.new_rustfunc(PyFloatRef::divmod), - "__floordiv__" => context.new_rustfunc(PyFloatRef::floordiv), - "__new__" => context.new_rustfunc(PyFloatRef::new_float), - "__mod__" => context.new_rustfunc(PyFloatRef::mod_), - "__neg__" => context.new_rustfunc(PyFloatRef::neg), - "__pow__" => context.new_rustfunc(PyFloatRef::pow), - "__sub__" => context.new_rustfunc(PyFloatRef::sub), - "__rsub__" => context.new_rustfunc(PyFloatRef::rsub), - "__repr__" => context.new_rustfunc(PyFloatRef::repr), + "__eq__" => context.new_rustfunc(PyFloat::eq), + "__lt__" => context.new_rustfunc(PyFloat::lt), + "__le__" => context.new_rustfunc(PyFloat::le), + "__gt__" => context.new_rustfunc(PyFloat::gt), + "__ge__" => context.new_rustfunc(PyFloat::ge), + "__abs__" => context.new_rustfunc(PyFloat::abs), + "__add__" => context.new_rustfunc(PyFloat::add), + "__radd__" => context.new_rustfunc(PyFloat::add), + "__bool__" => context.new_rustfunc(PyFloat::bool), + "__divmod__" => context.new_rustfunc(PyFloat::divmod), + "__floordiv__" => context.new_rustfunc(PyFloat::floordiv), + "__new__" => context.new_rustfunc(PyFloat::new_float), + "__mod__" => context.new_rustfunc(PyFloat::mod_), + "__neg__" => context.new_rustfunc(PyFloat::neg), + "__pow__" => context.new_rustfunc(PyFloat::pow), + "__sub__" => context.new_rustfunc(PyFloat::sub), + "__rsub__" => context.new_rustfunc(PyFloat::rsub), + "__repr__" => context.new_rustfunc(PyFloat::repr), "__doc__" => context.new_str(float_doc.to_string()), - "__truediv__" => context.new_rustfunc(PyFloatRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyFloatRef::rtruediv), - "__mul__" => context.new_rustfunc(PyFloatRef::mul), - "__rmul__" => context.new_rustfunc(PyFloatRef::mul), - "real" => context.new_property(PyFloatRef::real), - "is_integer" => context.new_rustfunc(PyFloatRef::is_integer), - "as_integer_ratio" => context.new_rustfunc(PyFloatRef::as_integer_ratio) + "__truediv__" => context.new_rustfunc(PyFloat::truediv), + "__rtruediv__" => context.new_rustfunc(PyFloat::rtruediv), + "__mul__" => context.new_rustfunc(PyFloat::mul), + "__rmul__" => context.new_rustfunc(PyFloat::mul), + "real" => context.new_property(PyFloat::real), + "is_integer" => context.new_rustfunc(PyFloat::is_integer), + "as_integer_ratio" => context.new_rustfunc(PyFloat::as_integer_ratio) }); } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index 410794696..05b66c346 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,5 +1,8 @@ use crate::frame::Scope; +use crate::function::{Args, KwArgs}; use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype::PyClassRef; use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; @@ -11,15 +14,22 @@ pub struct PyFunction { // TODO: these shouldn't be public pub code: PyCodeRef, pub scope: Scope, - pub defaults: PyObjectRef, + pub defaults: Option, + pub kw_only_defaults: Option, } impl PyFunction { - pub fn new(code: PyCodeRef, scope: Scope, defaults: PyObjectRef) -> Self { + pub fn new( + code: PyCodeRef, + scope: Scope, + defaults: Option, + kw_only_defaults: Option, + ) -> Self { PyFunction { code, scope, defaults, + kw_only_defaults, } } } @@ -31,9 +41,21 @@ impl PyValue for PyFunction { } impl PyFunctionRef { + fn call(self, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.into_object(), (&args, &kwargs)) + } + fn code(self, _vm: &VirtualMachine) -> PyCodeRef { self.code.clone() } + + fn defaults(self, _vm: &VirtualMachine) -> Option { + self.defaults.clone() + } + + fn kwdefaults(self, _vm: &VirtualMachine) -> Option { + self.kw_only_defaults.clone() + } } #[derive(Debug)] @@ -59,7 +81,10 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), - "__code__" => context.new_property(PyFunctionRef::code) + "__call__" => context.new_rustfunc(PyFunctionRef::call), + "__code__" => context.new_property(PyFunctionRef::code), + "__defaults__" => context.new_property(PyFunctionRef::defaults), + "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), }); let builtin_function_or_method_type = &context.builtin_function_or_method_type; diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 9718d29a4..2c51778fd 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -3,12 +3,13 @@ */ use crate::frame::{ExecutionResult, FrameRef}; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objtype::{isinstance, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; +#[pyclass(name = "generator")] #[derive(Debug)] pub struct PyGenerator { frame: FrameRef, @@ -20,38 +21,59 @@ impl PyValue for PyGenerator { } } -impl PyGeneratorRef { +#[pyimpl] +impl PyGenerator { pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { PyGenerator { frame }.into_ref(vm) } - fn iter(self, _vm: &VirtualMachine) -> PyGeneratorRef { - self + #[pymethod(name = "__iter__")] + fn iter(zelf: PyGeneratorRef, _vm: &VirtualMachine) -> PyGeneratorRef { + zelf } - fn next(self, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { self.send(vm.get_none(), vm) } - fn send(self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.frame.push_value(value.clone()); - match vm.run_frame(self.frame.clone())? { - ExecutionResult::Yield(value) => Ok(value), - ExecutionResult::Return(_value) => { - // Stop iteration! - let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); - Err(vm.new_exception(stop_iteration, "End of generator".to_string())) - } + let result = vm.run_frame(self.frame.clone())?; + handle_execution_result(result, vm) + } + + #[pymethod] + fn throw( + &self, + _exc_type: PyObjectRef, + exc_val: PyObjectRef, + _exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // TODO what should we do with the other parameters? CPython normalises them with + // PyErr_NormalizeException, do we want to do the same. + if !isinstance(&exc_val, &vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error("Can't throw non exception".to_string())); + } + let result = vm.frame_throw(self.frame.clone(), exc_val)?; + handle_execution_result(result, vm) + } +} + +fn handle_execution_result(result: ExecutionResult, vm: &VirtualMachine) -> PyResult { + match result { + ExecutionResult::Yield(value) => Ok(value), + ExecutionResult::Return(_value) => { + // Stop iteration! + let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); + Err(vm.new_exception(stop_iteration, "End of generator".to_string())) } } } -pub fn init(context: &PyContext) { - let generator_type = &context.generator_type; - extend_class!(context, generator_type, { - "__iter__" => context.new_rustfunc(PyGeneratorRef::iter), - "__next__" => context.new_rustfunc(PyGeneratorRef::next), - "send" => context.new_rustfunc(PyGeneratorRef::send) - }); +pub fn init(ctx: &PyContext) { + PyGenerator::extend_class(ctx, &ctx.generator_type); } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 08175dc07..20f5de4ca 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -8,15 +8,31 @@ use num_traits::{Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::OptionalArg; use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::VirtualMachine; -use super::objfloat; -use super::objstr; +use super::objfloat::{self, PyFloat}; +use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; +/// int(x=0) -> integer +/// int(x, base=10) -> integer +/// +/// Convert a number or string to an integer, or return 0 if no arguments +/// are given. If x is a number, return x.__int__(). For floating point +/// numbers, this truncates towards zero. +/// +/// If x is not a number or if base is given, then x must be a string, +/// bytes, or bytearray instance representing an integer literal in the +/// given base. The literal can be preceded by '+' or '-' and be surrounded +/// by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. +/// Base 0 means to interpret the base from the string as an integer literal. +/// >>> int('0b100', base=0) +/// 4 +#[pyclass] #[derive(Debug)] pub struct PyInt { value: BigInt, @@ -95,12 +111,10 @@ impl_try_from_object_int!( (u64, to_u64), ); -impl PyIntRef { - fn pass_value(self, _vm: &VirtualMachine) -> Self { - self - } - - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +#[pyimpl] +impl PyInt { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) } else { @@ -108,7 +122,8 @@ impl PyIntRef { } } - fn ne(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__ne__")] + fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) } else { @@ -116,7 +131,8 @@ impl PyIntRef { } } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) } else { @@ -124,7 +140,8 @@ impl PyIntRef { } } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) } else { @@ -132,7 +149,8 @@ impl PyIntRef { } } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) } else { @@ -140,7 +158,8 @@ impl PyIntRef { } } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) } else { @@ -148,7 +167,8 @@ impl PyIntRef { } } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) } else { @@ -156,7 +176,13 @@ impl PyIntRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.add(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) } else { @@ -164,7 +190,8 @@ impl PyIntRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) } else { @@ -172,7 +199,8 @@ impl PyIntRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__mul__")] + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) } else { @@ -180,7 +208,13 @@ impl PyIntRef { } } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.mul(other, vm) + } + + #[pymethod(name = "__truediv__")] + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) } else { @@ -188,7 +222,8 @@ impl PyIntRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rtruediv__")] + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) } else { @@ -196,7 +231,8 @@ impl PyIntRef { } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__floordiv__")] + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -209,7 +245,8 @@ impl PyIntRef { } } - fn lshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lshift__")] + fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -228,7 +265,8 @@ impl PyIntRef { } } - fn rshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rshift__")] + fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { return Ok(vm.ctx.not_implemented()); } @@ -247,7 +285,8 @@ impl PyIntRef { } } - fn xor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__xor__")] + fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) } else { @@ -255,7 +294,8 @@ impl PyIntRef { } } - fn rxor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) ^ (&self.value)) } else { @@ -263,7 +303,8 @@ impl PyIntRef { } } - fn or(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__or__")] + fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) } else { @@ -271,7 +312,8 @@ impl PyIntRef { } } - fn and(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__and__")] + fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); vm.ctx.new_int((&self.value) & v2) @@ -280,7 +322,8 @@ impl PyIntRef { } } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__pow__")] + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other).to_u32().unwrap(); vm.ctx.new_int(self.value.pow(v2)) @@ -292,7 +335,8 @@ impl PyIntRef { } } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__mod__")] + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -305,7 +349,8 @@ impl PyIntRef { } } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__divmod__")] + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -321,37 +366,82 @@ impl PyIntRef { } } - fn neg(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__neg__")] + fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } - fn hash(self, _vm: &VirtualMachine) -> u64 { + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() } - fn abs(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__abs__")] + fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } - fn round(self, _precision: OptionalArg, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__round__")] + fn round( + zelf: PyRef, + _precision: OptionalArg, + _vm: &VirtualMachine, + ) -> PyIntRef { + zelf } - fn float(self, _vm: &VirtualMachine) -> f64 { - self.value.to_f64().unwrap() + #[pymethod(name = "__int__")] + fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf } - fn invert(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__pos__")] + fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__float__")] + fn float(&self, vm: &VirtualMachine) -> PyResult { + self.value + .to_f64() + .map(PyFloat::from) + .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) + } + + #[pymethod(name = "__trunc__")] + fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__floor__")] + fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__ceil__")] + fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__index__")] + fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__invert__")] + fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } - fn repr(self, _vm: &VirtualMachine) -> String { + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn format(self, spec: PyRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__format__")] + fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -359,15 +449,28 @@ impl PyIntRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } - fn bit_length(self, _vm: &VirtualMachine) -> usize { + #[pymethod] + fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } - fn imag(self, _vm: &VirtualMachine) -> usize { + #[pymethod] + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pyproperty] + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pyproperty] + fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } } @@ -407,30 +510,24 @@ fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResul } // Casting function: +// TODO: this should just call `__int__` on the object pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { - let val = if objtype::isinstance(obj, &vm.ctx.int_type()) { - get_value(obj).clone() - } else if objtype::isinstance(obj, &vm.ctx.float_type()) { - objfloat::get_value(obj).to_bigint().unwrap() - } else if objtype::isinstance(obj, &vm.ctx.str_type()) { - let s = objstr::get_value(obj); - match i32::from_str_radix(&s, base) { - Ok(v) => v.to_bigint().unwrap(), - Err(err) => { - trace!("Error occurred during int conversion {:?}", err); - return Err(vm.new_value_error(format!( + match_class!(obj.clone(), + i @ PyInt => Ok(i.as_bigint().clone()), + f @ PyFloat => Ok(f.to_f64().to_bigint().unwrap()), + s @ PyString => { + i32::from_str_radix(s.as_str(), base) + .map(BigInt::from) + .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s - ))); - } - } - } else { - return Err(vm.new_type_error(format!( + ))) + }, + obj => Err(vm.new_type_error(format!( "int() argument must be a string or a number, not '{}'", obj.class().name - ))); - }; - Ok(val) + ))) + ) } // Retrieve inner int value: @@ -470,68 +567,9 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { } } -#[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let int_doc = "int(x=0) -> integer -int(x, base=10) -> integer - -Convert a number or string to an integer, or return 0 if no arguments -are given. If x is a number, return x.__int__(). For floating point -numbers, this truncates towards zero. - -If x is not a number or if base is given, then x must be a string, -bytes, or bytearray instance representing an integer literal in the -given base. The literal can be preceded by '+' or '-' and be surrounded -by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. -Base 0 means to interpret the base from the string as an integer literal. ->>> int('0b100', base=0) -4"; - let int_type = &context.int_type; - extend_class!(context, int_type, { - "__doc__" => context.new_str(int_doc.to_string()), - "__eq__" => context.new_rustfunc(PyIntRef::eq), - "__ne__" => context.new_rustfunc(PyIntRef::ne), - "__lt__" => context.new_rustfunc(PyIntRef::lt), - "__le__" => context.new_rustfunc(PyIntRef::le), - "__gt__" => context.new_rustfunc(PyIntRef::gt), - "__ge__" => context.new_rustfunc(PyIntRef::ge), - "__abs__" => context.new_rustfunc(PyIntRef::abs), - "__add__" => context.new_rustfunc(PyIntRef::add), - "__radd__" => context.new_rustfunc(PyIntRef::add), - "__and__" => context.new_rustfunc(PyIntRef::and), - "__divmod__" => context.new_rustfunc(PyIntRef::divmod), - "__float__" => context.new_rustfunc(PyIntRef::float), - "__round__" => context.new_rustfunc(PyIntRef::round), - "__ceil__" => context.new_rustfunc(PyIntRef::pass_value), - "__floor__" => context.new_rustfunc(PyIntRef::pass_value), - "__index__" => context.new_rustfunc(PyIntRef::pass_value), - "__trunc__" => context.new_rustfunc(PyIntRef::pass_value), - "__int__" => context.new_rustfunc(PyIntRef::pass_value), - "__floordiv__" => context.new_rustfunc(PyIntRef::floordiv), - "__hash__" => context.new_rustfunc(PyIntRef::hash), - "__lshift__" => context.new_rustfunc(PyIntRef::lshift), - "__rshift__" => context.new_rustfunc(PyIntRef::rshift), + PyInt::extend_class(context, &context.int_type); + extend_class!(context, &context.int_type, { "__new__" => context.new_rustfunc(int_new), - "__mod__" => context.new_rustfunc(PyIntRef::mod_), - "__mul__" => context.new_rustfunc(PyIntRef::mul), - "__rmul__" => context.new_rustfunc(PyIntRef::mul), - "__or__" => context.new_rustfunc(PyIntRef::or), - "__neg__" => context.new_rustfunc(PyIntRef::neg), - "__pos__" => context.new_rustfunc(PyIntRef::pass_value), - "__pow__" => context.new_rustfunc(PyIntRef::pow), - "__repr__" => context.new_rustfunc(PyIntRef::repr), - "__sub__" => context.new_rustfunc(PyIntRef::sub), - "__rsub__" => context.new_rustfunc(PyIntRef::rsub), - "__format__" => context.new_rustfunc(PyIntRef::format), - "__truediv__" => context.new_rustfunc(PyIntRef::truediv), - "__rtruediv__" => context.new_rustfunc(PyIntRef::rtruediv), - "__xor__" => context.new_rustfunc(PyIntRef::xor), - "__rxor__" => context.new_rustfunc(PyIntRef::rxor), - "__bool__" => context.new_rustfunc(PyIntRef::bool), - "__invert__" => context.new_rustfunc(PyIntRef::invert), - "bit_length" => context.new_rustfunc(PyIntRef::bit_length), - "conjugate" => context.new_rustfunc(PyIntRef::pass_value), - "real" => context.new_property(PyIntRef::pass_value), - "imag" => context.new_property(PyIntRef::imag) }); } diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 5c88a6337..6e1dc1d06 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,17 +2,13 @@ * Various types to support iteration. */ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; +use std::cell::Cell; + +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; -use super::objbool; -use super::objbytearray::PyByteArray; -use super::objbytes::PyBytes; -use super::objrange::PyRange; -use super::objsequence; use super::objtype; -use crate::obj::objtype::PyClassRef; +use super::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -20,10 +16,19 @@ use crate::obj::objtype::PyClassRef; * function 'iter' is called. */ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { - vm.call_method(iter_target, "__iter__", vec![]) - // let type_str = objstr::get_value(&vm.to_str(iter_target.class()).unwrap()); - // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); - // return Err(type_error); + if let Ok(method) = vm.get_method(iter_target.clone(), "__iter__") { + vm.invoke(method, vec![]) + } else if vm.get_method(iter_target.clone(), "__getitem__").is_ok() { + Ok(PySequenceIterator { + position: Cell::new(0), + obj: iter_target.clone(), + } + .into_ref(vm) + .into_object()) + } else { + let message = format!("Cannot iterate over {}", iter_target.class().name); + return Err(vm.new_type_error(message)); + } } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -70,118 +75,46 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -fn contains(vm: &VirtualMachine, args: PyFuncArgs, iter_type: PyClassRef) -> PyResult { - arg_check!( - vm, - args, - required = [(iter, Some(iter_type)), (needle, None)] - ); - loop { - if let Some(element) = get_next_object(vm, iter)? { - let equal = vm._eq(needle.clone(), element.clone())?; - if objbool::get_value(&equal) { - return Ok(vm.new_bool(true)); - } else { - continue; - } - } else { - return Ok(vm.new_bool(false)); - } +#[derive(Debug)] +pub struct PySequenceIterator { + pub position: Cell, + pub obj: PyObjectRef, +} + +impl PyValue for PySequenceIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.iter_type() } } -/// Common setup for iter types, adds __iter__ and __contains__ methods -pub fn iter_type_init(context: &PyContext, iter_type: &PyClassRef) { - let contains_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| contains(vm, args, cloned_iter_type.clone()) - }; +type PySequenceIteratorRef = PyRef; - let iter_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| { - arg_check!( - vm, - args, - required = [(iter, Some(cloned_iter_type.clone()))] - ); - // Return self: - Ok(iter.clone()) +impl PySequenceIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + let number = vm.ctx.new_int(self.position.get()); + match vm.call_method(&self.obj, "__getitem__", vec![number]) { + Ok(val) => { + self.position.set(self.position.get() + 1); + Ok(val) + } + Err(ref e) if objtype::isinstance(&e, &vm.ctx.exceptions.index_error) => { + Err(new_stop_iteration(vm)) + } + // also catches stop_iteration => stop_iteration + Err(e) => Err(e), } - }; + } - extend_class!(context, iter_type, { - "__contains__" => context.new_rustfunc(contains_func), - "__iter__" => context.new_rustfunc(iter_func) - }); -} - -// Sequence iterator: -fn iter_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter_target, None)]); - - get_iter(vm, iter_target) -} - -fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter, Some(vm.ctx.iter_type()))]); - - if let Some(PyIteratorValue { - ref position, - iterated_obj: ref iterated_obj_ref, - }) = iter.payload() - { - if let Some(range) = iterated_obj_ref.payload::() { - if let Some(int) = range.get(position.get()) { - position.set(position.get() + 1); - Ok(vm.ctx.new_int(int)) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.len() { - let obj_ref = vm.ctx.new_int(bytes[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.value.borrow().len() { - let obj_ref = vm.ctx.new_int(bytes.value.borrow()[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else { - let elements = objsequence::get_elements(iterated_obj_ref); - if position.get() < elements.len() { - let obj_ref = elements[position.get()].clone(); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } - } else { - panic!("NOT IMPL"); + fn iter(self, _vm: &VirtualMachine) -> Self { + self } } pub fn init(context: &PyContext) { let iter_type = &context.iter_type; - let iter_doc = "iter(iterable) -> iterator\n\ - iter(callable, sentinel) -> iterator\n\n\ - Get an iterator from an object. In the first form, the argument must\n\ - supply its own iterator, or be a sequence.\n\ - In the second form, the callable is called until it returns the sentinel."; - - iter_type_init(context, iter_type); extend_class!(context, iter_type, { - "__new__" => context.new_rustfunc(iter_new), - "__next__" => context.new_rustfunc(iter_next), - "__doc__" => context.new_str(iter_doc.to_string()) + "__next__" => context.new_rustfunc(PySequenceIteratorRef::next), + "__iter__" => context.new_rustfunc(PySequenceIteratorRef::iter), }); } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index dd5eb055b..a21bd9190 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,20 +1,25 @@ use std::cell::{Cell, RefCell}; use std::fmt; -use num_traits::ToPrimitive; +use std::ops::Range; + +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; +//use super::objint; +use super::objiter; use super::objsequence::{ get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, + SequenceIndex, }; +use super::objslice::PySliceRef; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -45,6 +50,54 @@ impl PyValue for PyList { } } +impl PyList { + pub fn get_len(&self) -> usize { + self.elements.borrow().len() + } + + pub fn get_pos(&self, p: i32) -> Option { + // convert a (potentially negative) positon into a real index + if p < 0 { + if -p as usize > self.get_len() { + None + } else { + Some(self.get_len() - ((-p) as usize)) + } + } else if p as usize >= self.get_len() { + None + } else { + Some(p as usize) + } + } + + pub fn get_slice_pos(&self, slice_pos: &BigInt) -> usize { + if let Some(pos) = slice_pos.to_i32() { + if let Some(index) = self.get_pos(pos) { + // within bounds + return index; + } + } + + if slice_pos.is_negative() { + // slice past start bound, round to start + 0 + } else { + // slice past end bound, round to end + self.get_len() + } + } + + pub fn get_slice_range(&self, start: &Option, stop: &Option) -> Range { + let start = start.as_ref().map(|x| self.get_slice_pos(x)).unwrap_or(0); + let stop = stop + .as_ref() + .map(|x| self.get_slice_pos(x)) + .unwrap_or_else(|| self.get_len()); + + start..stop + } +} + pub type PyListRef = PyRef; impl PyListRef { @@ -123,29 +176,199 @@ impl PyListRef { ) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyListIterator { + PyListIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + list: self, } } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.borrow_mut(); + fn setitem( + self, + subscript: SequenceIndex, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.setindex(index, value, vm), + SequenceIndex::Slice(slice) => { + if let Ok(sec) = PyIterable::try_from_object(vm, value) { + return self.setslice(slice, sec, vm); + } + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + } - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(pos_index) = elements.get_pos(idx) { - elements[pos_index] = value; - Ok(vm.get_none()) + fn setindex(self, index: i32, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut()[pos_index] = value; + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list assignment index out of range".to_string())) + } + } + + fn setslice(self, slice: PySliceRef, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&slice.start, &slice.stop); + if range.start < range.end { + match step.to_i32() { + Some(1) => self._set_slice(range, sec, vm), + Some(num) => { + // assign to extended slice + self._set_stepped_slice(range, num as usize, sec, vm) + } + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list, no question + #[allow(clippy::range_plus_one)] + self._set_stepped_slice(range.start..(range.start + 1), 1, sec, vm) + } + } } else { - Err(vm.new_index_error("list index out of range".to_string())) + // this functions as an insert of sec before range.start + self._set_slice(range.start..range.start, sec, vm) } } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = &slice.start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = &slice.stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); + let range = self.get_slice_range(&stop, &start); + match (-step).to_i32() { + Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list no question + self._set_stepped_slice_reverse(range.end - 1..range.end, 1, sec, vm) + } + } + } + } + + fn _set_slice(self, range: Range, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) + } + + fn _set_stepped_slice( + self, + range: Range, + step: usize, + sec: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } + + fn _set_stepped_slice_reverse( + self, + range: Range, + step: usize, + sec: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; + + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } + + fn _replace_indexes(self, indexes: I, items: &[PyObjectRef]) + where + I: Iterator, + { + let mut elements = self.elements.borrow_mut(); + + for (i, value) in indexes.zip(items) { + // clone for refrence count + elements[i] = value.clone(); } } @@ -163,6 +386,10 @@ impl PyListRef { Ok(s) } + fn hash(self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) + } + fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { let new_elements = seq_mul(&self.elements.borrow(), counter); vm.ctx.new_list(new_elements) @@ -307,6 +534,137 @@ impl PyListRef { Ok(vm.ctx.not_implemented()) } } + + fn delitem(self, subscript: SequenceIndex, vm: &VirtualMachine) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.delindex(index, vm), + SequenceIndex::Slice(slice) => self.delslice(slice, vm), + } + } + + fn delindex(self, index: i32, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut().remove(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + + fn delslice(self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { + let start = &slice.start; + let stop = &slice.stop; + let step = slice.step.clone().unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&start, &stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice_reverse(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } + } + + fn _del_slice(self, range: Range) { + self.elements.borrow_mut().drain(range); + } + + fn _del_stepped_slice(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().step_by(step).peekable(); + + for i in range.clone() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + elements.swap(i - deleted, i); + } + } + // then drain (the values to delete should now be contiguous at the end of the range) + elements.drain((range.end - deleted)..range.end); + } + + fn _del_stepped_slice_reverse(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().rev().step_by(step).peekable(); + + for i in range.clone().rev() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards back + elements.swap(i + deleted, i); + } + } + // then drain (the values to delete should now be contiguous at teh start of the range) + elements.drain(range.start..(range.start + deleted)); + } } fn list_new( @@ -412,6 +770,36 @@ fn list_sort(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[derive(Debug)] +pub struct PyListIterator { + pub position: Cell, + pub list: PyListRef, +} + +impl PyValue for PyListIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.listiterator_type() + } +} + +type PyListIteratorRef = PyRef; + +impl PyListIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.list.elements.borrow().len() { + let ret = self.list.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let list_type = &context.list_type; @@ -425,6 +813,7 @@ pub fn init(context: &PyContext) { "__iadd__" => context.new_rustfunc(PyListRef::iadd), "__bool__" => context.new_rustfunc(PyListRef::bool), "__contains__" => context.new_rustfunc(PyListRef::contains), + "__delitem__" => context.new_rustfunc(PyListRef::delitem), "__eq__" => context.new_rustfunc(PyListRef::eq), "__lt__" => context.new_rustfunc(PyListRef::lt), "__gt__" => context.new_rustfunc(PyListRef::gt), @@ -437,6 +826,7 @@ pub fn init(context: &PyContext) { "__len__" => context.new_rustfunc(PyListRef::len), "__new__" => context.new_rustfunc(list_new), "__repr__" => context.new_rustfunc(PyListRef::repr), + "__hash__" => context.new_rustfunc(PyListRef::hash), "__doc__" => context.new_str(list_doc.to_string()), "append" => context.new_rustfunc(PyListRef::append), "clear" => context.new_rustfunc(PyListRef::clear), @@ -450,4 +840,10 @@ pub fn init(context: &PyContext) { "pop" => context.new_rustfunc(PyListRef::pop), "remove" => context.new_rustfunc(PyListRef::remove) }); + + let listiterator_type = &context.listiterator_type; + extend_class!(context, listiterator_type, { + "__next__" => context.new_rustfunc(PyListIteratorRef::next), + "__iter__" => context.new_rustfunc(PyListIteratorRef::iter), + }); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index fbf0fa035..061f4179f 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -46,6 +46,10 @@ impl PyMapRef { // the mapper itself can raise StopIteration which does stop the map iteration vm.invoke(self.mapper.clone(), next_objs) } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { @@ -55,10 +59,10 @@ pub fn init(context: &PyContext) { Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted."; - objiter::iter_type_init(context, map_type); extend_class!(context, map_type, { "__new__" => context.new_rustfunc(map_new), "__next__" => context.new_rustfunc(PyMapRef::next), + "__iter__" => context.new_rustfunc(PyMapRef::iter), "__doc__" => context.new_str(map_doc.to_string()) }); } diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 7f53fb5ae..cfe44dea3 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -20,7 +20,7 @@ pub fn new_memory_view( bytes_object: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - vm.ctx.set_attr(&cls, "obj", bytes_object.clone()); + vm.set_attr(cls.as_object(), "obj", bytes_object.clone())?; PyMemoryView { obj: bytes_object }.into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index b641f0426..904487e4d 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -1,5 +1,5 @@ use crate::obj::objtype::PyClassRef; -use crate::pyobject::{DictProtocol, PyContext, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyContext, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] @@ -17,11 +17,7 @@ impl PyValue for PyModule { impl PyModuleRef { fn dir(self: PyModuleRef, vm: &VirtualMachine) -> PyResult { if let Some(dict) = &self.into_object().dict { - let keys = dict - .get_key_value_pairs() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); + let keys = dict.into_iter().map(|(k, _v)| k.clone()).collect(); Ok(vm.ctx.new_list(keys)) } else { panic!("Modules should definitely have a dict."); diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index c43e31a96..47cfa008d 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,4 +1,4 @@ -use super::objdict::{self, PyDictRef}; +use super::objdict::PyDictRef; use super::objlist::PyList; use super::objstr::PyStringRef; use super::objtype; @@ -6,7 +6,7 @@ use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, + IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -55,8 +55,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py vm.ctx.not_implemented() } -fn object_hash(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("unhashable type".to_string())) +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 { + zelf.get_id() as u64 } fn object_setattr( @@ -77,7 +77,7 @@ fn object_setattr( } if let Some(ref dict) = obj.clone().dict { - dict.set_item(&vm.ctx, &attr_name.value, value); + dict.set_item(attr_name, value, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -98,7 +98,7 @@ fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) } if let Some(ref dict) = obj.dict { - dict.del_item(&attr_name.value); + dict.del_item(attr_name, vm)?; Ok(()) } else { Err(vm.new_attribute_error(format!( @@ -117,13 +117,22 @@ fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id()) } -pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { - let attributes = get_attributes(&obj); - let attributes: Vec = attributes - .keys() - .map(|k| vm.ctx.new_str(k.to_string())) - .collect(); - PyList::from(attributes) +pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let attributes: PyAttributes = objtype::get_attributes(obj.class()); + + let dict = PyDictRef::from_attributes(attributes, vm)?; + + // Get instance attributes: + if let Some(object_dict) = &obj.dict { + vm.invoke( + vm.get_attribute(dict.clone().into_object(), "update")?, + object_dict.clone().into_object(), + )?; + } + + let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k.clone()).collect(); + + Ok(PyList::from(attributes)) } fn object_format( @@ -158,7 +167,11 @@ pub fn init(context: &PyContext) { "__ge__" => context.new_rustfunc(object_ge), "__setattr__" => context.new_rustfunc(object_setattr), "__delattr__" => context.new_rustfunc(object_delattr), - "__dict__" => context.new_property(object_dict), + "__dict__" => + PropertyBuilder::new(context) + .add_getter(object_dict) + .add_setter(object_dict_setter) + .create(), "__dir__" => context.new_rustfunc(object_dir), "__hash__" => context.new_rustfunc(object_hash), "__str__" => context.new_rustfunc(object_str), @@ -194,6 +207,16 @@ fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult } } +fn object_dict_setter( + _instance: PyObjectRef, + _value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on am object isn't yet implemented".to_string(), + )) +} + fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { let name = &name_str.value; trace!("object.__getattribute__({:?}, {:?})", obj, name); @@ -208,7 +231,7 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } - if let Some(obj_attr) = object_getattr(&obj, &name) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm)? { Ok(obj_attr) } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { vm.call_get_descriptor(attr, obj) @@ -219,24 +242,14 @@ fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMach } } -fn object_getattr(obj: &PyObjectRef, attr_name: &str) -> Option { +fn object_getattr( + obj: &PyObjectRef, + attr_name: &str, + vm: &VirtualMachine, +) -> PyResult> { if let Some(ref dict) = obj.dict { - dict.get_item(attr_name) + dict.get_item_option(attr_name, vm) } else { - None + Ok(None) } } - -pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { - // Get class attributes: - let mut attributes = objtype::get_attributes(obj.class()); - - // Get instance attributes: - if let Some(dict) = &obj.dict { - for (key, value) in objdict::get_key_value_pairs(dict.as_object()) { - attributes.insert(key.to_string(), value.clone()); - } - } - - attributes -} diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index c1a0b9401..1a21a49ad 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -6,11 +6,14 @@ use num_integer::Integer; use num_traits::{One, Signed, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{Either, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use super::objint::{PyInt, PyIntRef}; -use super::objslice::PySliceRef; +use super::objiter; +use super::objslice::{PySlice, PySliceRef}; use super::objtype::PyClassRef; #[derive(Debug, Clone)] @@ -98,26 +101,32 @@ pub fn init(context: &PyContext) { When step is given, it specifies the increment (or decrement)."; extend_class!(context, range_type, { - "__bool__" => context.new_rustfunc(PyRangeRef::bool), - "__contains__" => context.new_rustfunc(PyRangeRef::contains), + "__bool__" => context.new_rustfunc(PyRange::bool), + "__contains__" => context.new_rustfunc(PyRange::contains), "__doc__" => context.new_str(range_doc.to_string()), - "__getitem__" => context.new_rustfunc(PyRangeRef::getitem), - "__iter__" => context.new_rustfunc(PyRangeRef::iter), - "__len__" => context.new_rustfunc(PyRangeRef::len), + "__getitem__" => context.new_rustfunc(PyRange::getitem), + "__iter__" => context.new_rustfunc(PyRange::iter), + "__len__" => context.new_rustfunc(PyRange::len), "__new__" => context.new_rustfunc(range_new), - "__repr__" => context.new_rustfunc(PyRangeRef::repr), - "__reversed__" => context.new_rustfunc(PyRangeRef::reversed), - "count" => context.new_rustfunc(PyRangeRef::count), - "index" => context.new_rustfunc(PyRangeRef::index), - "start" => context.new_property(PyRangeRef::start), - "stop" => context.new_property(PyRangeRef::stop), - "step" => context.new_property(PyRangeRef::step), + "__repr__" => context.new_rustfunc(PyRange::repr), + "__reversed__" => context.new_rustfunc(PyRange::reversed), + "count" => context.new_rustfunc(PyRange::count), + "index" => context.new_rustfunc(PyRange::index), + "start" => context.new_property(PyRange::start), + "stop" => context.new_property(PyRange::stop), + "step" => context.new_property(PyRange::step), + }); + + let rangeiterator_type = &context.rangeiterator_type; + extend_class!(context, rangeiterator_type, { + "__next__" => context.new_rustfunc(PyRangeIteratorRef::next), + "__iter__" => context.new_rustfunc(PyRangeIteratorRef::iter), }); } type PyRangeRef = PyRef; -impl PyRangeRef { +impl PyRange { fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { PyRange { start: PyInt::new(BigInt::zero()).into_ref(vm), @@ -135,8 +144,8 @@ impl PyRangeRef { vm: &VirtualMachine, ) -> PyResult { PyRange { - start: start, - stop: stop, + start, + stop, step: step .into_option() .unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)), @@ -144,26 +153,26 @@ impl PyRangeRef { .into_ref_with_type(vm, cls) } - fn start(self, _vm: &VirtualMachine) -> PyIntRef { + fn start(&self, _vm: &VirtualMachine) -> PyIntRef { self.start.clone() } - fn stop(self, _vm: &VirtualMachine) -> PyIntRef { + fn stop(&self, _vm: &VirtualMachine) -> PyIntRef { self.stop.clone() } - fn step(self, _vm: &VirtualMachine) -> PyIntRef { + fn step(&self, _vm: &VirtualMachine) -> PyIntRef { self.step.clone() } - fn iter(self: PyRangeRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIterator { + PyRangeIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + range: zelf, } } - fn reversed(self: PyRangeRef, vm: &VirtualMachine) -> PyIteratorValue { + fn reversed(&self, vm: &VirtualMachine) -> PyRangeIterator { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -189,13 +198,13 @@ impl PyRangeRef { step: PyInt::new(-step).into_ref(vm), }; - PyIteratorValue { + PyRangeIterator { position: Cell::new(0), - iterated_obj: reversed.into_ref(vm).into_object(), + range: reversed.into_ref(vm), } } - fn len(self, _vm: &VirtualMachine) -> PyInt { + fn len(&self, _vm: &VirtualMachine) -> PyInt { let start = self.start.as_bigint(); let stop = self.stop.as_bigint(); let step = self.step.as_bigint(); @@ -208,7 +217,7 @@ impl PyRangeRef { } } - fn repr(self, _vm: &VirtualMachine) -> String { + fn repr(&self, _vm: &VirtualMachine) -> String { if self.step.as_bigint().is_one() { format!("range({}, {})", self.start, self.stop) } else { @@ -216,11 +225,11 @@ impl PyRangeRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.is_empty() } - fn contains(self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { + fn contains(&self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { if let Ok(int) = needle.downcast::() { match self.offset(int.as_bigint()) { Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), @@ -231,7 +240,7 @@ impl PyRangeRef { } } - fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(int) = needle.downcast::() { match self.index_of(int.as_bigint()) { Some(idx) => Ok(PyInt::new(idx)), @@ -242,7 +251,7 @@ impl PyRangeRef { } } - fn count(self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { + fn count(&self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { if let Ok(int) = item.downcast::() { if self.index_of(int.as_bigint()).is_some() { PyInt::new(1) @@ -254,16 +263,16 @@ impl PyRangeRef { } } - fn getitem(self, subscript: Either, vm: &VirtualMachine) -> PyResult { + fn getitem(&self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { match subscript { - Either::A(index) => { + RangeIndex::Int(index) => { if let Some(value) = self.get(index.as_bigint()) { Ok(PyInt::new(value).into_ref(vm).into_object()) } else { Err(vm.new_index_error("range object index out of range".to_string())) } } - Either::B(slice) => { + RangeIndex::Slice(slice) => { let new_start = if let Some(int) = slice.start_index(vm)? { if let Some(i) = self.get(int) { PyInt::new(i).into_ref(vm) @@ -305,11 +314,58 @@ impl PyRangeRef { fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let range = if args.args.len() <= 2 { let (cls, stop) = args.bind(vm)?; - PyRangeRef::new(cls, stop, vm) + PyRange::new(cls, stop, vm) } else { let (cls, start, stop, step) = args.bind(vm)?; - PyRangeRef::new_from(cls, start, stop, step, vm) + PyRange::new_from(cls, start, stop, step, vm) }?; Ok(range.into_object()) } + +#[derive(Debug)] +pub struct PyRangeIterator { + position: Cell, + range: PyRangeRef, +} + +impl PyValue for PyRangeIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.rangeiterator_type() + } +} + +type PyRangeIteratorRef = PyRef; + +impl PyRangeIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if let Some(int) = self.range.get(self.position.get()) { + self.position.set(self.position.get() + 1); + Ok(int) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + +pub enum RangeIndex { + Int(PyIntRef), + Slice(PySliceRef), +} + +impl TryFromObject for RangeIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(RangeIndex::Int(i)), + s @ PySlice => Ok(RangeIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } +} diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index a293314be..5594ac858 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -2,16 +2,16 @@ use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; +use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; -use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use super::objbool; use super::objint::PyInt; use super::objlist::PyList; -use super::objslice::PySlice; +use super::objslice::{PySlice, PySliceRef}; use super::objtuple::PyTuple; pub trait PySliceableSequence { @@ -87,8 +87,20 @@ pub trait PySliceableSequence { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { @@ -138,6 +150,24 @@ impl PySliceableSequence for Vec { } } +pub enum SequenceIndex { + Int(i32), + Slice(PySliceRef), +} + +impl TryFromObject for SequenceIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(SequenceIndex::Int(i32::try_from_object(vm, i.into_object())?)), + s @ PySlice => Ok(SequenceIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } +} + pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index ad82c34de..b8f9f6d3b 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -7,27 +7,28 @@ use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; use std::hash::{Hash, Hasher}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIteratorValue, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; -use super::objiter; +use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; #[derive(Default)] pub struct PySet { - elements: RefCell>, + inner: RefCell, } pub type PySetRef = PyRef; #[derive(Default)] pub struct PyFrozenSet { - elements: HashMap, + inner: PySetInner, } pub type PyFrozenSetRef = PyRef; @@ -57,45 +58,611 @@ impl PyValue for PyFrozenSet { } } -pub fn get_elements(obj: &PyObjectRef) -> HashMap { - if let Some(set) = obj.payload::() { - return set.elements.borrow().clone(); - } else if let Some(frozenset) = obj.payload::() { - return frozenset.elements.clone(); - } - panic!("Not frozenset or set"); -} - -fn validate_set_or_frozenset(vm: &VirtualMachine, cls: PyClassRef) -> PyResult<()> { - if !(objtype::issubclass(&cls, &vm.ctx.set_type()) - || objtype::issubclass(&cls, &vm.ctx.frozenset_type())) - { - return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))); - } - Ok(()) -} - -fn create_set( - vm: &VirtualMachine, +#[derive(Default, Clone)] +struct PySetInner { elements: HashMap, - cls: PyClassRef, -) -> PyResult { - if objtype::issubclass(&cls, &vm.ctx.set_type()) { +} + +impl PySetInner { + fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + let elements: HashMap = match iterable { + OptionalArg::Missing => HashMap::new(), + OptionalArg::Present(iterable) => { + let mut elements = HashMap::new(); + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; + } + elements + } + }; + + Ok(PySetInner { elements }) + } + + fn len(&self) -> usize { + self.elements.len() + } + fn copy(&self) -> PySetInner { + PySetInner { + elements: self.elements.clone(), + } + } + + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + for element in self.elements.iter() { + let value = vm._eq(needle.clone(), element.1.clone())?; + if objbool::get_value(&value) { + return Ok(true); + } + } + Ok(false) + } + + fn _compare_inner( + &self, + other: &PySetInner, + size_func: &Fn(usize, usize) -> bool, + swap: bool, + vm: &VirtualMachine, + ) -> PyResult { + let get_zelf = |swap: bool| -> &PySetInner { + if swap { + other + } else { + self + } + }; + let get_other = |swap: bool| -> &PySetInner { + if swap { + self + } else { + other + } + }; + + if size_func(get_zelf(swap).len(), get_other(swap).len()) { + return Ok(vm.new_bool(false)); + } + for element in get_other(swap).elements.iter() { + if !get_zelf(swap).contains(element.1.clone(), vm)? { + return Ok(vm.new_bool(false)); + } + } + Ok(vm.new_bool(true)) + } + + fn eq(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf != other }, + false, + vm, + ) + } + + fn ge(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + false, + vm, + ) + } + + fn gt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + false, + vm, + ) + } + + fn le(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + true, + vm, + ) + } + + fn lt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + true, + vm, + ) + } + + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + for item in other.iter(vm)? { + insert_into_set(vm, &mut elements, &item?)?; + } + + Ok(PySetInner { elements }) + } + + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut elements = HashMap::new(); + for item in other.iter(vm)? { + let obj = item?; + if self.contains(obj.clone(), vm)? { + insert_into_set(vm, &mut elements, &obj)?; + } + } + Ok(PySetInner { elements }) + } + + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut elements = self.elements.clone(); + for item in other.iter(vm)? { + let obj = item?; + if self.contains(obj.clone(), vm)? { + remove_from_set(vm, &mut elements, &obj)?; + } + } + Ok(PySetInner { elements }) + } + + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut new_inner = self.clone(); + + for item in other.iter(vm)? { + let obj = item?; + if !self.contains(obj.clone(), vm)? { + new_inner.add(&obj, vm)?; + } else { + new_inner.remove(&obj, vm)?; + } + } + + Ok(new_inner) + } + + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + let items = self.elements.values().cloned().collect(); + let set_list = vm.ctx.new_list(items); + PyListIterator { + position: Cell::new(0), + list: set_list.downcast().unwrap(), + } + } + + fn repr(&self, vm: &VirtualMachine) -> PyResult { + let mut str_parts = vec![]; + for elem in self.elements.values() { + let part = vm.to_repr(elem)?; + str_parts.push(part.value.clone()); + } + + Ok(format!("{{{}}}", str_parts.join(", "))) + } + + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + insert_into_set(vm, &mut self.elements, &item) + } + + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + remove_from_set(vm, &mut self.elements, &item) + } + + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn discard( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + _value: &PyObjectRef, + ) -> PyResult { + elements.remove(&key); + Ok(vm.get_none()) + } + perform_action_with_hash(vm, &mut self.elements, &item, &discard) + } + + fn clear(&mut self) { + self.elements.clear(); + } + + fn pop(&mut self, vm: &VirtualMachine) -> PyResult { + let elements = &mut self.elements; + match elements.clone().keys().next() { + Some(key) => Ok(elements.remove(key).unwrap()), + None => Err(vm.new_key_error("pop from an empty set".to_string())), + } + } + + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + insert_into_set(vm, &mut self.elements, &item?)?; + } + Ok(vm.get_none()) + } + + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + let temp_inner = self.copy(); + self.clear(); + for item in iterable.iter(vm)? { + let obj = item?; + if temp_inner.contains(obj.clone(), vm)? { + self.add(&obj, vm)?; + } + } + Ok(vm.get_none()) + } + + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if self.contains(obj.clone(), vm)? { + self.remove(&obj, vm)?; + } + } + Ok(vm.get_none()) + } + + fn symmetric_difference_update( + &mut self, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + for item in iterable.iter(vm)? { + let obj = item?; + if !self.contains(obj.clone(), vm)? { + self.add(&obj, vm)?; + } else { + self.remove(&obj, vm)?; + } + } + Ok(vm.get_none()) + } +} + +impl PySetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PySet { + inner: RefCell::new(PySetInner::new(iterable, vm)?), + } + .into_ref_with_type(vm, cls) + } + + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.borrow().len() + } + + fn copy(self, _vm: &VirtualMachine) -> PySet { + PySet { + inner: RefCell::new(self.inner.borrow().copy()), + } + } + + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( PySet { - elements: RefCell::new(elements), + inner: RefCell::new(self.inner.borrow().union(other, vm)?), }, PySet::class(vm), None, )) - } else if objtype::issubclass(&cls, &vm.ctx.frozenset_type()) { + } + + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { Ok(PyObject::new( - PyFrozenSet { elements: elements }, + PySet { + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), + }, + PySet::class(vm), + None, + )) + } + + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), + }, + PySet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PySet { + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), + }, + PySet::class(vm), + None, + )) + } + + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.borrow().iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = self.inner.borrow(); + let s = if inner.len() == 0 { + "set()".to_string() + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + inner.repr(vm)? + } else { + "set(...)".to_string() + }; + Ok(vm.new_str(s)) + } + + fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().add(&item, vm) + } + + fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().remove(&item, vm) + } + + fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().discard(&item, vm) + } + + fn clear(self, _vm: &VirtualMachine) { + self.inner.borrow_mut().clear() + } + + fn pop(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().pop(vm) + } + + fn ior(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable.iterable, vm)?; + Ok(self.as_object().clone()) + } + + fn update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn intersection_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().intersection_update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn iand(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .intersection_update(iterable.iterable, vm)?; + Ok(self.as_object().clone()) + } + + fn difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().difference_update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn isub(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .difference_update(iterable.iterable, vm)?; + Ok(self.as_object().clone()) + } + + fn symmetric_difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable, vm)?; + Ok(vm.get_none()) + } + + fn ixor(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable.iterable, vm)?; + Ok(self.as_object().clone()) + } +} + +impl PyFrozenSetRef { + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyFrozenSet { + inner: PySetInner::new(iterable, vm)?, + } + .into_ref_with_type(vm, cls) + } + + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } + + fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { + PyFrozenSet { + inner: self.inner.copy(), + } + } + + fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.contains(needle, vm) + } + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.eq(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.ge(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.gt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.le(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match_class!(other, + set @ PySet => self.inner.lt(&set.inner.borrow(), vm), + frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), + other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, + ) + } + + fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.union(other, vm)?, + }, PyFrozenSet::class(vm), None, )) - } else { - Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", cls))) + } + + fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.intersection(other, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.difference(other, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(PyObject::new( + PyFrozenSet { + inner: self.inner.symmetric_difference(other, vm)?, + }, + PyFrozenSet::class(vm), + None, + )) + } + + fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + + fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + + fn iter(self, vm: &VirtualMachine) -> PyListIterator { + self.inner.iter(vm) + } + + fn repr(self, vm: &VirtualMachine) -> PyResult { + let inner = &self.inner; + let s = if inner.len() == 0 { + "frozenset()".to_string() + } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + format!("frozenset({})", inner.repr(vm)?) + } else { + "frozenset(...)".to_string() + }; + Ok(vm.new_str(s)) } } @@ -131,461 +698,51 @@ fn insert_into_set( perform_action_with_hash(vm, elements, item, &insert) } -fn set_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.add called with: {:?}", args); - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (item, None)] - ); - match zelf.payload::() { - Some(set) => insert_into_set(vm, &mut set.elements.borrow_mut(), item), - _ => Err(vm.new_type_error("set.add is called with no item".to_string())), - } -} - -fn set_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.remove called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &remove) - } - _ => Err(vm.new_type_error("set.remove is called with no item".to_string())), - } -} - -fn set_discard(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.discard called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &discard) - } - None => Err(vm.new_type_error("set.discard is called with no item".to_string())), - } -} - -fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.clear called"); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - match s.payload::() { - Some(set) => { - set.elements.borrow_mut().clear(); - Ok(vm.get_none()) - } - None => Err(vm.new_type_error("".to_string())), - } -} - -/* Create a new object of sub-type of set */ -fn set_new(cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, cls.clone())?; - - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; - } - elements - } - }; - - create_set(vm, elements, cls.clone()) -} - -fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.len called with: {:?}", args); - arg_check!(vm, args, required = [(s, None)]); - validate_set_or_frozenset(vm, s.class())?; - let elements = get_elements(s); - Ok(vm.context().new_int(elements.len())) -} - -fn set_copy(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - trace!("set.copy called with: {:?}", obj); - validate_set_or_frozenset(vm, obj.class())?; - let elements = get_elements(&obj).clone(); - create_set(vm, elements, obj.class()) -} - -fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.set_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(o) { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("{{{}}}", str_parts.join(", ")) - } else { - "set(...)".to_string() - }; - Ok(vm.new_str(s)) -} - -pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(set, None), (needle, None)]); - validate_set_or_frozenset(vm, set.class())?; - for element in get_elements(set).iter() { - match vm._eq(needle.clone(), element.1.clone()) { - Ok(value) => { - if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), - } - } - - Ok(vm.new_bool(false)) -} - -fn set_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf != other }, - false, - ) -} - -fn set_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - false, - ) -} - -fn set_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - false, - ) -} - -fn set_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - true, - ) -} - -fn set_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - true, - ) -} - -fn set_compare_inner( +fn remove_from_set( vm: &VirtualMachine, - args: PyFuncArgs, - size_func: &Fn(usize, usize) -> bool, - swap: bool, + elements: &mut HashMap, + item: &PyObjectRef, ) -> PyResult { - arg_check!(vm, args, required = [(zelf, None), (other, None)]); + fn remove( + vm: &VirtualMachine, + elements: &mut HashMap, + key: u64, + value: &PyObjectRef, + ) -> PyResult { + match elements.remove(&key) { + None => { + let item_str = format!("{:?}", value); + Err(vm.new_key_error(item_str)) + } + Some(_) => Ok(vm.get_none()), + } + } + perform_action_with_hash(vm, elements, item, &remove) +} - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; +struct SetIterable { + iterable: PyIterable, +} - let get_zelf = |swap: bool| -> &PyObjectRef { - if swap { - other +impl TryFromObject for SetIterable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if objtype::issubclass(&obj.class(), &vm.ctx.set_type()) + || objtype::issubclass(&obj.class(), &vm.ctx.frozenset_type()) + { + Ok(SetIterable { + iterable: PyIterable::try_from_object(vm, obj)?, + }) } else { - zelf - } - }; - let get_other = |swap: bool| -> &PyObjectRef { - if swap { - zelf - } else { - other - } - }; - - let zelf_elements = get_elements(get_zelf(swap)); - let other_elements = get_elements(get_other(swap)); - if size_func(zelf_elements.len(), other_elements.len()) { - return Ok(vm.new_bool(false)); - } - for element in other_elements.iter() { - match vm.call_method(get_zelf(swap), "__contains__", vec![element.1.clone()]) { - Ok(value) => { - if !objbool::get_value(&value) { - return Ok(vm.new_bool(false)); - } - } - Err(_) => return Err(vm.new_type_error("".to_string())), + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) } } - Ok(vm.new_bool(true)) } -fn set_union(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - - let mut elements = get_elements(&zelf).clone(); - elements.extend(get_elements(&other).clone()); - - create_set(vm, elements, zelf.class()) -} - -fn set_intersection(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Intersection) -} - -fn set_difference(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Difference) -} - -fn set_symmetric_difference( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - for element in get_elements(&other).iter() { - let value = vm.call_method(&zelf, "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - -enum SetCombineOperation { - Intersection, - Difference, -} - -fn set_combine_inner( - zelf: PyObjectRef, - other: PyObjectRef, - vm: &VirtualMachine, - op: SetCombineOperation, -) -> PyResult { - validate_set_or_frozenset(vm, zelf.class())?; - validate_set_or_frozenset(vm, other.class())?; - let mut elements = HashMap::new(); - - for element in get_elements(&zelf).iter() { - let value = vm.call_method(&other, "__contains__", vec![element.1.clone()])?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), - }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); - } - } - - create_set(vm, elements, zelf.class()) -} - -fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - - match s.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), - } - } - _ => Err(vm.new_type_error("".to_string())), - } -} - -fn set_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ior(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ior(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - } - _ => return Err(vm.new_type_error("set.update is called with no other".to_string())), - } - Ok(zelf.clone()) -} - -fn set_intersection_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection)?; - Ok(vm.get_none()) -} - -fn set_iand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection) -} - -fn set_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference)?; - Ok(vm.get_none()) -} - -fn set_isub(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference) -} - -fn set_combine_update_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - op: SetCombineOperation, -) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - Ok(zelf.clone()) -} - -fn set_symmetric_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ixor(vm, args)?; - Ok(vm.get_none()) -} - -fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let elements_original = set.elements.borrow().clone(); - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - set.elements.borrow_mut().remove(&element.0.clone()); - } - } - } - _ => return Err(vm.new_type_error("".to_string())), - } - - Ok(zelf.clone()) -} - -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyIteratorValue { - let items = zelf.elements.borrow().values().cloned().collect(); - let set_list = vm.ctx.new_list(items); - PyIteratorValue { - position: Cell::new(0), - iterated_obj: set_list, - } -} - -fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.frozenset_type()))]); - - let elements = get_elements(o); - let s = if elements.is_empty() { - "frozenset()".to_string() - } else { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); - } - - format!("frozenset({{{}}})", str_parts.join(", ")) - }; - Ok(vm.new_str(s)) +fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("unhashable type".to_string())) } pub fn init(context: &PyContext) { @@ -596,41 +753,42 @@ pub fn init(context: &PyContext) { Build an unordered collection of unique elements."; extend_class!(context, set_type, { - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), - "__new__" => context.new_rustfunc(set_new), - "__repr__" => context.new_rustfunc(set_repr), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), + "__hash__" => context.new_rustfunc(set_hash), + "__contains__" => context.new_rustfunc(PySetRef::contains), + "__len__" => context.new_rustfunc(PySetRef::len), + "__new__" => context.new_rustfunc(PySetRef::new), + "__repr__" => context.new_rustfunc(PySetRef::repr), + "__eq__" => context.new_rustfunc(PySetRef::eq), + "__ge__" => context.new_rustfunc(PySetRef::ge), + "__gt__" => context.new_rustfunc(PySetRef::gt), + "__le__" => context.new_rustfunc(PySetRef::le), + "__lt__" => context.new_rustfunc(PySetRef::lt), + "issubset" => context.new_rustfunc(PySetRef::le), + "issuperset" => context.new_rustfunc(PySetRef::ge), + "union" => context.new_rustfunc(PySetRef::union), + "__or__" => context.new_rustfunc(PySetRef::or), + "intersection" => context.new_rustfunc(PySetRef::intersection), + "__and__" => context.new_rustfunc(PySetRef::and), + "difference" => context.new_rustfunc(PySetRef::difference), + "__sub__" => context.new_rustfunc(PySetRef::sub), + "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PySetRef::xor), "__doc__" => context.new_str(set_doc.to_string()), - "add" => context.new_rustfunc(set_add), - "remove" => context.new_rustfunc(set_remove), - "discard" => context.new_rustfunc(set_discard), - "clear" => context.new_rustfunc(set_clear), - "copy" => context.new_rustfunc(set_copy), - "pop" => context.new_rustfunc(set_pop), - "update" => context.new_rustfunc(set_update), - "__ior__" => context.new_rustfunc(set_ior), - "intersection_update" => context.new_rustfunc(set_intersection_update), - "__iand__" => context.new_rustfunc(set_iand), - "difference_update" => context.new_rustfunc(set_difference_update), - "__isub__" => context.new_rustfunc(set_isub), - "symmetric_difference_update" => context.new_rustfunc(set_symmetric_difference_update), - "__ixor__" => context.new_rustfunc(set_ixor), - "__iter__" => context.new_rustfunc(set_iter) + "add" => context.new_rustfunc(PySetRef::add), + "remove" => context.new_rustfunc(PySetRef::remove), + "discard" => context.new_rustfunc(PySetRef::discard), + "clear" => context.new_rustfunc(PySetRef::clear), + "copy" => context.new_rustfunc(PySetRef::copy), + "pop" => context.new_rustfunc(PySetRef::pop), + "update" => context.new_rustfunc(PySetRef::update), + "__ior__" => context.new_rustfunc(PySetRef::ior), + "intersection_update" => context.new_rustfunc(PySetRef::intersection_update), + "__iand__" => context.new_rustfunc(PySetRef::iand), + "difference_update" => context.new_rustfunc(PySetRef::difference_update), + "__isub__" => context.new_rustfunc(PySetRef::isub), + "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), + "__ixor__" => context.new_rustfunc(PySetRef::ixor), + "__iter__" => context.new_rustfunc(PySetRef::iter) }); let frozenset_type = &context.frozenset_type; @@ -640,26 +798,27 @@ pub fn init(context: &PyContext) { Build an immutable unordered collection of unique elements."; extend_class!(context, frozenset_type, { - "__new__" => context.new_rustfunc(set_new), - "__eq__" => context.new_rustfunc(set_eq), - "__ge__" => context.new_rustfunc(set_ge), - "__gt__" => context.new_rustfunc(set_gt), - "__le__" => context.new_rustfunc(set_le), - "__lt__" => context.new_rustfunc(set_lt), - "issubset" => context.new_rustfunc(set_le), - "issuperset" => context.new_rustfunc(set_ge), - "union" => context.new_rustfunc(set_union), - "__or__" => context.new_rustfunc(set_union), - "intersection" => context.new_rustfunc(set_intersection), - "__and__" => context.new_rustfunc(set_intersection), - "difference" => context.new_rustfunc(set_difference), - "__sub__" => context.new_rustfunc(set_difference), - "symmetric_difference" => context.new_rustfunc(set_symmetric_difference), - "__xor__" => context.new_rustfunc(set_symmetric_difference), - "__contains__" => context.new_rustfunc(set_contains), - "__len__" => context.new_rustfunc(set_len), + "__new__" => context.new_rustfunc(PyFrozenSetRef::new), + "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), + "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), + "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), + "__le__" => context.new_rustfunc(PyFrozenSetRef::le), + "__lt__" => context.new_rustfunc(PyFrozenSetRef::lt), + "issubset" => context.new_rustfunc(PyFrozenSetRef::le), + "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), + "union" => context.new_rustfunc(PyFrozenSetRef::union), + "__or__" => context.new_rustfunc(PyFrozenSetRef::or), + "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), + "__and__" => context.new_rustfunc(PyFrozenSetRef::and), + "difference" => context.new_rustfunc(PyFrozenSetRef::difference), + "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), + "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), + "__xor__" => context.new_rustfunc(PyFrozenSetRef::xor), + "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), + "__len__" => context.new_rustfunc(PyFrozenSetRef::len), "__doc__" => context.new_str(frozenset_doc.to_string()), - "__repr__" => context.new_rustfunc(frozenset_repr), - "copy" => context.new_rustfunc(set_copy) + "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), + "copy" => context.new_rustfunc(PyFrozenSetRef::copy), + "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter) }); } diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 82e4cef8c..cebfee330 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,5 +1,5 @@ use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use crate::obj::objint::PyInt; diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index aa62812a2..1638a712c 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -10,8 +10,8 @@ use unicode_segmentation::UnicodeSegmentation; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TryIntoRef, TypeProtocol, + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, + PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -20,11 +20,37 @@ use super::objsequence::PySliceableSequence; use super::objslice::PySlice; use super::objtype::{self, PyClassRef}; +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." +#[pyclass(name = "str")] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public pub value: String, } + +impl PyString { + pub fn as_str(&self) -> &str { + &self.value + } +} + +impl From<&str> for PyString { + fn from(s: &str) -> PyString { + PyString { + value: s.to_string(), + } + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { @@ -48,8 +74,30 @@ impl TryIntoRef for &str { } } -impl PyStringRef { - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyString { + // TODO: should with following format + // class str(object='') + // class str(object=b'', encoding='utf-8', errors='strict') + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + object: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let string = match object { + OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), + OptionalArg::Missing => vm.new_str("".to_string()), + }; + if string.class().is(&cls) { + TryFromObject::try_from_object(vm, string) + } else { + let payload = string.payload::().unwrap(); + payload.clone().into_ref_with_type(vm, cls) + } + } + #[pymethod(name = "__add__")] + fn add(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) } else { @@ -57,11 +105,13 @@ impl PyStringRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() } - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + #[pymethod(name = "__eq__")] + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) } else { @@ -69,15 +119,18 @@ impl PyStringRef { } } - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__getitem__")] + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__gt__")] + fn gt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) } else { @@ -85,7 +138,8 @@ impl PyStringRef { } } - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ge__")] + fn ge(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) } else { @@ -93,7 +147,8 @@ impl PyStringRef { } } - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lt__")] + fn lt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) } else { @@ -101,7 +156,8 @@ impl PyStringRef { } } - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__le__")] + fn le(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) } else { @@ -109,17 +165,20 @@ impl PyStringRef { } } - fn hash(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> usize { let mut hasher = std::collections::hash_map::DefaultHasher::new(); self.value.hash(&mut hasher); hasher.finish() as usize } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__mul__")] + fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&val, &vm.ctx.int_type()) { let value = &self.value; let multiplier = objint::get_value(&val).to_i32().unwrap(); @@ -133,11 +192,13 @@ impl PyStringRef { } } - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + #[pymethod(name = "__str__")] + fn str(zelf: PyRef, _vm: &VirtualMachine) -> PyStringRef { + zelf } - fn repr(self, _vm: &VirtualMachine) -> String { + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' @@ -167,27 +228,32 @@ impl PyStringRef { formatted } - fn lower(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn lower(&self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn casefold(&self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } - fn upper(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn upper(&self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } - fn capitalize(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn capitalize(&self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } + #[pymethod] fn split( - self, - pattern: OptionalArg, + &self, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { @@ -206,9 +272,10 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn rsplit( - self, - pattern: OptionalArg, + &self, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { @@ -227,20 +294,24 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn strip(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn strip(&self, _vm: &VirtualMachine) -> String { self.value.trim().to_string() } - fn lstrip(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn lstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_start().to_string() } - fn rstrip(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn rstrip(&self, _vm: &VirtualMachine) -> String { self.value.trim_end().to_string() } + #[pymethod] fn endswith( - self, + &self, suffix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -253,8 +324,9 @@ impl PyStringRef { } } + #[pymethod] fn startswith( - self, + &self, prefix: PyStringRef, start: OptionalArg, end: OptionalArg, @@ -267,15 +339,18 @@ impl PyStringRef { } } - fn isalnum(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isalnum(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } - fn isnumeric(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isnumeric(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } - fn isdigit(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isdigit(&self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, @@ -291,7 +366,8 @@ impl PyStringRef { } } - fn isdecimal(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isdecimal(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -299,11 +375,66 @@ impl PyStringRef { } } - fn title(self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + #[pymethod] + fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + if args.args.is_empty() { + return Err(vm.new_type_error( + "descriptor 'format' of 'str' object needs an argument".to_string(), + )); + } + + let zelf = &args.args[0]; + if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { + let zelf_typ = zelf.class(); + let actual_type = vm.to_pystr(&zelf_typ)?; + return Err(vm.new_type_error(format!( + "descriptor 'format' requires a 'str' object but received a '{}'", + actual_type + ))); + } + let format_string_text = get_value(zelf); + match FormatString::from_str(format_string_text.as_str()) { + Ok(format_string) => perform_format(vm, &format_string, &args), + Err(err) => match err { + FormatParseError::UnmatchedBracket => { + Err(vm.new_value_error("expected '}' before end of string".to_string())) + } + _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), + }, + } } - fn swapcase(self, _vm: &VirtualMachine) -> String { + /// Return a titlecased version of the string where words start with an + /// uppercase character and the remaining characters are lowercase. + #[pymethod] + fn title(&self, _vm: &VirtualMachine) -> String { + let mut title = String::new(); + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_lowercase() { + if !previous_is_cased { + title.extend(c.to_uppercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else if c.is_uppercase() { + if previous_is_cased { + title.extend(c.to_lowercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else { + previous_is_cased = false; + title.push(c); + } + } + title + } + + #[pymethod] + fn swapcase(&self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char @@ -318,14 +449,16 @@ impl PyStringRef { swapped_str } - fn isalpha(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isalpha(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn replace( - self, - old: Self, - new: Self, + &self, + old: PyStringRef, + new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { @@ -337,11 +470,13 @@ impl PyStringRef { // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isspace(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isupper(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -350,7 +485,8 @@ impl PyStringRef { .all(char::is_uppercase) } - fn islower(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn islower(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -359,12 +495,14 @@ impl PyStringRef { .all(char::is_lowercase) } - fn isascii(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isascii(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn splitlines(&self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value .split('\n') @@ -373,7 +511,8 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn join(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { @@ -387,9 +526,10 @@ impl PyStringRef { Ok(joined) } + #[pymethod] fn find( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -405,9 +545,10 @@ impl PyStringRef { } } + #[pymethod] fn rfind( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -423,9 +564,10 @@ impl PyStringRef { } } + #[pymethod] fn index( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, @@ -441,9 +583,10 @@ impl PyStringRef { } } + #[pymethod] fn rindex( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, @@ -459,7 +602,8 @@ impl PyStringRef { } } - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn partition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -477,7 +621,8 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn rpartition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -496,17 +641,40 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn istitle(self, _vm: &VirtualMachine) -> bool { + /// Return `true` if the sequence is ASCII titlecase and the sequence is not + /// empty, `false` otherwise. + #[pymethod] + fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { - false - } else { - self.value.split(' ').all(|word| word == make_title(word)) + return false; } + + let mut cased = false; + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_uppercase() { + if previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else if c.is_lowercase() { + if !previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else { + previous_is_cased = false; + } + } + cased } + #[pymethod] fn count( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -519,7 +687,8 @@ impl PyStringRef { } } - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + #[pymethod] + fn zfill(&self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { value.to_string() @@ -528,7 +697,10 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + fn get_fill_char<'a>( + rep: &'a OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<&'a str> { let rep_str = match rep { OptionalArg::Present(ref st) => &st.value, OptionalArg::Missing => " ", @@ -542,23 +714,55 @@ impl PyStringRef { } } - fn ljust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn ljust( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", value, rep_char.repeat(len))) } - fn rjust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn rjust( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; + let rep_char = Self::get_fill_char(&rep, vm)?; Ok(format!("{}{}", rep_char.repeat(len), value)) } - fn center(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn center( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; - let left_buff: usize = (len - value.len()) / 2; - let right_buff = len - value.len() - left_buff; + let rep_char = Self::get_fill_char(&rep, vm)?; + let value_len = self.value.chars().count(); + + if len <= value_len { + return Ok(value.to_string()); + } + let diff: usize = len - value_len; + let mut left_buff: usize = diff / 2; + let mut right_buff: usize = left_buff; + + if diff % 2 != 0 && value_len % 2 == 0 { + left_buff += 1 + } + + if diff % 2 != 0 && value_len % 2 != 0 { + right_buff += 1 + } Ok(format!( "{}{}{}", rep_char.repeat(left_buff), @@ -567,7 +771,8 @@ impl PyStringRef { )) } - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + #[pymethod] + fn expandtabs(&self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); let mut expanded_str = String::new(); let mut tab_size = tab_stop; @@ -590,7 +795,8 @@ impl PyStringRef { expanded_str } - fn isidentifier(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isidentifier(&self, _vm: &VirtualMachine) -> bool { let value = &self.value; // a string is not an identifier if it has whitespace or starts with a number if !value.chars().any(|c| c.is_ascii_whitespace()) @@ -620,78 +826,20 @@ impl IntoPyObject for String { } } -#[rustfmt::skip] // to avoid line splitting -pub fn init(context: &PyContext) { - let str_type = &context.str_type; - let str_doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'."; +impl IntoPyObject for &str { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.to_string())) + } +} - extend_class!(context, str_type, { - "__add__" => context.new_rustfunc(PyStringRef::add), - "__bool__" => context.new_rustfunc(PyStringRef::bool), - "__contains__" => context.new_rustfunc(PyStringRef::contains), - "__doc__" => context.new_str(str_doc.to_string()), - "__eq__" => context.new_rustfunc(PyStringRef::eq), - "__ge__" => context.new_rustfunc(PyStringRef::ge), - "__getitem__" => context.new_rustfunc(PyStringRef::getitem), - "__gt__" => context.new_rustfunc(PyStringRef::gt), - "__hash__" => context.new_rustfunc(PyStringRef::hash), - "__lt__" => context.new_rustfunc(PyStringRef::lt), - "__le__" => context.new_rustfunc(PyStringRef::le), - "__len__" => context.new_rustfunc(PyStringRef::len), - "__mul__" => context.new_rustfunc(PyStringRef::mul), - "__new__" => context.new_rustfunc(str_new), - "__repr__" => context.new_rustfunc(PyStringRef::repr), - "__str__" => context.new_rustfunc(PyStringRef::str), - "capitalize" => context.new_rustfunc(PyStringRef::capitalize), - "casefold" => context.new_rustfunc(PyStringRef::casefold), - "center" => context.new_rustfunc(PyStringRef::center), - "count" => context.new_rustfunc(PyStringRef::count), - "endswith" => context.new_rustfunc(PyStringRef::endswith), - "expandtabs" => context.new_rustfunc(PyStringRef::expandtabs), - "find" => context.new_rustfunc(PyStringRef::find), - "format" => context.new_rustfunc(str_format), - "index" => context.new_rustfunc(PyStringRef::index), - "isalnum" => context.new_rustfunc(PyStringRef::isalnum), - "isalpha" => context.new_rustfunc(PyStringRef::isalpha), - "isascii" => context.new_rustfunc(PyStringRef::isascii), - "isdecimal" => context.new_rustfunc(PyStringRef::isdecimal), - "isdigit" => context.new_rustfunc(PyStringRef::isdigit), - "isidentifier" => context.new_rustfunc(PyStringRef::isidentifier), - "islower" => context.new_rustfunc(PyStringRef::islower), - "isnumeric" => context.new_rustfunc(PyStringRef::isnumeric), - "isspace" => context.new_rustfunc(PyStringRef::isspace), - "isupper" => context.new_rustfunc(PyStringRef::isupper), - "istitle" => context.new_rustfunc(PyStringRef::istitle), - "join" => context.new_rustfunc(PyStringRef::join), - "lower" => context.new_rustfunc(PyStringRef::lower), - "ljust" => context.new_rustfunc(PyStringRef::ljust), - "lstrip" => context.new_rustfunc(PyStringRef::lstrip), - "partition" => context.new_rustfunc(PyStringRef::partition), - "replace" => context.new_rustfunc(PyStringRef::replace), - "rfind" => context.new_rustfunc(PyStringRef::rfind), - "rindex" => context.new_rustfunc(PyStringRef::rindex), - "rjust" => context.new_rustfunc(PyStringRef::rjust), - "rpartition" => context.new_rustfunc(PyStringRef::rpartition), - "rsplit" => context.new_rustfunc(PyStringRef::rsplit), - "rstrip" => context.new_rustfunc(PyStringRef::rstrip), - "split" => context.new_rustfunc(PyStringRef::split), - "splitlines" => context.new_rustfunc(PyStringRef::splitlines), - "startswith" => context.new_rustfunc(PyStringRef::startswith), - "strip" => context.new_rustfunc(PyStringRef::strip), - "swapcase" => context.new_rustfunc(PyStringRef::swapcase), - "title" => context.new_rustfunc(PyStringRef::title), - "upper" => context.new_rustfunc(PyStringRef::upper), - "zfill" => context.new_rustfunc(PyStringRef::zfill), - }); +impl IntoPyObject for &String { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.clone())) + } +} + +pub fn init(ctx: &PyContext) { + PyString::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { @@ -706,34 +854,6 @@ fn count_char(s: &str, c: char) -> usize { s.chars().filter(|x| *x == c).count() } -fn str_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - return Err( - vm.new_type_error("descriptor 'format' of 'str' object needs an argument".to_string()) - ); - } - - let zelf = &args.args[0]; - if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { - let zelf_typ = zelf.class(); - let actual_type = vm.to_pystr(&zelf_typ)?; - return Err(vm.new_type_error(format!( - "descriptor 'format' requires a 'str' object but received a '{}'", - actual_type - ))); - } - let format_string_text = get_value(zelf); - match FormatString::from_str(format_string_text.as_str()) { - Ok(format_string) => perform_format(vm, &format_string, &args), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_string())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), - }, - } -} - fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { let returned_type = vm.ctx.new_str(format_spec.to_string()); let result = vm.call_method(&argument, "__format__", vec![returned_type])?; @@ -797,26 +917,6 @@ fn perform_format( Ok(vm.ctx.new_str(final_string)) } -// TODO: should with following format -// class str(object='') -// class str(object=b'', encoding='utf-8', errors='strict') -fn str_new( - cls: PyClassRef, - object: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let string = match object { - OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), - OptionalArg::Missing => vm.new_str("".to_string()), - }; - if string.class().is(&cls) { - TryFromObject::try_from_object(vm, string) - } else { - let payload = string.payload::().unwrap(); - payload.clone().into_ref_with_type(vm, cls) - } -} - impl PySliceableSequence for String { fn do_slice(&self, range: Range) -> Self { to_graphemes(self) @@ -934,22 +1034,55 @@ fn adjust_indices( } } -// helper function to title strings -fn make_title(s: &str) -> String { - let mut titled_str = String::new(); - let mut capitalize_char: bool = true; - for c in s.chars() { - if c.is_alphabetic() { - if !capitalize_char { - titled_str.push(c); - } else if capitalize_char { - titled_str.push(c.to_ascii_uppercase()); - capitalize_char = false; - } - } else { - titled_str.push(c); - capitalize_char = true; +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn str_title() { + let vm = VirtualMachine::new(); + + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyString::from(input).title(&vm).as_str(), title); + } + } + + #[test] + fn str_istitle() { + let vm = VirtualMachine::new(); + + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + ]; + + for s in pos { + assert!(PyString::from(s).istitle(&vm)); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyString::from(s).istitle(&vm)); } } - titled_str } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 24424d265..194212a49 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -12,7 +12,7 @@ use crate::obj::objfunction::PyMethod; use crate::obj::objstr; use crate::obj::objtype::{PyClass, PyClassRef}; use crate::pyobject::{ - DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -124,11 +124,13 @@ fn super_new( } else { let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg) { + match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index a199e34b1..216945284 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -3,13 +3,12 @@ use std::fmt; use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, -}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; use super::objint; +use super::objiter; use super::objsequence::{ get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; @@ -140,10 +139,10 @@ impl PyTupleRef { Ok(hasher.finish()) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { + PyTupleIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + tuple: self, } } @@ -225,6 +224,36 @@ fn tuple_new( PyTuple::from(elements).into_ref_with_type(vm, cls) } +#[derive(Debug)] +pub struct PyTupleIterator { + position: Cell, + tuple: PyTupleRef, +} + +impl PyValue for PyTupleIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.tupleiterator_type() + } +} + +type PyTupleIteratorRef = PyRef; + +impl PyTupleIteratorRef { + fn next(self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.tuple.elements.borrow().len() { + let ret = self.tuple.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let tuple_type = &context.tuple_type; @@ -252,4 +281,10 @@ If the argument is a tuple, the return value is the same object."; "__doc__" => context.new_str(tuple_doc.to_string()), "index" => context.new_rustfunc(PyTupleRef::index) }); + + let tupleiterator_type = &context.tupleiterator_type; + extend_class!(context, tupleiterator_type, { + "__next__" => context.new_rustfunc(PyTupleIteratorRef::next), + "__iter__" => context.new_rustfunc(PyTupleIteratorRef::iter), + }); } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index 14958300b..819f85088 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -9,7 +9,6 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -use super::objdict; use super::objdict::PyDictRef; use super::objlist::PyList; use super::objproperty::PropertyBuilder; @@ -163,6 +162,13 @@ impl PyClassRef { Ok(()) } + // This is used for class initialisation where the vm is not yet available. + pub fn set_str_attr>(&self, attr_name: &str, value: V) { + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value.into()); + } + fn subclasses(self, _vm: &VirtualMachine) -> PyList { let mut subclasses = self.subclasses.borrow_mut(); subclasses.retain(|x| x.upgrade().is_some()); @@ -229,7 +235,7 @@ pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { let (typ, name, bases, dict) = args.bind(vm)?; - type_new_class(vm, typ, name, bases, dict).map(|x| x.into_object()) + type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { Err(vm.new_type_error(format!(": type_new: {:?}", args))) } @@ -244,12 +250,7 @@ pub fn type_new_class( ) -> PyResult { let mut bases: Vec = bases.iter(vm)?.collect::, _>>()?; bases.push(vm.ctx.object()); - new( - typ.clone(), - &name.value, - bases, - objdict::py_dict_to_attributes(dict.as_object()), - ) + new(typ.clone(), &name.value, bases, dict.to_attributes()) } pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs new file mode 100644 index 000000000..735670f4e --- /dev/null +++ b/vm/src/obj/objweakproxy.rs @@ -0,0 +1,50 @@ +use super::objweakref::PyWeak; +use crate::function::OptionalArg; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[derive(Debug)] +pub struct PyWeakProxy { + weak: PyWeak, +} + +impl PyValue for PyWeakProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.weakproxy_type() + } +} + +pub type PyWeakProxyRef = PyRef; + +impl PyWeakProxyRef { + // TODO callbacks + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyWeakProxy { + weak: PyWeak::downgrade(&referent), + } + .into_ref_with_type(vm, cls) + } + + fn getattr(self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.get_attribute(obj, attr_name), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } +} + +pub fn init(context: &PyContext) { + extend_class!(context, &context.weakproxy_type, { + "__new__" => context.new_rustfunc(PyWeakProxyRef::create), + "__getattr__" => context.new_rustfunc(PyWeakProxyRef::getattr), + }); +} diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index 685e32b70..b3b7e03c9 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -1,3 +1,4 @@ +use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; use crate::pyobject::PyValue; use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; @@ -32,7 +33,12 @@ pub type PyWeakRef = PyRef; impl PyWeakRef { // TODO callbacks - fn create(cls: PyClassRef, referent: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { PyWeak::downgrade(&referent).into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index a12152eee..81640a724 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -40,13 +40,17 @@ impl PyZipRef { Ok(vm.ctx.new_tuple(next_objs)) } } + + fn iter(self, _vm: &VirtualMachine) -> Self { + self + } } pub fn init(context: &PyContext) { let zip_type = &context.zip_type; - objiter::iter_type_init(context, zip_type); extend_class!(context, zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(PyZipRef::next) + "__next__" => context.new_rustfunc(PyZipRef::next), + "__iter__" => context.new_rustfunc(PyZipRef::iter), }); } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index aa3348be5..fdac31b83 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,5 +1,6 @@ use std::any::Any; -use std::cell::{Cell, RefCell}; +use std::cell::Cell; +use std::cell::RefCell; use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; @@ -48,8 +49,9 @@ use crate::obj::objslice; use crate::obj::objstaticmethod; use crate::obj::objstr; use crate::obj::objsuper; -use crate::obj::objtuple::{self, PyTuple}; +use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; +use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; @@ -86,7 +88,6 @@ pub type PyAttributes = HashMap; impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { let type_name = self.class().name.clone(); // We don't have access to a vm, so just assume that if its parent's name @@ -108,7 +109,9 @@ impl fmt::Display for PyObject { #[derive(Debug)] pub struct PyContext { pub bytes_type: PyClassRef, + pub bytesiterator_type: PyClassRef, pub bytearray_type: PyClassRef, + pub bytearrayiterator_type: PyClassRef, pub bool_type: PyClassRef, pub classmethod_type: PyClassRef, pub code_type: PyClassRef, @@ -126,17 +129,26 @@ pub struct PyContext { pub true_value: PyIntRef, pub false_value: PyIntRef, pub list_type: PyClassRef, + pub listiterator_type: PyClassRef, + pub dictkeyiterator_type: PyClassRef, + pub dictvalueiterator_type: PyClassRef, + pub dictitemiterator_type: PyClassRef, + pub dictkeys_type: PyClassRef, + pub dictvalues_type: PyClassRef, + pub dictitems_type: PyClassRef, pub map_type: PyClassRef, pub memoryview_type: PyClassRef, pub none: PyNoneRef, pub ellipsis: PyEllipsisRef, pub not_implemented: PyNotImplementedRef, pub tuple_type: PyClassRef, + pub tupleiterator_type: PyClassRef, pub set_type: PyClassRef, pub staticmethod_type: PyClassRef, pub super_type: PyClassRef, pub str_type: PyClassRef, pub range_type: PyClassRef, + pub rangeiterator_type: PyClassRef, pub slice_type: PyClassRef, pub type_type: PyClassRef, pub zip_type: PyClassRef, @@ -147,6 +159,7 @@ pub struct PyContext { pub module_type: PyClassRef, pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, + pub weakproxy_type: PyClassRef, pub object: PyClassRef, pub exceptions: exceptions::ExceptionZoo, } @@ -244,10 +257,18 @@ impl PyContext { let readonly_property_type = create_type("readonly_property", &type_type, &object_type); let super_type = create_type("super", &type_type, &object_type); let weakref_type = create_type("ref", &type_type, &object_type); + let weakproxy_type = create_type("weakproxy", &type_type, &object_type); let generator_type = create_type("generator", &type_type, &object_type); let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); + let listiterator_type = create_type("list_iterator", &type_type, &object_type); + let dictkeys_type = create_type("dict_keys", &type_type, &object_type); + let dictvalues_type = create_type("dict_values", &type_type, &object_type); + let dictitems_type = create_type("dict_items", &type_type, &object_type); + let dictkeyiterator_type = create_type("dict_keyiterator", &type_type, &object_type); + let dictvalueiterator_type = create_type("dict_valueiterator", &type_type, &object_type); + let dictitemiterator_type = create_type("dict_itemiterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -255,8 +276,11 @@ impl PyContext { let frame_type = create_type("frame", &type_type, &object_type); let complex_type = create_type("complex", &type_type, &object_type); let bytes_type = create_type("bytes", &type_type, &object_type); + let bytesiterator_type = create_type("bytes_iterator", &type_type, &object_type); let bytearray_type = create_type("bytearray", &type_type, &object_type); + let bytearrayiterator_type = create_type("bytearray_iterator", &type_type, &object_type); let tuple_type = create_type("tuple", &type_type, &object_type); + let tupleiterator_type = create_type("tuple_iterator", &type_type, &object_type); let iter_type = create_type("iter", &type_type, &object_type); let enumerate_type = create_type("enumerate", &type_type, &object_type); let filter_type = create_type("filter", &type_type, &object_type); @@ -266,6 +290,7 @@ impl PyContext { let memoryview_type = create_type("memoryview", &type_type, &object_type); let code_type = create_type("code", &type_type, &int_type); let range_type = create_type("range", &type_type, &object_type); + let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); let exceptions = exceptions::ExceptionZoo::new(&type_type, &object_type); @@ -291,7 +316,9 @@ impl PyContext { bool_type, memoryview_type, bytearray_type, + bytearrayiterator_type, bytes_type, + bytesiterator_type, code_type, complex_type, classmethod_type, @@ -300,11 +327,19 @@ impl PyContext { frame_type, staticmethod_type, list_type, + listiterator_type, + dictkeys_type, + dictvalues_type, + dictitems_type, + dictkeyiterator_type, + dictvalueiterator_type, + dictitemiterator_type, set_type, frozenset_type, true_value, false_value, tuple_type, + tupleiterator_type, iter_type, ellipsis_type, enumerate_type, @@ -317,6 +352,7 @@ impl PyContext { not_implemented, str_type, range_type, + rangeiterator_type, slice_type, object: object_type, function_type, @@ -328,6 +364,7 @@ impl PyContext { module_type, bound_method_type, weakref_type, + weakproxy_type, type_type, exceptions, }; @@ -363,6 +400,7 @@ impl PyContext { objcode::init(&context); objframe::init(&context); objweakref::init(&context); + objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); exceptions::init(&context); @@ -373,10 +411,18 @@ impl PyContext { self.bytearray_type.clone() } + pub fn bytearrayiterator_type(&self) -> PyClassRef { + self.bytearrayiterator_type.clone() + } + pub fn bytes_type(&self) -> PyClassRef { self.bytes_type.clone() } + pub fn bytesiterator_type(&self) -> PyClassRef { + self.bytesiterator_type.clone() + } + pub fn code_type(&self) -> PyClassRef { self.code_type.clone() } @@ -405,6 +451,10 @@ impl PyContext { self.list_type.clone() } + pub fn listiterator_type(&self) -> PyClassRef { + self.listiterator_type.clone() + } + pub fn module_type(&self) -> PyClassRef { self.module_type.clone() } @@ -417,6 +467,10 @@ impl PyContext { self.range_type.clone() } + pub fn rangeiterator_type(&self) -> PyClassRef { + self.rangeiterator_type.clone() + } + pub fn slice_type(&self) -> PyClassRef { self.slice_type.clone() } @@ -437,6 +491,10 @@ impl PyContext { self.tuple_type.clone() } + pub fn tupleiterator_type(&self) -> PyClassRef { + self.tupleiterator_type.clone() + } + pub fn iter_type(&self) -> PyClassRef { self.iter_type.clone() } @@ -501,6 +559,10 @@ impl PyContext { self.weakref_type.clone() } + pub fn weakproxy_type(&self) -> PyClassRef { + self.weakproxy_type.clone() + } + pub fn type_type(&self) -> PyClassRef { self.type_type.clone() } @@ -623,10 +685,11 @@ impl PyContext { &self, code_obj: PyCodeRef, scope: Scope, - defaults: PyObjectRef, + defaults: Option, + kw_only_defaults: Option, ) -> PyObjectRef { PyObject::new( - PyFunction::new(code_obj, scope, defaults), + PyFunction::new(code_obj, scope, defaults, kw_only_defaults), self.function_type(), Some(self.new_dict()), ) @@ -643,40 +706,12 @@ impl PyContext { pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ: class, - dict: dict, + dict, payload: objobject::PyInstance, } .into_ref() } - // Item set/get: - pub fn set_item(&self, obj: &PyObjectRef, key: &str, v: PyObjectRef) { - if let Some(dict) = obj.payload::() { - let key = self.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - unimplemented!() - }; - } - - pub fn set_attr<'a, T: Into<&'a PyObjectRef>, V: Into>( - &'a self, - obj: T, - attr_name: &str, - value: V, - ) { - let obj = obj.into(); - if let Some(PyClass { ref attributes, .. }) = obj.payload::() { - attributes - .borrow_mut() - .insert(attr_name.to_string(), value.into()); - } else if let Some(ref dict) = obj.dict { - dict.set_item(self, attr_name, value.into()); - } else { - unimplemented!("set_attr unimplemented for: {:?}", obj); - }; - } - pub fn unwrap_constant(&self, value: &bytecode::Constant) -> PyObjectRef { match *value { bytecode::Constant::Integer { ref value } => self.new_int(value.clone()), @@ -905,52 +940,49 @@ impl TypeProtocol for PyRef { } } -pub trait DictProtocol { - fn contains_key(&self, k: &str) -> bool; - fn get_item(&self, k: &str) -> Option; - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef); - fn del_item(&self, key: &str); +pub trait ItemProtocol { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item_option( + &self, + key: T, + vm: &VirtualMachine, + ) -> PyResult> { + match self.get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) => { + if objtype::isinstance(&exc, &vm.ctx.exceptions.key_error) { + Ok(None) + } else { + Err(exc) + } + } + } + } } -impl DictProtocol for PyObjectRef { - fn contains_key(&self, k: &str) -> bool { - if let Some(dict) = self.payload::() { - objdict::content_contains_key_str(&dict.entries.borrow(), k) - } else { - unimplemented!() - } +impl ItemProtocol for PyObjectRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } - fn get_item(&self, k: &str) -> Option { - if let Some(dict) = self.payload::() { - objdict::content_get_key_str(&dict.entries.borrow(), k) - } else { - panic!("TODO {:?}", k) - } + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - if self.payload_is::() { - objdict::get_key_value_pairs(self) - } else { - panic!("TODO") - } - } - - // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - if let Some(dict) = self.payload::() { - let key = ctx.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - panic!("TODO {:?}", self); - } - } - - fn del_item(&self, key: &str) { - let mut elements = objdict::get_mut_elements(self); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } @@ -1039,10 +1071,24 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - Ok(PyIterable { - method: vm.get_method(obj, "__iter__")?, - _item: std::marker::PhantomData, - }) + if let Ok(method) = vm.get_method(obj.clone(), "__iter__") { + Ok(PyIterable { + method, + _item: std::marker::PhantomData, + }) + } else if vm.get_method(obj.clone(), "__getitem__").is_ok() { + Self::try_from_object( + vm, + objiter::PySequenceIterator { + position: Cell::new(0), + obj: obj.clone(), + } + .into_ref(vm) + .into_object(), + ) + } else { + Err(vm.new_type_error(format!("'{}' object is not iterable", obj.class().name))) + } } } @@ -1127,31 +1173,12 @@ where } } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -#[derive(Debug)] -pub struct PyIteratorValue { - pub position: Cell, - pub iterated_obj: PyObjectRef, -} - -impl PyValue for PyIteratorValue { - fn class(vm: &VirtualMachine) -> PyClassRef { - vm.ctx.iter_type() - } -} - impl PyObject where T: Sized + PyObjectPayload, { pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { - PyObject { - typ, - dict: dict, - payload, - } - .into_ref() + PyObject { typ, dict, payload }.into_ref() } // Move this object into a reference object, transferring ownership. @@ -1173,6 +1200,8 @@ impl PyObject { } pub trait PyValue: fmt::Debug + Sized + 'static { + const HAVE_DICT: bool = false; + fn class(vm: &VirtualMachine) -> PyClassRef; fn into_ref(self, vm: &VirtualMachine) -> PyRef { @@ -1185,7 +1214,7 @@ pub trait PyValue: fmt::Debug + Sized + 'static { fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyClassRef) -> PyResult> { let class = Self::class(vm); if objtype::issubclass(&cls, &class) { - let dict = if cls.is(&class) { + let dict = if !Self::HAVE_DICT && cls.is(&class) { None } else { Some(vm.ctx.new_dict()) @@ -1259,6 +1288,36 @@ where } } +pub trait PyClassDef { + const NAME: &'static str; + const DOC: Option<&'static str> = None; +} + +impl PyClassDef for PyRef +where + T: PyClassDef, +{ + const NAME: &'static str = T::NAME; + const DOC: Option<&'static str> = T::DOC; +} + +pub trait PyClassImpl: PyClassDef { + fn impl_extend_class(ctx: &PyContext, class: &PyClassRef); + + fn extend_class(ctx: &PyContext, class: &PyClassRef) { + Self::impl_extend_class(ctx, class); + if let Some(doc) = Self::DOC { + class.set_str_attr("__doc__", ctx.new_str(doc.into())); + } + } + + fn make_class(ctx: &PyContext) -> PyClassRef { + let py_class = ctx.new_class(Self::NAME, ctx.object()); + Self::extend_class(ctx, &py_class); + py_class + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 0011407ab..816882f31 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -9,15 +9,15 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; -use crate::function::PyFuncArgs; -use crate::obj::objstr; +use crate::obj::objlist::PyListRef; +use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; #[derive(Debug)] struct AstNode; -// type AstNodeRef = PyRef; +type AstNodeRef = PyRef; impl PyValue for AstNode { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -25,72 +25,57 @@ impl PyValue for AstNode { } } -/* - * Idea: maybe we can create a sort of struct with some helper functions? -struct AstToPyAst { - ctx: &PyContext, -} - -impl AstToPyAst { - fn new(ctx: &PyContext) -> Self { - AstToPyAst { - ctx: ctx, +macro_rules! node { + ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { + { + let node = create_node($vm, stringify!($node_name))?; + $( + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value)?; + )* + Ok(node) } + }; + ( $vm: expr, $node_name:ident) => { + create_node($vm, stringify!($node_name)) } - } -*/ -fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyObjectRef { - let mut body = vec![]; - for statement in &program.statements { - body.push(statement_to_ast(&vm, statement)); - } - // TODO: create Module node: - // let ast_node = ctx.new_instance(this.Module); - let ast_node = create_node(vm, "program"); - let py_body = vm.ctx.new_list(body); - vm.ctx.set_attr(&ast_node, "body", py_body); - ast_node +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyResult { + let py_body = statements_to_ast(vm, &program.statements)?; + node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(vm: &VirtualMachine, _name: &str) -> PyObjectRef { - // TODO: instantiate a class of type given by name - // TODO: lookup in the current module? - PyObject::new(AstNode, AstNode::class(vm), Some(vm.ctx.new_dict())) +fn create_node(vm: &VirtualMachine, name: &str) -> PyResult { + AstNode.into_ref_with_type(vm, vm.class("ast", name)) } -fn statements_to_ast(vm: &VirtualMachine, statements: &[ast::LocatedStatement]) -> PyObjectRef { - let mut py_statements = vec![]; - for statement in statements { - py_statements.push(statement_to_ast(vm, statement)); - } - vm.ctx.new_list(py_statements) +fn statements_to_ast( + vm: &VirtualMachine, + statements: &[ast::LocatedStatement], +) -> PyResult { + let body: PyResult> = statements + .iter() + .map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object())) + .collect(); + Ok(vm.ctx.new_list(body?).downcast().unwrap()) } -fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> PyObjectRef { +fn statement_to_ast( + vm: &VirtualMachine, + statement: &ast::LocatedStatement, +) -> PyResult { let node = match &statement.node { ast::Statement::ClassDef { name, body, decorator_list, .. - } => { - let node = create_node(vm, "ClassDef"); - - // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); - node - } + } => node!(vm, ClassDef, { + name => vm.ctx.new_str(name.to_string()), + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + }), ast::Statement::FunctionDef { name, args, @@ -98,179 +83,116 @@ fn statement_to_ast(vm: &VirtualMachine, statement: &ast::LocatedStatement) -> P decorator_list, returns, } => { - let node = create_node(vm, "FunctionDef"); - - // Set name: - vm.ctx - .set_attr(&node, "name", vm.ctx.new_str(name.to_string())); - - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); - - // Set body: - let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_decorator_list = expressions_to_ast(vm, decorator_list); - vm.ctx.set_attr(&node, "decorator_list", py_decorator_list); - let py_returns = if let Some(hint) = returns { - expression_to_ast(vm, hint) + expression_to_ast(vm, hint)?.into_object() } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "returns", py_returns); - node + node!(vm, FunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + returns => py_returns + }) } - ast::Statement::Continue => create_node(vm, "Continue"), - ast::Statement::Break => create_node(vm, "Break"), - ast::Statement::Pass => create_node(vm, "Pass"), + ast::Statement::Continue => node!(vm, Continue), + ast::Statement::Break => node!(vm, Break), + ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { - let node = create_node(vm, "Pass"); - - vm.ctx.set_attr(&node, "test", expression_to_ast(vm, test)); - let py_msg = match msg { - Some(msg) => expression_to_ast(vm, msg), + Some(msg) => expression_to_ast(vm, msg)?.into_object(), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "msg", py_msg); - - node + node!(vm, Assert, { + test => expression_to_ast(vm, test)?, + msg => py_msg + }) } ast::Statement::Delete { targets } => { - let node = create_node(vm, "Delete"); - - let py_targets = vm - .ctx - .new_tuple(targets.iter().map(|v| expression_to_ast(vm, v)).collect()); - vm.ctx.set_attr(&node, "targets", py_targets); - - node + let targets: PyResult<_> = targets + .iter() + .map(|v| Ok(expression_to_ast(vm, v)?.into_object())) + .collect(); + let py_targets = vm.ctx.new_tuple(targets?); + node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { - let node = create_node(vm, "Return"); - let py_value = if let Some(value) = value { - vm.ctx - .new_tuple(value.iter().map(|v| expression_to_ast(vm, v)).collect()) + expression_to_ast(vm, value)?.into_object() } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "value", py_value); - node - } - ast::Statement::If { test, body, orelse } => { - let node = create_node(vm, "If"); - - let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); - - let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) - } else { - vm.ctx.none() - }; - vm.ctx.set_attr(&node, "orelse", py_orelse); - - node + node!(vm, Return, { + value => py_value + }) } + ast::Statement::If { test, body, orelse } => node!(vm, If, { + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() + } else { + vm.ctx.none() + } + }), ast::Statement::For { target, iter, body, orelse, - } => { - let node = create_node(vm, "For"); - - let py_target = expression_to_ast(vm, target); - vm.ctx.set_attr(&node, "target", py_target); - - let py_iter = expressions_to_ast(vm, iter); - vm.ctx.set_attr(&node, "iter", py_iter); - - let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } => node!(vm, For, { + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() - }; - vm.ctx.set_attr(&node, "orelse", py_orelse); - - node - } - ast::Statement::While { test, body, orelse } => { - let node = create_node(vm, "While"); - - let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); - - let py_body = statements_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(vm, orelse) + } + }), + ast::Statement::While { test, body, orelse } => node!(vm, While, { + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() } else { vm.ctx.none() - }; - vm.ctx.set_attr(&node, "orelse", py_orelse); - - node - } - ast::Statement::Expression { expression } => { - let node = create_node(vm, "Expr"); - - let value = expression_to_ast(vm, expression); - vm.ctx.set_attr(&node, "value", value); - - node - } + } + }), + ast::Statement::Expression { expression } => node!(vm, Expr, { + value => expression_to_ast(vm, expression)? + }), x => { - unimplemented!("{:?}", x); + return Err(vm.new_type_error(format!("Ast not implemented: {:?}", x))); } - }; + }?; // set lineno on node: let lineno = vm.ctx.new_int(statement.location.get_row()); - vm.ctx.set_attr(&node, "lineno", lineno); + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyObjectRef { - let mut py_expression_nodes = vec![]; - for expression in expressions { - py_expression_nodes.push(expression_to_ast(vm, expression)); - } - vm.ctx.new_list(py_expression_nodes) +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyResult { + let py_expression_nodes: PyResult<_> = expressions + .iter() + .map(|expression| Ok(expression_to_ast(vm, expression)?.into_object())) + .collect(); + Ok(vm.ctx.new_list(py_expression_nodes?).downcast().unwrap()) } -fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObjectRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult { let node = match &expression { - ast::Expression::Call { function, args, .. } => { - let node = create_node(vm, "Call"); - - let py_func_ast = expression_to_ast(vm, function); - vm.ctx.set_attr(&node, "func", py_func_ast); - - let py_args = expressions_to_ast(vm, args); - vm.ctx.set_attr(&node, "args", py_args); - - node - } + ast::Expression::Call { function, args, .. } => node!(vm, Call, { + func => expression_to_ast(vm, function)?, + args => expressions_to_ast(vm, args)?, + }), ast::Expression::Binop { a, op, b } => { - let node = create_node(vm, "BinOp"); - - let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "left", py_a); - // Operator: - let str_op = match op { + let op = match op { ast::Operator::Add => "Add", ast::Operator::Sub => "Sub", ast::Operator::Mult => "Mult", @@ -285,56 +207,46 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Operator::BitAnd => "BitAnd", ast::Operator::FloorDiv => "FloorDiv", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); - - let py_b = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "right", py_b); - node + node!(vm, BinOp, { + left => expression_to_ast(vm, a)?, + op => vm.ctx.new_str(op.to_string()), + right => expression_to_ast(vm, b)?, + }) } ast::Expression::Unop { op, a } => { - let node = create_node(vm, "UnaryOp"); - - let str_op = match op { + let op = match op { ast::UnaryOperator::Not => "Not", ast::UnaryOperator::Inv => "Invert", ast::UnaryOperator::Neg => "USub", ast::UnaryOperator::Pos => "UAdd", }; - let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); - - let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "operand", py_a); - - node + node!(vm, UnaryOp, { + op => vm.ctx.new_str(op.to_string()), + operand => expression_to_ast(vm, a)?, + }) } ast::Expression::BoolOp { a, op, b } => { - let node = create_node(vm, "BoolOp"); - // Attach values: - let py_a = expression_to_ast(vm, a); - let py_b = expression_to_ast(vm, b); + let py_a = expression_to_ast(vm, a)?.into_object(); + let py_b = expression_to_ast(vm, b)?.into_object(); let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); - vm.ctx.set_attr(&node, "values", py_values); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; let py_op = vm.ctx.new_str(str_op.to_string()); - vm.ctx.set_attr(&node, "op", py_op); - node + node!(vm, BoolOp, { + op => py_op, + values => py_values, + }) } - ast::Expression::Compare { a, op, b } => { - let node = create_node(vm, "Compare"); - - let py_a = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "left", py_a); + ast::Expression::Compare { vals, ops } => { + let left = expression_to_ast(vm, &vals[0])?; // Operator: - let str_op = match op { + let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => "Eq", ast::Comparison::NotEqual => "NotEq", ast::Comparison::Less => "Lt", @@ -346,48 +258,37 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = vm.ctx.new_list(vec![vm.ctx.new_str(str_op.to_string())]); - vm.ctx.set_attr(&node, "ops", py_ops); + let ops = vm.ctx.new_list( + ops.iter() + .map(|x| vm.ctx.new_str(to_operator(x).to_string())) + .collect(), + ); - let py_b = vm.ctx.new_list(vec![expression_to_ast(vm, b)]); - vm.ctx.set_attr(&node, "comparators", py_b); - node - } - ast::Expression::Identifier { name } => { - let node = create_node(vm, "Identifier"); - - // Id: - let py_name = vm.ctx.new_str(name.clone()); - vm.ctx.set_attr(&node, "id", py_name); - node - } - ast::Expression::Lambda { args, body } => { - let node = create_node(vm, "Lambda"); - - vm.ctx.set_attr(&node, "args", parameters_to_ast(vm, args)); - - let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - node - } - ast::Expression::IfExpression { test, body, orelse } => { - let node = create_node(vm, "IfExp"); - - let py_test = expression_to_ast(vm, test); - vm.ctx.set_attr(&node, "test", py_test); - - let py_body = expression_to_ast(vm, body); - vm.ctx.set_attr(&node, "body", py_body); - - let py_orelse = expression_to_ast(vm, orelse); - vm.ctx.set_attr(&node, "orelse", py_orelse); - - node + let comparators: PyResult<_> = vals + .iter() + .skip(1) + .map(|x| Ok(expression_to_ast(vm, x)?.into_object())) + .collect(); + let comparators = vm.ctx.new_list(comparators?); + node!(vm, Compare, { + left => left, + ops => ops, + comparators => comparators, + }) } + ast::Expression::Identifier { name } => node!(vm, Identifier, { + id => vm.ctx.new_str(name.clone()) + }), + ast::Expression::Lambda { args, body } => node!(vm, Lambda, { + args => parameters_to_ast(vm, args)?, + body => expression_to_ast(vm, body)?, + }), + ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { + text => expression_to_ast(vm, test)?, + body => expression_to_ast(vm, body)?, + or_else => expression_to_ast(vm, orelse)?, + }), ast::Expression::Number { value } => { - let node = create_node(vm, "Num"); - let py_n = match value { ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), ast::Number::Float { value } => vm.ctx.new_float(*value), @@ -395,256 +296,216 @@ fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyObj vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - vm.ctx.set_attr(&node, "n", py_n); - - node - } - ast::Expression::True => { - let node = create_node(vm, "NameConstant"); - - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(true)); - - node - } - ast::Expression::False => { - let node = create_node(vm, "NameConstant"); - - vm.ctx.set_attr(&node, "value", vm.ctx.new_bool(false)); - - node - } - ast::Expression::None => { - let node = create_node(vm, "NameConstant"); - - vm.ctx.set_attr(&node, "value", vm.ctx.none()); - - node - } - ast::Expression::Ellipsis => create_node(vm, "Ellipsis"), - ast::Expression::List { elements } => { - let node = create_node(vm, "List"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); - - node - } - ast::Expression::Tuple { elements } => { - let node = create_node(vm, "Tuple"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); - - node - } - ast::Expression::Set { elements } => { - let node = create_node(vm, "Set"); - - let elts = elements.iter().map(|e| expression_to_ast(vm, e)).collect(); - let py_elts = vm.ctx.new_list(elts); - vm.ctx.set_attr(&node, "elts", py_elts); - - node + node!(vm, Num, { + n => py_n + }) } + ast::Expression::True => node!(vm, NameConstant, { + value => vm.ctx.new_bool(true) + }), + ast::Expression::False => node!(vm, NameConstant, { + value => vm.ctx.new_bool(false) + }), + ast::Expression::None => node!(vm, NameConstant, { + value => vm.ctx.none() + }), + ast::Expression::Ellipsis => node!(vm, Ellipsis), + ast::Expression::List { elements } => node!(vm, List, { + elts => expressions_to_ast(vm, &elements)? + }), + ast::Expression::Tuple { elements } => node!(vm, Tuple, { + elts => expressions_to_ast(vm, &elements)? + }), + ast::Expression::Set { elements } => node!(vm, Set, { + elts => expressions_to_ast(vm, &elements)? + }), ast::Expression::Dict { elements } => { - let node = create_node(vm, "Dict"); - let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(vm, k)); - values.push(expression_to_ast(vm, v)); + keys.push(expression_to_ast(vm, k)?.into_object()); + values.push(expression_to_ast(vm, v)?.into_object()); } - let py_keys = vm.ctx.new_list(keys); - vm.ctx.set_attr(&node, "keys", py_keys); - - let py_values = vm.ctx.new_list(values); - vm.ctx.set_attr(&node, "values", py_values); - - node + node!(vm, Dict, { + keys => vm.ctx.new_list(keys), + values => vm.ctx.new_list(values), + }) } ast::Expression::Comprehension { kind, generators } => { - let node = match kind.deref() { + let py_generators = map_ast(comprehension_to_ast, vm, generators)?; + + match kind.deref() { ast::ComprehensionKind::GeneratorExpression { .. } => { - create_node(vm, "GeneratorExp") + node!(vm, GeneratorExp, {generators => py_generators}) } - ast::ComprehensionKind::List { .. } => create_node(vm, "ListComp"), - ast::ComprehensionKind::Set { .. } => create_node(vm, "SetComp"), - ast::ComprehensionKind::Dict { .. } => create_node(vm, "DictComp"), - }; - - let g = generators - .iter() - .map(|g| comprehension_to_ast(vm, g)) - .collect(); - let py_generators = vm.ctx.new_list(g); - vm.ctx.set_attr(&node, "generators", py_generators); - - node + ast::ComprehensionKind::List { .. } => { + node!(vm, ListComp, {generators => py_generators}) + } + ast::ComprehensionKind::Set { .. } => { + node!(vm, SetComp, {generators => py_generators}) + } + ast::ComprehensionKind::Dict { .. } => { + node!(vm, DictComp, {generators => py_generators}) + } + } } ast::Expression::Yield { value } => { - let node = create_node(vm, "Yield"); - let py_value = match value { - Some(value) => expression_to_ast(vm, value), + Some(value) => expression_to_ast(vm, value)?.into_object(), None => vm.ctx.none(), }; - vm.ctx.set_attr(&node, "value", py_value); - - node + node!(vm, Yield, { + value => py_value + }) } ast::Expression::YieldFrom { value } => { - let node = create_node(vm, "YieldFrom"); - - let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); - - node - } - ast::Expression::Subscript { a, b } => { - let node = create_node(vm, "Subscript"); - - let py_value = expression_to_ast(vm, a); - vm.ctx.set_attr(&node, "value", py_value); - - let py_slice = expression_to_ast(vm, b); - vm.ctx.set_attr(&node, "slice", py_slice); - - node - } - ast::Expression::Attribute { value, name } => { - let node = create_node(vm, "Attribute"); - - let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); - - let py_attr = vm.ctx.new_str(name.to_string()); - vm.ctx.set_attr(&node, "attr", py_attr); - - node - } - ast::Expression::Starred { value } => { - let node = create_node(vm, "Starred"); - - let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); - - node - } - ast::Expression::Slice { elements } => { - let node = create_node(vm, "Slice"); - - let py_value = expressions_to_ast(vm, elements); - vm.ctx.set_attr(&node, "bounds", py_value); - - node + let py_value = expression_to_ast(vm, value)?; + node!(vm, YieldFrom, { + value => py_value + }) } + ast::Expression::Subscript { a, b } => node!(vm, Subscript, { + value => expression_to_ast(vm, a)?, + slice => expression_to_ast(vm, b)?, + }), + ast::Expression::Attribute { value, name } => node!(vm, Attribute, { + value => expression_to_ast(vm, value)?, + attr => vm.ctx.new_str(name.to_string()), + }), + ast::Expression::Starred { value } => node!(vm, Starred, { + value => expression_to_ast(vm, value)? + }), + ast::Expression::Slice { elements } => node!(vm, Slice, { + bounds => expressions_to_ast(vm, elements)? + }), ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { - let node = create_node(vm, "Bytes"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_bytes(value.clone())); - node + node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } - }; + }?; // TODO: retrieve correct lineno: let lineno = vm.ctx.new_int(1); - vm.ctx.set_attr(&node, "lineno", lineno); - - node + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); + Ok(node) } -fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyObjectRef { - let node = create_node(vm, "arguments"); - - vm.ctx.set_attr( - &node, - "args", - vm.ctx - .new_list(args.args.iter().map(|a| parameter_to_ast(vm, a)).collect()), - ); - - node +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyResult { + let args = map_ast(parameter_to_ast, vm, &args.args)?; + node!(vm, arguments, { args => args }) } -fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyObjectRef { - let node = create_node(vm, "arg"); - - let py_arg = vm.ctx.new_str(parameter.arg.to_string()); - vm.ctx.set_attr(&node, "arg", py_arg); - +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(vm, annotation) + expression_to_ast(vm, annotation)?.into_object() } else { vm.ctx.none() }; - vm.ctx.set_attr(&node, "annotation", py_annotation); - node + node!(vm, arg, { + arg => vm.ctx.new_str(parameter.arg.to_string()), + annotation => py_annotation + }) } -fn comprehension_to_ast(vm: &VirtualMachine, comprehension: &ast::Comprehension) -> PyObjectRef { - let node = create_node(vm, "comprehension"); - - let py_target = expression_to_ast(vm, &comprehension.target); - vm.ctx.set_attr(&node, "target", py_target); - - let py_iter = expression_to_ast(vm, &comprehension.iter); - vm.ctx.set_attr(&node, "iter", py_iter); - - let py_ifs = expressions_to_ast(vm, &comprehension.ifs); - vm.ctx.set_attr(&node, "ifs", py_ifs); - - node +fn map_ast( + f: fn(vm: &VirtualMachine, &T) -> PyResult, + vm: &VirtualMachine, + items: &Vec, +) -> PyResult { + let list: PyResult> = + items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); + Ok(vm.ctx.new_list(list?)) } -fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyObjectRef { +fn comprehension_to_ast( + vm: &VirtualMachine, + comprehension: &ast::Comprehension, +) -> PyResult { + node!(vm, comprehension, { + target => expression_to_ast(vm, &comprehension.target)?, + iter => expression_to_ast(vm, &comprehension.iter)?, + ifs => expressions_to_ast(vm, &comprehension.ifs)?, + }) +} + +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyResult { match string { ast::StringGroup::Constant { value } => { - let node = create_node(vm, "Str"); - vm.ctx.set_attr(&node, "s", vm.ctx.new_str(value.clone())); - node + node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - let node = create_node(vm, "FormattedValue"); - let py_value = expression_to_ast(vm, value); - vm.ctx.set_attr(&node, "value", py_value); - node + node!(vm, FormattedValue, { value => expression_to_ast(vm, value)? }) } ast::StringGroup::Joined { values } => { - let node = create_node(vm, "JoinedStr"); - let py_values = vm.ctx.new_list( - values - .iter() - .map(|value| string_to_ast(vm, value)) - .collect(), - ); - vm.ctx.set_attr(&node, "values", py_values); - node + let py_values = map_ast(string_to_ast, vm, &values)?; + node!(vm, JoinedStr, { values => py_values }) } } } -fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); - - let source_string = objstr::get_value(source); - let internal_ast = parser::parse_program(&source_string) +fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { + let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - let ast_node = program_to_ast(&vm, &internal_ast); - Ok(ast_node) + program_to_ast(&vm, &internal_ast) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "ast", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); + py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), - "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}), - "AST" => py_class!(ctx, "_ast.AST", ctx.object(), {}), + "AST" => ast_base.clone(), + // TODO: There's got to be a better way! + "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), + "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), + "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), + "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), + "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), + "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), + "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), + "Bytes" => py_class!(ctx, "_ast.Bytes", ast_base.clone(), {}), + "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Compare" => py_class!(ctx, "_ast.Compare", ast_base.clone(), {}), + "comprehension" => py_class!(ctx, "_ast.comprehension", ast_base.clone(), {}), + "Continue" => py_class!(ctx, "_ast.Continue", ast_base.clone(), {}), + "Delete" => py_class!(ctx, "_ast.Delete", ast_base.clone(), {}), + "Dict" => py_class!(ctx, "_ast.Dict", ast_base.clone(), {}), + "DictComp" => py_class!(ctx, "_ast.DictComp", ast_base.clone(), {}), + "Ellipsis" => py_class!(ctx, "_ast.Ellipsis", ast_base.clone(), {}), + "Expr" => py_class!(ctx, "_ast.Expr", ast_base.clone(), {}), + "For" => py_class!(ctx, "_ast.For", ast_base.clone(), {}), + "FormattedValue" => py_class!(ctx, "_ast.FormattedValue", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "GeneratorExp" => py_class!(ctx, "_ast.GeneratorExp", ast_base.clone(), {}), + "Identifier" => py_class!(ctx, "_ast.Identifier", ast_base.clone(), {}), + "If" => py_class!(ctx, "_ast.If", ast_base.clone(), {}), + "IfExp" => py_class!(ctx, "_ast.IfExp", ast_base.clone(), {}), + "JoinedStr" => py_class!(ctx, "_ast.JoinedStr", ast_base.clone(), {}), + "Lambda" => py_class!(ctx, "_ast.Lambda", ast_base.clone(), {}), + "List" => py_class!(ctx, "_ast.List", ast_base.clone(), {}), + "ListComp" => py_class!(ctx, "_ast.ListComp", ast_base.clone(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "Num" => py_class!(ctx, "_ast.Num", ast_base.clone(), {}), + "Pass" => py_class!(ctx, "_ast.Pass", ast_base.clone(), {}), + "Return" => py_class!(ctx, "_ast.Return", ast_base.clone(), {}), + "Set" => py_class!(ctx, "_ast.Set", ast_base.clone(), {}), + "SetComp" => py_class!(ctx, "_ast.SetComp", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Str" => py_class!(ctx, "_ast.Str", ast_base.clone(), {}), + "Subscript" => py_class!(ctx, "_ast.Subscript", ast_base.clone(), {}), + "Tuple" => py_class!(ctx, "_ast.Tuple", ast_base.clone(), {}), + "UnaryOp" => py_class!(ctx, "_ast.UnaryOp", ast_base.clone(), {}), + "While" => py_class!(ctx, "_ast.While", ast_base.clone(), {}), + "Yield" => py_class!(ctx, "_ast.Yield", ast_base.clone(), {}), + "YieldFrom" => py_class!(ctx, "_ast.YieldFrom", ast_base.clone(), {}), }) } diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index 222f23383..4c35f45e9 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -1,5 +1,5 @@ use crate::obj::objcode::PyCodeRef; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TryFromObject}; +use crate::pyobject::{PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -17,8 +17,10 @@ fn dis_disassemble(co: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "dis", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "dis", { "dis" => ctx.new_rustfunc(dis_dis), "disassemble" => ctx.new_rustfunc(dis_disassemble) }) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 9daaf95d2..b8e72cc15 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -20,9 +20,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objstr; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{ - BufferProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; fn compute_c_flag(mode: &str) -> u16 { @@ -97,9 +95,12 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +// TODO Check if closed, then if so raise ValueError +fn io_base_flush(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); - vm.ctx.set_attr(buffered, "raw", raw.clone()); + vm.set_attr(buffered, "raw", raw.clone())?; Ok(vm.get_none()) } @@ -148,10 +149,10 @@ fn file_io_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let args = vec![name.clone(), vm.ctx.new_int(os_mode)]; let file_no = os::os_open(vm, PyFuncArgs::new(args, vec![]))?; - vm.ctx.set_attr(file_io, "name", name.clone()); - vm.ctx.set_attr(file_io, "fileno", file_no); - vm.ctx.set_attr(file_io, "closefd", vm.new_bool(false)); - vm.ctx.set_attr(file_io, "closed", vm.new_bool(false)); + vm.set_attr(file_io, "name", name.clone())?; + vm.set_attr(file_io, "fileno", file_no)?; + vm.set_attr(file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(file_io, "closed", vm.new_bool(false))?; Ok(vm.get_none()) } @@ -215,7 +216,7 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }; let updated = os::raw_file_number(f.into_inner()); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; Ok(vm.get_none()) } @@ -241,7 +242,7 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(len) => { //reset raw fd on the FileIO object let updated = os::raw_file_number(handle); - vm.ctx.set_attr(file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; //return number of bytes written Ok(vm.ctx.new_int(len)) @@ -273,7 +274,7 @@ fn text_io_wrapper_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(text_io_wrapper, None), (buffer, None)] ); - vm.ctx.set_attr(text_io_wrapper, "buffer", buffer.clone()); + vm.set_attr(text_io_wrapper, "buffer", buffer.clone())?; Ok(vm.get_none()) } @@ -365,11 +366,14 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), - "__exit__" => ctx.new_rustfunc(io_base_cm_exit) + "__exit__" => ctx.new_rustfunc(io_base_cm_exit), + "flush" => ctx.new_rustfunc(io_base_flush) }); // IOBase Subclasses @@ -421,7 +425,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "getvalue" => ctx.new_rustfunc(bytes_io_getvalue) }); - py_module!(ctx, "io", { + py_module!(vm, "io", { "open" => ctx.new_rustfunc(io_open), "IOBase" => io_base, "RawIOBase" => raw_io_base, diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index fbe7ea6b1..dceec8a7d 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -7,13 +7,13 @@ use serde_json; use crate::function::PyFuncArgs; use crate::obj::{ - objbool, objdict, objfloat, objint, objsequence, + objbool, + objdict::PyDictRef, + objfloat, objint, objsequence, objstr::{self, PyString}, objtype, }; -use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol, -}; +use crate::pyobject::{create_type, IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use crate::VirtualMachine; use num_traits::cast::ToPrimitive; @@ -63,10 +63,11 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { let elements = objsequence::get_elements(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_elements(self.pyobject); + let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); + let pairs: Vec<_> = dict.into_iter().collect(); let mut map = serializer.serialize_map(Some(pairs.len()))?; for (key, e) in pairs.iter() { - map.serialize_entry(&key, &self.clone_with_object(&e.1))?; + map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; } map.end() } else if self.pyobject.is(&self.vm.get_none()) { @@ -175,7 +176,7 @@ impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { Some(PyString { ref value }) => value.clone(), _ => unimplemented!("map keys must be strings"), }; - dict.set_item(&self.vm.ctx, &key, value); + dict.set_item(&key, value, self.vm).unwrap(); } Ok(dict.into_object()) } @@ -201,14 +202,16 @@ pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { let json_decode_error = vm .get_attribute(vm.sys_module.clone(), "modules") .unwrap() - .get_item("json") + .get_item("json", vm) .unwrap() - .get_item("JSONDecodeError") + .get_item("JSONDecodeError", vm) .unwrap(); let json_decode_error = json_decode_error.downcast().unwrap(); let exc = vm.new_exception(json_decode_error, format!("{}", err)); - vm.ctx.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())); - vm.ctx.set_attr(&exc, "colno", vm.ctx.new_int(err.column())); + vm.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())) + .unwrap(); + vm.set_attr(&exc, "colno", vm.ctx.new_int(err.column())) + .unwrap(); exc }) } @@ -229,7 +232,9 @@ fn json_loads(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { de_pyobject(vm, &objstr::get_value(&string)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + // TODO: Make this a proper type with a constructor let json_decode_error = create_type( "JSONDecodeError", @@ -237,7 +242,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { &ctx.exceptions.exception_type, ); - py_module!(ctx, "json", { + py_module!(vm, "json", { "dumps" => ctx.new_rustfunc(json_dumps), "loads" => ctx.new_rustfunc(json_loads), "JSONDecodeError" => json_decode_error diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index fe27399c5..062f41df7 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,7 +18,9 @@ fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let keyword_kwlist = ctx.new_list( lexer::get_keywords() .keys() @@ -26,7 +28,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { .collect(), ); - py_module!(ctx, "keyword", { + py_module!(vm, "keyword", { "iskeyword" => ctx.new_rustfunc(keyword_iskeyword), "kwlist" => keyword_kwlist }) diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 5af599cb7..2ee584a79 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -8,7 +8,7 @@ use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; // Helper macro: @@ -172,8 +172,10 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "math", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "math", { // Number theory functions: "fabs" => ctx.new_rustfunc(math_fabs), "isfinite" => ctx.new_rustfunc(math_isfinite), diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index b0b61119c..0933b7192 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -9,20 +9,23 @@ mod random; mod re; pub mod socket; mod string; +mod thread; mod time_module; mod tokenize; mod types; mod weakref; use std::collections::HashMap; +use crate::vm::VirtualMachine; + #[cfg(not(target_arch = "wasm32"))] pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; -pub type StdlibInitFunc = Box PyObjectRef>; +pub type StdlibInitFunc = Box PyObjectRef>; pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); @@ -39,6 +42,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("random".to_string(), Box::new(random::make_module)); modules.insert("string".to_string(), Box::new(string::make_module)); modules.insert("struct".to_string(), Box::new(pystruct::make_module)); + modules.insert("_thread".to_string(), Box::new(thread::make_module)); modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); modules.insert("types".to_string(), Box::new(types::make_module)); @@ -48,7 +52,7 @@ pub fn get_module_inits() -> HashMap { #[cfg(not(target_arch = "wasm32"))] { modules.insert("io".to_string(), Box::new(io::make_module)); - modules.insert("os".to_string(), Box::new(os::make_module)); + modules.insert("_os".to_string(), Box::new(os::make_module)); modules.insert("socket".to_string(), Box::new(socket::make_module)); } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 42b4160ca..fae9d6adb 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,13 +1,18 @@ use std::fs::File; use std::fs::OpenOptions; -use std::io::ErrorKind; +use std::io::{ErrorKind, Read, Write}; +use std::{env, fs}; use num_traits::cast::ToPrimitive; use crate::function::PyFuncArgs; +use crate::obj::objbytes::PyBytesRef; +use crate::obj::objdict::PyDictRef; use crate::obj::objint; +use crate::obj::objint::PyIntRef; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -113,25 +118,110 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "os", { +fn os_read(fd: PyIntRef, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; n.as_bigint().to_usize().unwrap()]; + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + match file.read_exact(&mut buffer) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_bytes(buffer)) +} + +fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + let written = match file.write(&data) { + Ok(written) => written, + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_int(written)) +} + +fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_file(&path.value).map_err(|s| vm.new_os_error(s.to_string())) +} + +fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) +} + +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir_all(&path.value).map_err(|s| vm.new_os_error(s.to_string())) +} + +fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::remove_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) +} + +fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => { + let res: PyResult> = iter + .map(|entry| match entry { + Ok(path) => Ok(vm.ctx.new_str(path.file_name().into_string().unwrap())), + Err(s) => Err(vm.new_os_error(s.to_string())), + }) + .collect(); + Ok(vm.ctx.new_list(res?)) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) { + env::set_var(&key.value, &value.value) +} + +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) { + env::remove_var(&key.value) +} + +fn _os_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { + environ.set_item(&key, vm.new_str(value), vm).unwrap(); + } + environ +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let os_name = if cfg!(windows) { + "nt".to_string() + } else { + "posix".to_string() + }; + + let environ = _os_environ(vm); + + py_module!(vm, "_os", { "open" => ctx.new_rustfunc(os_open), "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), + "read" => ctx.new_rustfunc(os_read), + "write" => ctx.new_rustfunc(os_write), + "remove" => ctx.new_rustfunc(os_remove), + "unlink" => ctx.new_rustfunc(os_remove), + "mkdir" => ctx.new_rustfunc(os_mkdir), + "mkdirs" => ctx.new_rustfunc(os_mkdirs), + "rmdir" => ctx.new_rustfunc(os_rmdir), + "listdir" => ctx.new_rustfunc(os_listdir), + "putenv" => ctx.new_rustfunc(os_putenv), + "unsetenv" => ctx.new_rustfunc(os_unsetenv), + "environ" => environ, + "name" => ctx.new_str(os_name), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), "O_NONBLOCK" => ctx.new_int(4), "O_APPEND" => ctx.new_int(8), "O_CREAT" => ctx.new_int(512) - }); - - if cfg!(windows) { - ctx.set_attr(&py_mod, "name", ctx.new_str("nt".to_string())); - } else { - // Assume we're on a POSIX system - ctx.set_attr(&py_mod, "name", ctx.new_str("posix".to_string())); - } - - py_mod + }) } diff --git a/vm/src/stdlib/platform.rs b/vm/src/stdlib/platform.rs index a87383a72..6d4751a35 100644 --- a/vm/src/stdlib/platform.rs +++ b/vm/src/stdlib/platform.rs @@ -1,9 +1,10 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "platform", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "platform", { "python_compiler" => ctx.new_rustfunc(platform_python_compiler), "python_implementation" => ctx.new_rustfunc(platform_python_implementation), "python_version" => ctx.new_rustfunc(platform_python_version), diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 9dadc520a..4a454b0c9 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -15,7 +15,7 @@ use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; #[derive(Debug)] @@ -297,8 +297,10 @@ fn struct_unpack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_tuple(items)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "struct", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "struct", { "pack" => ctx.new_rustfunc(struct_pack), "unpack" => ctx.new_rustfunc(struct_unpack) }) diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index 82c1f410c..cf3c0ea18 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,11 +4,13 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "random", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "random", { "guass" => ctx.new_rustfunc(random_gauss), "normalvariate" => ctx.new_rustfunc(random_normalvariate), "random" => ctx.new_rustfunc(random_random), diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index fee100bfe..b9f921502 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -8,7 +8,7 @@ use regex::{Match, Regex}; use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; impl PyValue for Regex { @@ -98,7 +98,9 @@ impl PyMatchRef { } /// Create the python `re` module with all its members. -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let match_type = py_class!(ctx, "Match", ctx.object(), { "start" => ctx.new_rustfunc(PyMatchRef::start), "end" => ctx.new_rustfunc(PyMatchRef::end) @@ -109,7 +111,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "search" => ctx.new_rustfunc(PyRegexRef::search) }); - py_module!(ctx, "re", { + py_module!(vm, "re", { "compile" => ctx.new_rustfunc(re_compile), "Match" => match_type, "match" => ctx.new_rustfunc(re_match), diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index fe0ce29a0..e893e0dab 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -10,9 +10,7 @@ use crate::obj::objbytes; use crate::obj::objint; use crate::obj::objsequence::get_elements; use crate::obj::objstr; -use crate::pyobject::{ - PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; use crate::obj::objtype::PyClassRef; @@ -411,7 +409,9 @@ fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { Ok(vm.ctx.new_tuple(vec![ip, port])) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let socket = py_class!(ctx, "socket", ctx.object(), { "__new__" => ctx.new_rustfunc(socket_new), "connect" => ctx.new_rustfunc(socket_connect), @@ -426,7 +426,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "recvfrom" => ctx.new_rustfunc(socket_recvfrom), }); - py_module!(ctx, "socket", { + py_module!(vm, "socket", { "AF_INET" => ctx.new_int(AddressFamily::Inet as i32), "SOCK_STREAM" => ctx.new_int(SocketKind::Stream as i32), "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 157dae035..c255ffc93 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -3,9 +3,12 @@ * */ -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { let ascii_lowercase = "abcdefghijklmnopqrstuvwxyz".to_string(); let ascii_uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_string(); let ascii_letters = format!("{}{}", ascii_lowercase, ascii_uppercase); @@ -19,7 +22,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { */ // Constants: - py_module!(ctx, "string", { + py_module!(vm, "string", { "ascii_letters" => ctx.new_str(ascii_letters), "ascii_lowercase" => ctx.new_str(ascii_lowercase), "ascii_uppercase" => ctx.new_str(ascii_uppercase), diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs new file mode 100644 index 000000000..8de881893 --- /dev/null +++ b/vm/src/stdlib/thread.rs @@ -0,0 +1,30 @@ +/// Implementation of the _thread module, currently noop implementation as RustPython doesn't yet +/// support threading +use super::super::pyobject::PyObjectRef; +use crate::function::PyFuncArgs; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +fn rlock_acquire(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) +} + +fn rlock_release(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + +fn get_ident(_vm: &VirtualMachine) -> u32 { + 1 +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let rlock_type = py_class!(ctx, "_thread.RLock", ctx.object(), { + "acquire" => ctx.new_rustfunc(rlock_acquire), + "release" => ctx.new_rustfunc(rlock_release), + }); + + py_module!(vm, "_thread", { + "RLock" => rlock_type, + "get_ident" => ctx.new_rustfunc(get_ident) + }) +} diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index dfe3ab8e3..6c818689e 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -32,8 +32,10 @@ fn time_time(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "time", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "time", { "sleep" => ctx.new_rustfunc(time_sleep), "time" => ctx.new_rustfunc(time_time) }) diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 0aa5df3d2..000dd5e1a 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -25,8 +25,10 @@ fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: create main function when called with -m -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "tokenize", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "tokenize", { "tokenize" => ctx.new_rustfunc(tokenize_tokenize) }) } diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs index 8a5ab2939..e224a93c6 100644 --- a/vm/src/stdlib/types.rs +++ b/vm/src/stdlib/types.rs @@ -7,7 +7,7 @@ use crate::obj::objdict::PyDict; use crate::obj::objstr::PyStringRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; +use crate::pyobject::{PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; use crate::VirtualMachine; fn types_new_class( @@ -25,10 +25,13 @@ fn types_new_class( objtype::type_new_class(vm, vm.ctx.type_type(), name, bases, dict) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "types", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "types", { "new_class" => ctx.new_rustfunc(types_new_class), "FunctionType" => ctx.function_type(), + "MethodType" => ctx.bound_method_type(), "LambdaType" => ctx.function_type(), "CodeType" => ctx.code_type(), "FrameType" => ctx.frame_type() diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 5616c3ddf..d4529599e 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -5,10 +5,14 @@ //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -use super::super::pyobject::{PyContext, PyObjectRef}; +use super::super::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "_weakref", { - "ref" => ctx.weakref_type() +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "_weakref", { + "ref" => ctx.weakref_type(), + "proxy" => ctx.weakproxy_type(), }) } diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs new file mode 100644 index 000000000..d7f30c870 --- /dev/null +++ b/vm/src/symboltable.rs @@ -0,0 +1,463 @@ +/* Python code is pre-scanned for symbols in the ast. + +This ensures that global and nonlocal keywords are picked up. +Then the compiler can use the symbol table to generate proper +load and store instructions for names. +*/ + +use rustpython_parser::ast; +use std::collections::HashMap; + +pub fn make_symbol_table(program: &ast::Program) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_program(program).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +pub fn statements_to_symbol_table(statements: &[ast::LocatedStatement]) -> SymbolScope { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_statements(statements).unwrap(); + assert!(builder.scopes.len() == 1); + builder.scopes.pop().unwrap() +} + +#[derive(Debug)] +pub enum SymbolRole { + Global, + Nonlocal, + Used, + Assigned, +} + +/// Symbolscope captures all symbols in the current scope, and +/// has a list of subscopes in this scope. +pub struct SymbolScope { + /// A set of symbols present on this scope level. + pub symbols: HashMap, + + /// A list of subscopes in the order as found in the + /// AST nodes. + pub sub_scopes: Vec, +} + +type SymbolTableResult = Result<(), String>; + +impl SymbolScope { + pub fn new() -> Self { + SymbolScope { + symbols: HashMap::new(), + sub_scopes: vec![], + } + } + + pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { + self.symbols.get(name) + } +} + +impl std::fmt::Debug for SymbolScope { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "SymbolScope({:?} symbols, {:?} sub scopes)", + self.symbols.len(), + self.sub_scopes.len() + ) + } +} + +pub struct SymbolTableBuilder { + // Scope stack. + pub scopes: Vec, +} + +impl SymbolTableBuilder { + pub fn new() -> Self { + SymbolTableBuilder { scopes: vec![] } + } + + pub fn enter_scope(&mut self) { + // Create new scope and push into scope stack. + let scope = SymbolScope::new(); + self.scopes.push(scope); + } + + pub fn leave_scope(&mut self) { + // Pop scope and add to subscopes of parent scope. + let scope = self.scopes.pop().unwrap(); + self.scopes.last_mut().unwrap().sub_scopes.push(scope); + } + + pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { + self.scan_statements(&program.statements)?; + Ok(()) + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult { + for statement in statements { + self.scan_statement(statement)?; + } + Ok(()) + } + + fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter(parameter)?; + } + Ok(()) + } + + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.register_name(¶meter.arg, SymbolRole::Assigned) + } + + fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter_annotation(parameter)?; + } + Ok(()) + } + + fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; + } + Ok(()) + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult { + match &statement.node { + ast::Statement::Global { names } => { + for name in names { + self.register_name(name, SymbolRole::Global)?; + } + } + ast::Statement::Nonlocal { names } => { + for name in names { + self.register_name(name, SymbolRole::Nonlocal)?; + } + } + ast::Statement::FunctionDef { + name, + body, + args, + decorator_list, + returns, + } => { + self.scan_expressions(decorator_list)?; + self.register_name(name, SymbolRole::Assigned)?; + + self.enter_function(args)?; + + self.scan_statements(body)?; + if let Some(expression) = returns { + self.scan_expression(expression)?; + } + self.leave_scope(); + } + ast::Statement::ClassDef { + name, + body, + bases, + keywords, + decorator_list, + } => { + self.register_name(name, SymbolRole::Assigned)?; + self.enter_scope(); + self.scan_statements(body)?; + self.leave_scope(); + self.scan_expressions(bases)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + self.scan_expressions(decorator_list)?; + } + ast::Statement::Expression { expression } => self.scan_expression(expression)?, + ast::Statement::If { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::For { + target, + iter, + body, + orelse, + } => { + self.scan_expression(target)?; + self.scan_expression(iter)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::While { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { + // No symbols here. + } + ast::Statement::Import { import_parts } => { + for part in import_parts { + if let Some(alias) = &part.alias { + self.register_name(alias, SymbolRole::Assigned)?; + } else { + if let Some(symbol) = &part.symbol { + self.register_name(symbol, SymbolRole::Assigned)?; + } else { + self.register_name(&part.module, SymbolRole::Assigned)?; + } + } + } + } + ast::Statement::Return { value } => { + if let Some(expression) = value { + self.scan_expression(expression)?; + } + } + ast::Statement::Assert { test, msg } => { + self.scan_expression(test)?; + if let Some(expression) = msg { + self.scan_expression(expression)?; + } + } + ast::Statement::Delete { targets } => { + self.scan_expressions(targets)?; + } + ast::Statement::Assign { targets, value } => { + self.scan_expressions(targets)?; + self.scan_expression(value)?; + } + ast::Statement::AugAssign { target, value, .. } => { + self.scan_expression(target)?; + self.scan_expression(value)?; + } + ast::Statement::With { items, body } => { + for item in items { + self.scan_expression(&item.context_expr)?; + if let Some(expression) = &item.optional_vars { + self.scan_expression(expression)?; + } + } + self.scan_statements(body)?; + } + ast::Statement::Try { + body, + handlers, + orelse, + finalbody, + } => { + self.scan_statements(body)?; + for handler in handlers { + if let Some(expression) = &handler.typ { + self.scan_expression(expression)?; + } + if let Some(name) = &handler.name { + self.register_name(name, SymbolRole::Assigned)?; + } + self.scan_statements(&handler.body)?; + } + if let Some(code) = orelse { + self.scan_statements(code)?; + } + if let Some(code) = finalbody { + self.scan_statements(code)?; + } + } + ast::Statement::Raise { exception, cause } => { + if let Some(expression) = exception { + self.scan_expression(expression)?; + } + if let Some(expression) = cause { + self.scan_expression(expression)?; + } + } + } + Ok(()) + } + + fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult { + for expression in expressions { + self.scan_expression(expression)?; + } + Ok(()) + } + + fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { + match expression { + ast::Expression::Binop { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::BoolOp { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::Compare { vals, .. } => { + self.scan_expressions(vals)?; + } + ast::Expression::Subscript { a, b } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::Attribute { value, .. } => { + self.scan_expression(value)?; + } + ast::Expression::Dict { elements } => { + for (key, value) in elements { + self.scan_expression(key)?; + self.scan_expression(value)?; + } + } + ast::Expression::Yield { value } => { + if let Some(expression) = value { + self.scan_expression(expression)?; + } + } + ast::Expression::YieldFrom { value } => { + self.scan_expression(value)?; + } + ast::Expression::Unop { a, .. } => { + self.scan_expression(a)?; + } + ast::Expression::True + | ast::Expression::False + | ast::Expression::None + | ast::Expression::Ellipsis => {} + ast::Expression::Number { .. } => {} + ast::Expression::Starred { value } => { + self.scan_expression(value)?; + } + ast::Expression::Bytes { .. } => {} + ast::Expression::Tuple { elements } + | ast::Expression::Set { elements } + | ast::Expression::List { elements } + | ast::Expression::Slice { elements } => { + self.scan_expressions(elements)?; + } + ast::Expression::Comprehension { kind, generators } => { + match **kind { + ast::ComprehensionKind::GeneratorExpression { ref element } + | ast::ComprehensionKind::List { ref element } + | ast::ComprehensionKind::Set { ref element } => { + self.scan_expression(element)?; + } + ast::ComprehensionKind::Dict { ref key, ref value } => { + self.scan_expression(&key)?; + self.scan_expression(&value)?; + } + } + + for generator in generators { + self.scan_expression(&generator.target)?; + self.scan_expression(&generator.iter)?; + for if_expr in &generator.ifs { + self.scan_expression(if_expr)?; + } + } + } + ast::Expression::Call { + function, + args, + keywords, + } => { + self.scan_expression(function)?; + self.scan_expressions(args)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + } + ast::Expression::String { value } => { + self.scan_string_group(value)?; + } + ast::Expression::Identifier { name } => { + self.register_name(name, SymbolRole::Used)?; + } + ast::Expression::Lambda { args, body } => { + self.enter_function(args)?; + self.scan_expression(body)?; + self.leave_scope(); + } + ast::Expression::IfExpression { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_expression(body)?; + self.scan_expression(orelse)?; + } + } + Ok(()) + } + + fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { + // Evaulate eventual default parameters: + self.scan_expressions(&args.defaults)?; + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression)?; + } + } + + // Annotations are scanned in outer scope: + self.scan_parameters_annotations(&args.args)?; + self.scan_parameters_annotations(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter_annotation(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter_annotation(name)?; + } + + self.enter_scope(); + + // Fill scope with parameter names: + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter(name)?; + } + Ok(()) + } + + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { + match group { + ast::StringGroup::Constant { .. } => {} + ast::StringGroup::FormattedValue { value, .. } => { + self.scan_expression(value)?; + } + ast::StringGroup::Joined { values } => { + for subgroup in values { + self.scan_string_group(subgroup)?; + } + } + } + Ok(()) + } + + fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { + let current_scope = self.scopes.last_mut().unwrap(); + if let Some(_old_role) = current_scope.symbols.get(name) { + // Role already set.. + // debug!("TODO: {:?}", old_role); + match role { + SymbolRole::Global => return Err("global must appear first".to_string()), + _ => { + // Ok? + } + } + } else { + current_scope.symbols.insert(name.to_string(), role); + } + Ok(()) + } +} diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index a4a37b350..a5bf2a6f4 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -3,7 +3,8 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{ItemProtocol, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* @@ -39,7 +40,14 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } -pub fn make_module(ctx: &PyContext, builtins: PyObjectRef) -> PyObjectRef { +// TODO implement string interning, this will be key for performance +fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + value +} + +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { + let ctx = &vm.ctx; + let path_list = match env::var_os("PYTHONPATH") { Some(paths) => env::split_paths(&paths) .map(|path| { @@ -124,22 +132,21 @@ setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; let modules = ctx.new_dict(); - let sys_name = "sys"; - let sys_mod = py_module!(ctx, sys_name, { + extend_module!(vm, module, { "argv" => argv(ctx), "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, "ps1" => ctx.new_str(">>>>> ".to_string()), "ps2" => ctx.new_str("..... ".to_string()), "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), + "modules" => modules.clone(), + "warnoptions" => ctx.new_list(vec![]), }); - modules.set_item(&ctx, sys_name, sys_mod.clone()); - modules.set_item(&ctx, "builtins", builtins); - ctx.set_attr(&sys_mod, "modules", modules); - - sys_mod + modules.set_item("sys", module.clone(), vm).unwrap(); + modules.set_item("builtins", builtins.clone(), vm).unwrap(); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 36f4450e6..e360b8e5e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -21,16 +21,15 @@ use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; -use crate::obj::objgenerator::PyGeneratorRef; +use crate::obj::objgenerator::PyGenerator; use crate::obj::objiter; -use crate::obj::objlist::PyList; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; -use crate::obj::objtuple::PyTuple; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::stdlib; @@ -58,18 +57,22 @@ impl VirtualMachine { let ctx = PyContext::new(); // Hard-core modules: - let builtins = builtins::make_module(&ctx); - let sysmod = sysmodule::make_module(&ctx, builtins.clone()); + let builtins = ctx.new_module("builtins", ctx.new_dict()); + let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); - VirtualMachine { - builtins, - sys_module: sysmod, + let vm = VirtualMachine { + builtins: builtins.clone(), + sys_module: sysmod.clone(), stdlib_inits, ctx, frames: RefCell::new(vec![]), wasm_id: None, - } + }; + + builtins::make_module(&vm, builtins.clone()); + sysmodule::make_module(&vm, sysmod, builtins); + vm } pub fn run_code_obj(&self, code: PyCodeRef, scope: Scope) -> PyResult { @@ -91,6 +94,17 @@ impl VirtualMachine { result } + pub fn frame_throw( + &self, + frame: FrameRef, + exception: PyObjectRef, + ) -> PyResult { + self.frames.borrow_mut().push(frame.clone()); + let result = frame.throw(self, exception); + self.frames.borrow_mut().pop(); + result + } + pub fn current_frame(&self) -> Option> { let frames = self.frames.borrow(); if frames.is_empty() { @@ -220,6 +234,11 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } + pub fn new_syntax_error(&self, msg: &T) -> PyObjectRef { + let syntax_error = self.ctx.exceptions.syntax_error.clone(); + self.new_exception(syntax_error, msg.to_string()) + } + pub fn get_none(&self) -> PyObjectRef { self.ctx.none() } @@ -328,9 +347,10 @@ impl VirtualMachine { ref code, ref scope, ref defaults, + ref kw_only_defaults, }) = func_ref.payload() { - return self.invoke_python_function(code, scope, defaults, args); + return self.invoke_python_function(code, scope, defaults, kw_only_defaults, args); } if let Some(PyMethod { ref function, @@ -352,18 +372,25 @@ impl VirtualMachine { &self, code: &PyCodeRef, scope: &Scope, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, args: PyFuncArgs, ) -> PyResult { let scope = scope.child_scope(&self.ctx); - self.fill_locals_from_args(&code.code, &scope.get_locals(), args, defaults)?; + self.fill_locals_from_args( + &code.code, + &scope.get_locals(), + args, + defaults, + kw_only_defaults, + )?; // Construct frame: let frame = Frame::new(code.clone(), scope).into_ref(self); // If we have a generator, create a new generator if code.code.is_generator { - Ok(PyGeneratorRef::new(frame, self).into_object()) + Ok(PyGenerator::new(frame, self).into_object()) } else { self.run_frame_full(frame) } @@ -375,12 +402,7 @@ impl VirtualMachine { cells: PyDictRef, locals: PyDictRef, ) -> PyResult { - if let Some(PyFunction { - code, - scope, - defaults: _, - }) = &function.payload() - { + if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope .child_scope_with_locals(cells) .child_scope_with_locals(locals); @@ -398,7 +420,8 @@ impl VirtualMachine { code_object: &bytecode::CodeObject, locals: &PyDictRef, args: PyFuncArgs, - defaults: &PyObjectRef, + defaults: &Option, + kw_only_defaults: &Option, ) -> PyResult<()> { let nargs = args.args.len(); let nexpected_args = code_object.arg_names.len(); @@ -419,7 +442,7 @@ impl VirtualMachine { for i in 0..n { let arg_name = &code_object.arg_names[i]; let arg = &args.args[i]; - locals.set_item(&self.ctx, arg_name, arg.clone()); + locals.set_item(arg_name, arg.clone(), self)?; } // Pack other positional arguments in to *args: @@ -432,12 +455,9 @@ impl VirtualMachine { } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(&self.ctx, vararg_name, vararg_value); + locals.set_item(vararg_name, vararg_value, self)?; } - bytecode::Varargs::Unnamed => { - // just ignore the rest of the args - } - bytecode::Varargs::None => { + bytecode::Varargs::Unnamed | bytecode::Varargs::None => { // Check the number of positional arguments if nargs > nexpected_args { return Err(self.new_type_error(format!( @@ -452,7 +472,7 @@ impl VirtualMachine { let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { let d = self.ctx.new_dict(); - locals.set_item(&self.ctx, kwargs_name, d.as_object().clone()); + locals.set_item(kwargs_name, d.as_object().clone(), self)?; Some(d) } bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), @@ -464,15 +484,15 @@ impl VirtualMachine { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { - if locals.contains_key(&name) { + if locals.contains_key(&name, self) { return Err( self.new_type_error(format!("Got multiple values for argument '{}'", name)) ); } - locals.set_item(&self.ctx, &name, value); + locals.set_item(&name, value, self)?; } else if let Some(d) = &kwargs { - d.set_item(&self.ctx, &name, value); + d.set_item(&name, value, self)?; } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -483,23 +503,15 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let available_defaults = if defaults.is(&self.get_none()) { - vec![] - } else if let Some(list) = defaults.payload::() { - list.elements.borrow().clone() - } else if let Some(tuple) = defaults.payload::() { - tuple.elements.borrow().clone() - } else { - panic!("function defaults not tuple or None"); - }; + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.borrow().len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception - let required_args = nexpected_args - available_defaults.len(); + let required_args = nexpected_args - num_defaults_available; let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; - if !locals.contains_key(variable_name) { + if !locals.contains_key(variable_name, self) { missing.push(variable_name) } } @@ -510,35 +522,32 @@ impl VirtualMachine { missing ))); } - - // We have sufficient defaults, so iterate over the corresponding names and use - // the default if we don't already have a value - for (default_index, i) in (required_args..nexpected_args).enumerate() { - let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name) { - locals.set_item( - &self.ctx, - arg_name, - available_defaults[default_index].clone(), - ); + if let Some(defaults) = defaults { + let defaults = defaults.elements.borrow(); + // We have sufficient defaults, so iterate over the corresponding names and use + // the default if we don't already have a value + for (default_index, i) in (required_args..nexpected_args).enumerate() { + let arg_name = &code_object.arg_names[i]; + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, defaults[default_index].clone(), self)?; + } } } }; // Check if kw only arguments are all present: - let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { - if !locals.contains_key(arg_name) { - if kwdefs.contains_key(arg_name) { - // If not yet specified, take the default value - unimplemented!(); - } else { - // No default value and not specified. - return Err(self.new_type_error(format!( - "Missing required kw only argument: '{}'", - arg_name - ))); + if !locals.contains_key(arg_name, self) { + if let Some(kw_only_defaults) = kw_only_defaults { + if let Some(default) = kw_only_defaults.get_item_option(arg_name, self)? { + locals.set_item(arg_name, default, self)?; + continue; + } } + + // No default value and not specified. + return Err(self + .new_type_error(format!("Missing required kw only argument: '{}'", arg_name))); } } @@ -568,13 +577,17 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr( - &self, - obj: &PyObjectRef, - attr_name: PyObjectRef, - attr_value: PyObjectRef, - ) -> PyResult { - self.call_method(&obj, "__setattr__", vec![attr_name, attr_value]) + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: V) -> PyResult + where + K: TryIntoRef, + V: Into, + { + let attr_name = attr_name.try_into_ref(self)?; + self.call_method( + obj, + "__setattr__", + vec![attr_name.into_object(), attr_value.into()], + ) } pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult<()> { @@ -867,6 +880,31 @@ impl VirtualMachine { Err(vm.new_unsupported_operand_error(a, b, ">=")) }) } + + // https://docs.python.org/3/reference/expressions.html#membership-test-operations + fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + let iter = objiter::get_iter(self, &haystack)?; + loop { + if let Some(element) = objiter::get_next_object(self, &iter)? { + let equal = self._eq(needle.clone(), element.clone())?; + if objbool::get_value(&equal) { + return Ok(self.new_bool(true)); + } else { + continue; + } + } else { + return Ok(self.new_bool(false)); + } + } + } + + pub fn _membership(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + if let Ok(method) = self.get_method(haystack.clone(), "__contains__") { + self.invoke(method, vec![needle]) + } else { + self._membership_iter_search(haystack, needle) + } + } } impl Default for VirtualMachine { diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index 2d664fded..b4010c753 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -2,8 +2,7 @@ w = 50.0 h = 50.0 def mandel(): - """Print a mandelbrot fractal to the console, yielding after each character - is printed""" + """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 while y < h: x = 0.0 @@ -32,12 +31,14 @@ def mandel(): y += 1 yield +# run the mandelbrot + try: from browser import request_animation_frame except: request_animation_frame = None gen = mandel() def gen_cb(_time=None): - gen.__next__() + for _ in range(4): gen.__next__() request_animation_frame(gen_cb) if request_animation_frame: gen_cb() -else: list(gen) +else: any(gen) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index 9b58afe39..602e25241 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -1,6 +1,6 @@ import './style.css'; import 'codemirror/lib/codemirror.css'; -import 'xterm/dist/xterm.css'; +import 'xterm/lib/xterm.css'; // A dependency graph that contains any wasm must all be imported // asynchronously. This `index.js` file does the single async import, so diff --git a/wasm/demo/src/style.css b/wasm/demo/src/style.css index a8a5de500..8fc18ecce 100644 --- a/wasm/demo/src/style.css +++ b/wasm/demo/src/style.css @@ -1,6 +1,6 @@ textarea { font-family: monospace; - resize: none; + resize: vertical; } #code, diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 0e13aeb3a..478ebb9c5 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,17 +7,13 @@ use wasm_bindgen_futures::{future_to_promise, JsFuture}; use rustpython_vm::function::{OptionalArg, PyFuncArgs}; use rustpython_vm::obj::{ - objfunction::PyFunction, - objint, - objstr::{self, PyStringRef}, + objdict::PyDictRef, objfunction::PyFunctionRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, }; -use rustpython_vm::pyobject::{ - PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use rustpython_vm::pyobject::{PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; -use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; +use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; enum FetchResponseFormat { Json, @@ -43,25 +39,38 @@ impl FetchResponseFormat { } } -fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); +#[derive(FromArgs)] +struct FetchArgs { + #[pyarg(keyword_only, default = "None")] + response_format: Option, + #[pyarg(keyword_only, default = "None")] + method: Option, + #[pyarg(keyword_only, default = "None")] + headers: Option, + #[pyarg(keyword_only, default = "None")] + body: Option, + #[pyarg(keyword_only, default = "None")] + content_type: Option, +} - let response_format = - args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; - let method = args.get_optional_kwarg_with_type("method", vm.ctx.str_type(), vm)?; - let headers = args.get_optional_kwarg_with_type("headers", vm.ctx.dict_type(), vm)?; - let body = args.get_optional_kwarg("body"); - let content_type = args.get_optional_kwarg_with_type("content_type", vm.ctx.str_type(), vm)?; +fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyResult { + let FetchArgs { + response_format, + method, + headers, + body, + content_type, + } = args; let response_format = match response_format { - Some(s) => FetchResponseFormat::from_str(vm, &objstr::get_value(&s))?, + Some(s) => FetchResponseFormat::from_str(vm, s.as_str())?, None => FetchResponseFormat::Text, }; let mut opts = web_sys::RequestInit::new(); match method { - Some(s) => opts.method(&objstr::get_value(&s)), + Some(s) => opts.method(s.as_str()), None => opts.method("GET"), }; @@ -69,15 +78,15 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { opts.body(Some(&convert::py_to_js(vm, body))); } - let request = web_sys::Request::new_with_str_and_init(&objstr::get_value(url), &opts) + let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; if let Some(headers) = headers { let h = request.headers(); - for (key, value) in rustpython_vm::obj::objdict::get_key_value_pairs(&headers) { - let key = &vm.to_str(&key)?.value; - let value = &vm.to_str(&value)?.value; - h.set(key, value) + for (key, value) in headers { + let key = vm.to_str(&key)?; + let value = vm.to_str(&value)?; + h.set(key.as_str(), value.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } } @@ -85,7 +94,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(content_type) = content_type { request .headers() - .set("Content-Type", &objstr::get_value(&content_type)) + .set("Content-Type", content_type.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } @@ -104,9 +113,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(func, Some(vm.ctx.function_type()))]); - +fn browser_request_animation_frame(func: PyFunctionRef, vm: &VirtualMachine) -> PyResult { use std::{cell::RefCell, rc::Rc}; // this basic setup for request_animation_frame taken from: @@ -115,18 +122,16 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR let f = Rc::new(RefCell::new(None)); let g = f.clone(); - let func = func.clone(); - - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); *g.borrow_mut() = Some(Closure::wrap(Box::new(move |time: f64| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid from inside of request_animation_frame"); let vm = &stored_vm.vm; let func = func.clone(); let args = vec![vm.ctx.new_float(time)]; - let _ = vm.invoke(func, args); + let _ = vm.invoke(func.into_object(), args); let closure = f.borrow_mut().take(); drop(closure); @@ -141,10 +146,8 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR Ok(vm.ctx.new_int(id)) } -fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(id, Some(vm.ctx.int_type()))]); - - let id = objint::get_value(id).to_i32().ok_or_else(|| { +fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult { + let id = id.as_bigint().to_i32().ok_or_else(|| { vm.new_exception( vm.ctx.exceptions.value_error.clone(), "Integer too large to convert to i32 for animationFrame id".into(), @@ -158,6 +161,7 @@ fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyRe Ok(vm.get_none()) } +#[pyclass(name = "Promise")] #[derive(Debug)] pub struct PyPromise { value: Promise, @@ -170,6 +174,7 @@ impl PyValue for PyPromise { } } +#[pyimpl] impl PyPromise { pub fn new(value: Promise) -> PyPromise { PyPromise { value } @@ -184,16 +189,17 @@ impl PyPromise { self.value.clone() } + #[pymethod] fn then( - zelf: PyPromiseRef, - on_fulfill: PyRef, - on_reject: OptionalArg>, + &self, + on_fulfill: PyFunctionRef, + on_reject: OptionalArg, vm: &VirtualMachine, ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { - let stored_vm = &acc_vm + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { + let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -217,16 +223,13 @@ impl PyPromise { PyPromise::from_future(ret_future).into_ref(vm) } - fn catch( - zelf: PyPromiseRef, - on_reject: PyRef, - vm: &VirtualMachine, - ) -> PyPromiseRef { - let acc_vm = AccessibleVM::from(vm); + #[pymethod] + fn catch(&self, on_reject: PyFunctionRef, vm: &VirtualMachine) -> PyPromiseRef { + let weak_vm = weak_vm(vm); - let ret_future = JsFuture::from(zelf.value.clone()).then(move |res| { + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { res.or_else(|err| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; @@ -240,11 +243,11 @@ impl PyPromise { } } +#[pyclass] #[derive(Debug)] struct Document { doc: web_sys::Document, } -type DocumentRef = PyRef; impl PyValue for Document { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -252,9 +255,11 @@ impl PyValue for Document { } } +#[pyimpl] impl Document { - fn query(zelf: DocumentRef, query: PyStringRef, vm: &VirtualMachine) -> PyResult { - let elem = zelf + #[pymethod] + fn query(&self, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = self .doc .query_selector(&query.value) .map_err(|err| convert::js_py_typeerror(vm, err))?; @@ -266,11 +271,11 @@ impl Document { } } +#[pyclass] #[derive(Debug)] struct Element { elem: web_sys::Element, } -type ElementRef = PyRef; impl PyValue for Element { fn class(vm: &VirtualMachine) -> PyClassRef { @@ -278,66 +283,54 @@ impl PyValue for Element { } } +#[pyimpl] impl Element { + #[pymethod] fn get_attr( - zelf: ElementRef, + &self, attr: PyStringRef, default: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { - match zelf.elem.get_attribute(&attr.value) { + match self.elem.get_attribute(&attr.value) { Some(s) => vm.new_str(s), None => default.into_option().unwrap_or_else(|| vm.get_none()), } } - fn set_attr( - zelf: ElementRef, - attr: PyStringRef, - value: PyStringRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.elem + #[pymethod] + fn set_attr(&self, attr: PyStringRef, value: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + self.elem .set_attribute(&attr.value, &value.value) .map_err(|err| convert::js_to_py(vm, err)) } } -fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_alert(message: PyStringRef, vm: &VirtualMachine) -> PyResult { window() - .alert_with_message(&objstr::get_value(message)) + .alert_with_message(message.as_str()) .expect("alert() not to fail"); Ok(vm.get_none()) } -fn browser_confirm(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - +fn browser_confirm(message: PyStringRef, vm: &VirtualMachine) -> PyResult { let result = window() - .confirm_with_message(&objstr::get_value(message)) + .confirm_with_message(message.as_str()) .expect("confirm() not to fail"); Ok(vm.new_bool(result)) } -fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(message, Some(vm.ctx.str_type()))], - optional = [(default, Some(vm.ctx.str_type()))] - ); - - let result = if let Some(default) = default { - window().prompt_with_message_and_default( - &objstr::get_value(message), - &objstr::get_value(default), - ) +fn browser_prompt( + message: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let result = if let OptionalArg::Present(default) = default { + window().prompt_with_message_and_default(message.as_str(), default.as_str()) } else { - window().prompt_with_message(&objstr::get_value(message)) + window().prompt_with_message(message.as_str()) }; let result = match result.expect("prompt() not to fail") { @@ -348,15 +341,12 @@ fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(result) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let promise = py_class!(ctx, "Promise", ctx.object(), { - "then" => ctx.new_rustfunc(PyPromise::then), - "catch" => ctx.new_rustfunc(PyPromise::catch), - }); +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; - let document_class = py_class!(ctx, "Document", ctx.object(), { - "query" => ctx.new_rustfunc(Document::query), - }); + let promise = PyPromise::make_class(ctx); + + let document_class = Document::make_class(ctx); let document = PyObject::new( Document { @@ -366,12 +356,9 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { None, ); - let element = py_class!(ctx, "Element", ctx.object(), { - "get_attr" => ctx.new_rustfunc(Element::get_attr), - "set_attr" => ctx.new_rustfunc(Element::set_attr), - }); + let element = Element::make_class(ctx); - py_module!(ctx, "browser", { + py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), "request_animation_frame" => ctx.new_rustfunc(browser_request_animation_frame), "cancel_animation_frame" => ctx.new_rustfunc(browser_cancel_animation_frame), diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index af6ad029e..6f40fb182 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -4,11 +4,11 @@ use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbytes, objint, objsequence, objtype}; -use rustpython_vm::pyobject::{DictProtocol, PyObjectRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; -use crate::vm_class::{AccessibleVM, WASMVirtualMachine}; +use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { macro_rules! map_exceptions { @@ -81,11 +81,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { return Err(err); } }; - let acc_vm = AccessibleVM::from(wasm_vm.clone()); - let stored_vm = acc_vm - .upgrade() - .expect("acc. VM to be invalid when WASM vm is valid"); - let vm = &stored_vm.vm; + let vm = &stored_vm_from_wasm(&wasm_vm).vm; let mut py_func_args = PyFuncArgs::default(); if let Some(ref args) = args { for arg in args.values() { @@ -192,7 +188,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&vm.ctx, &String::from(js_sys::JsString::from(key)), py_val); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm) + .unwrap(); } dict.into_object() } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c7e637a9f..ff72dddbb 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -8,7 +8,7 @@ use wasm_bindgen::prelude::*; use rustpython_vm::compile; use rustpython_vm::frame::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyResult}; +use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult}; use rustpython_vm::VirtualMachine; use crate::browser_module::setup_browser_module; @@ -43,8 +43,26 @@ impl StoredVirtualMachine { // probably gets compiled down to a normal-ish static varible, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { - static STORED_VMS: RefCell>> = - RefCell::default(); + static STORED_VMS: RefCell>> = RefCell::default(); +} + +pub fn get_vm_id(vm: &VirtualMachine) -> &str { + vm.wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") +} +pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { + STORED_VMS.with(|cell| { + cell.borrow() + .get(&wasm_vm.id) + .expect("VirtualMachine is not valid") + .clone() + }) +} +pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak { + let id = get_vm_id(vm); + STORED_VMS + .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } #[wasm_bindgen(js_name = vmStore)] @@ -104,44 +122,6 @@ impl VMStore { } } -#[derive(Clone)] -pub(crate) struct AccessibleVM { - weak: Weak, - id: String, -} - -impl AccessibleVM { - pub fn from_id(id: String) -> AccessibleVM { - let weak = STORED_VMS - .with(|cell| Rc::downgrade(cell.borrow().get(&id).expect("WASM VM to be valid"))); - AccessibleVM { weak, id } - } - - pub fn upgrade(&self) -> Option> { - self.weak.upgrade() - } -} - -impl From for AccessibleVM { - fn from(vm: WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id) - } -} -impl From<&WASMVirtualMachine> for AccessibleVM { - fn from(vm: &WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id.clone()) - } -} -impl From<&VirtualMachine> for AccessibleVM { - fn from(vm: &VirtualMachine) -> AccessibleVM { - AccessibleVM::from_id( - vm.wasm_id - .clone() - .expect("VM passed to from::() to have wasm_id be Some()"), - ) - } -} - #[wasm_bindgen(js_name = VirtualMachine)] #[derive(Clone)] pub struct WASMVirtualMachine { @@ -219,15 +199,15 @@ impl WASMVirtualMachine { fn error() -> JsValue { TypeError::new("Unknown stdout option, please pass a function or 'console'").into() } - let print_fn: Box PyResult> = - if let Some(s) = stdout.as_string() { - match s.as_str() { - "console" => Box::new(wasm_builtins::builtin_print_console), - _ => return Err(error()), - } - } else if stdout.is_function() { - let func = js_sys::Function::from(stdout); - Box::new(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { + let print_fn: PyObjectRef = if let Some(s) = stdout.as_string() { + match s.as_str() { + "console" => vm.ctx.new_rustfunc(wasm_builtins::builtin_print_console), + _ => return Err(error()), + } + } else if stdout.is_function() { + let func = js_sys::Function::from(stdout); + vm.ctx + .new_rustfunc(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { func.call1( &JsValue::UNDEFINED, &wasm_builtins::format_print_args(vm, args)?.into(), @@ -235,16 +215,15 @@ impl WASMVirtualMachine { .map_err(|err| convert::js_to_py(vm, err))?; Ok(vm.get_none()) }) - } else if stdout.is_undefined() || stdout.is_null() { - fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { - Ok(vm.get_none()) - } - Box::new(noop) - } else { - return Err(error()); - }; - let rustfunc = vm.ctx.new_rustfunc(print_fn); - vm.ctx.set_attr(&vm.builtins, "print", rustfunc); + } else if stdout.is_undefined() || stdout.is_null() { + fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) + } + vm.ctx.new_rustfunc(noop) + } else { + return Err(error()); + }; + vm.set_attr(&vm.builtins, "print", print_fn).unwrap(); Ok(()) })? } @@ -261,12 +240,12 @@ impl WASMVirtualMachine { let mod_name = name.clone(); - let stdlib_init_fn = move |ctx: &PyContext| { - let py_mod = ctx.new_module(&name, ctx.new_dict()); + let stdlib_init_fn = move |vm: &VirtualMachine| { + let module = vm.ctx.new_module(&name, vm.ctx.new_dict()); for (key, value) in module_items.clone() { - ctx.set_attr(&py_mod, &key, value); + vm.set_attr(&module, key, value).unwrap(); } - py_mod + module }; vm.stdlib_inits diff --git a/whats_left.sh b/whats_left.sh new file mode 100755 index 000000000..e05051d28 --- /dev/null +++ b/whats_left.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +cd "$(dirname "$0")" +cd tests + +python3 not_impl_gen.py + +cd .. + +cargo run -- tests/snippets/whats_left_to_implement.py