diff --git a/Cargo.lock b/Cargo.lock index 762db6df6..4ee04733e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1315,6 +1315,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "optional" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" + [[package]] name = "parking_lot" version = "0.11.2" @@ -1524,9 +1530,9 @@ dependencies = [ [[package]] name = "radium" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "radix_trie" @@ -1954,6 +1960,7 @@ dependencies = [ "libc", "log", "memchr", + "memoffset", "nix 0.23.0", "num-bigint", "num-complex", @@ -1963,6 +1970,7 @@ dependencies = [ "num_cpus", "num_enum", "once_cell", + "optional", "parking_lot", "paste", "rand", diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index b267780f0..2a2768432 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -51,10 +51,14 @@ class WeakSet: self.update(data) def _commit_removals(self): - l = self._pending_removals + pop = self._pending_removals.pop discard = self.data.discard - while l: - discard(l.pop()) + while True: + try: + item = pop() + except IndexError: + return + discard(item) def __iter__(self): with _IterationGuard(self): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index c0127cabe..4df060766 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1753,8 +1753,6 @@ class AssortedBytesTest(unittest.TestCase): self.assertEqual(bytes(b"abc") < b"ab", False) self.assertEqual(bytes(b"abc") <= b"ab", False) - # TODO: RUSTPYTHON - @unittest.expectedFailure @test.support.requires_docstrings def test_doc(self): self.assertIsNotNone(bytearray.__doc__) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 87fe6044a..c0988f953 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -826,18 +826,12 @@ class TestCopy(unittest.TestCase): v[x] = y self.assertNotIn(x, u) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_weakkeydict(self): self._check_copy_weakdict(weakref.WeakKeyDictionary) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_weakvaluedict(self): self._check_copy_weakdict(weakref.WeakValueDictionary) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deepcopy_weakkeydict(self): class C(object): def __init__(self, i): @@ -857,8 +851,6 @@ class TestCopy(unittest.TestCase): del c self.assertEqual(len(v), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deepcopy_weakvaluedict(self): class C(object): def __init__(self, i): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 22142358a..8455d0bfb 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -90,8 +90,6 @@ class DictSetTest(unittest.TestCase): self.assertTrue(r == "dict_values(['ABC', 10])" or r == "dict_values([10, 'ABC'])") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_keys_set_operations(self): d1 = {'a': 1, 'b': 2} d2 = {'b': 3, 'c': 2} @@ -148,8 +146,6 @@ class DictSetTest(unittest.TestCase): self.assertTrue(de.keys().isdisjoint(de.keys())) self.assertTrue(de.keys().isdisjoint([1])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_items_set_operations(self): d1 = {'a': 1, 'b': 2} d2 = {'a': 2, 'b': 2} diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py index 92d52c3d1..5785cb464 100644 --- a/Lib/test/test_enumerate.py +++ b/Lib/test/test_enumerate.py @@ -94,8 +94,6 @@ class EnumerateTestCase(unittest.TestCase, PickleTest): self.assertEqual(list(self.enum(self.seq)), self.res) self.enum.__doc__ - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle(self): self.check_pickle(self.enum(self.seq), self.res) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index df6894947..18609f4e5 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -500,8 +500,6 @@ class TestPartialMethod(unittest.TestCase): a = A() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_arg_combinations(self): self.assertEqual(self.a.nothing(), ((self.a,), {})) self.assertEqual(self.a.nothing(5), ((self.a, 5), {})) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index c90c063cd..91d071f7b 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1,4 +1,3 @@ - # Python test set -- math module # XXXX Should not do tests around zero only @@ -19,7 +18,6 @@ eps = 1E-05 NAN = float('nan') INF = float('inf') NINF = float('-inf') - FLOAT_MAX = sys.float_info.max FLOAT_MIN = sys.float_info.min @@ -43,8 +41,10 @@ def to_ulps(x): adjacent floats are converted to adjacent integers. Then abs(ulps(x) - ulps(y)) gives the difference in ulps between two floats. + The results from this function will only make sense on platforms where native doubles are represented in IEEE 754 binary64 format. + Note: 0.0 and -0.0 are converted to 0 and -1, respectively. """ n = struct.unpack('> 1 if not numfactors: @@ -94,6 +95,7 @@ def partial_product(start, stop): def py_factorial(n): """Factorial of nonnegative integer n, via "Binary Split Factorial Formula" described at http://www.luschny.de/math/factorial/binarysplitfact.html + """ inner = outer = 1 for i in reversed(range(n.bit_length())): @@ -105,6 +107,7 @@ def ulp_abs_check(expected, got, ulp_tol, abs_tol): """Given finite floats `expected` and `got`, check that they're approximately equal to within the given number of ulps or the given absolute tolerance, whichever is bigger. + Returns None on success and an error message on failure. """ ulp_error = abs(to_ulps(expected) - to_ulps(got)) @@ -120,12 +123,14 @@ def ulp_abs_check(expected, got, ulp_tol, abs_tol): def parse_mtestfile(fname): """Parse a file with test values + -- starts a comment blank lines, or lines containing only a comment, are ignored other lines are expected to have the form id fn arg -> expected [flag]* + """ - with open(fname) as fp: + with open(fname, encoding="utf-8") as fp: for line in fp: # strip comments, and skip blank lines if '--' in line: @@ -144,10 +149,11 @@ def parse_mtestfile(fname): def parse_testfile(fname): """Parse a file with test values + Empty lines or lines starting with -- are ignored yields id, fn, arg_real, arg_imag, exp_real, exp_imag """ - with open(fname) as fp: + with open(fname, encoding="utf-8") as fp: for line in fp: # skip comment lines and blank lines if line.startswith('--') or not line.strip(): @@ -170,9 +176,11 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): """Compare arguments expected and got, as floats, if either is a float, using a tolerance expressed in multiples of ulp(expected) or absolutely (if given and greater). + As a convenience, when neither argument is a float, and for non-finite floats, exact equality is demanded. Also, nan==nan as far as this function is concerned. + Returns None on success and an error message on failure. """ @@ -232,6 +240,7 @@ class MathTests(unittest.TestCase): """Compare arguments expected and got, as floats, if either is a float, using a tolerance expressed in multiples of ulp(expected) or absolutely, whichever is greater. + As a convenience, when neither argument is a float, and for non-finite floats, exact equality is demanded. Also, nan==nan in this function. @@ -492,17 +501,11 @@ class MathTests(unittest.TestCase): self.assertRaises(ValueError, math.factorial, -1) self.assertRaises(ValueError, math.factorial, -10**100) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testFactorialNonIntegers(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual(math.factorial(5.0), 120) - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, math.factorial, 5.2) - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, math.factorial, -1.0) - with self.assertWarns(DeprecationWarning): - self.assertRaises(ValueError, math.factorial, -1e100) + self.assertRaises(TypeError, math.factorial, 5.0) + self.assertRaises(TypeError, math.factorial, 5.2) + self.assertRaises(TypeError, math.factorial, -1.0) + self.assertRaises(TypeError, math.factorial, -1e100) self.assertRaises(TypeError, math.factorial, decimal.Decimal('5')) self.assertRaises(TypeError, math.factorial, decimal.Decimal('5.2')) self.assertRaises(TypeError, math.factorial, "5") @@ -513,8 +516,7 @@ class MathTests(unittest.TestCase): # Currently raises OverflowError for inputs that are too large # to fit into a C long. self.assertRaises(OverflowError, math.factorial, 10**100) - with self.assertWarns(DeprecationWarning): - self.assertRaises(OverflowError, math.factorial, 1e100) + self.assertRaises(TypeError, math.factorial, 1e100) def testFloor(self): self.assertRaises(TypeError, math.floor) @@ -587,7 +589,6 @@ class MathTests(unittest.TestCase): self.assertEqual(math.frexp(NINF)[0], NINF) self.assertTrue(math.isnan(math.frexp(NAN)[0])) - @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "fsum is not exact on machines with double rounding") @@ -611,6 +612,7 @@ class MathTests(unittest.TestCase): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function at http://code.activestate.com/recipes/393090/ + """ tmant, texp = 0, 0 for x in iterable: @@ -684,8 +686,6 @@ class MathTests(unittest.TestCase): s = msum(vals) self.assertEqual(msum(vals), math.fsum(vals)) - - # Python 3.9 def testGcd(self): gcd = math.gcd self.assertEqual(gcd(0, 0), 0) @@ -726,7 +726,7 @@ class MathTests(unittest.TestCase): self.assertRaises(TypeError, gcd, 120.0, 84) self.assertRaises(TypeError, gcd, 120, 84.0) self.assertRaises(TypeError, gcd, 120, 1, 84.0) - #self.assertEqual(gcd(MyIndexable(120), MyIndexable(84)), 12) # TODO: RUSTPYTHON + self.assertEqual(gcd(MyIndexable(120), MyIndexable(84)), 12) def testHypot(self): from decimal import Decimal @@ -795,13 +795,79 @@ class MathTests(unittest.TestCase): # Verify scaling for extremely large values fourthmax = FLOAT_MAX / 4.0 for n in range(32): - self.assertEqual(hypot(*([fourthmax]*n)), fourthmax * math.sqrt(n)) + self.assertTrue(math.isclose(hypot(*([fourthmax]*n)), + fourthmax * math.sqrt(n))) # Verify scaling for extremely small values for exp in range(32): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "hypot() loses accuracy on machines with double rounding") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def testHypotAccuracy(self): + # Verify improved accuracy in cases that were known to be inaccurate. + # + # The new algorithm's accuracy depends on IEEE 754 arithmetic + # guarantees, on having the usual ROUND HALF EVEN rounding mode, on + # the system not having double rounding due to extended precision, + # and on the compiler maintaining the specified order of operations. + # + # This test is known to succeed on most of our builds. If it fails + # some build, we either need to add another skipIf if the cause is + # identifiable; otherwise, we can remove this test entirely. + + hypot = math.hypot + Decimal = decimal.Decimal + high_precision = decimal.Context(prec=500) + + for hx, hy in [ + # Cases with a 1 ulp error in Python 3.7 compiled with Clang + ('0x1.10e89518dca48p+29', '0x1.1970f7565b7efp+30'), + ('0x1.10106eb4b44a2p+29', '0x1.ef0596cdc97f8p+29'), + ('0x1.459c058e20bb7p+30', '0x1.993ca009b9178p+29'), + ('0x1.378371ae67c0cp+30', '0x1.fbe6619854b4cp+29'), + ('0x1.f4cd0574fb97ap+29', '0x1.50fe31669340ep+30'), + ('0x1.494b2cdd3d446p+29', '0x1.212a5367b4c7cp+29'), + ('0x1.f84e649f1e46dp+29', '0x1.1fa56bef8eec4p+30'), + ('0x1.2e817edd3d6fap+30', '0x1.eb0814f1e9602p+29'), + ('0x1.0d3a6e3d04245p+29', '0x1.32a62fea52352p+30'), + ('0x1.888e19611bfc5p+29', '0x1.52b8e70b24353p+29'), + + # Cases with 2 ulp error in Python 3.8 + ('0x1.538816d48a13fp+29', '0x1.7967c5ca43e16p+29'), + ('0x1.57b47b7234530p+29', '0x1.74e2c7040e772p+29'), + ('0x1.821b685e9b168p+30', '0x1.677dc1c1e3dc6p+29'), + ('0x1.9e8247f67097bp+29', '0x1.24bd2dc4f4baep+29'), + ('0x1.b73b59e0cb5f9p+29', '0x1.da899ab784a97p+28'), + ('0x1.94a8d2842a7cfp+30', '0x1.326a51d4d8d8ap+30'), + ('0x1.e930b9cd99035p+29', '0x1.5a1030e18dff9p+30'), + ('0x1.1592bbb0e4690p+29', '0x1.a9c337b33fb9ap+29'), + ('0x1.1243a50751fd4p+29', '0x1.a5a10175622d9p+29'), + ('0x1.57a8596e74722p+30', '0x1.42d1af9d04da9p+30'), + + # Cases with 1 ulp error in version fff3c28052e6b0 + ('0x1.ee7dbd9565899p+29', '0x1.7ab4d6fc6e4b4p+29'), + ('0x1.5c6bfbec5c4dcp+30', '0x1.02511184b4970p+30'), + ('0x1.59dcebba995cap+30', '0x1.50ca7e7c38854p+29'), + ('0x1.768cdd94cf5aap+29', '0x1.9cfdc5571d38ep+29'), + ('0x1.dcf137d60262ep+29', '0x1.1101621990b3ep+30'), + ('0x1.3a2d006e288b0p+30', '0x1.e9a240914326cp+29'), + ('0x1.62a32f7f53c61p+29', '0x1.47eb6cd72684fp+29'), + ('0x1.d3bcb60748ef2p+29', '0x1.3f13c4056312cp+30'), + ('0x1.282bdb82f17f3p+30', '0x1.640ba4c4eed3ap+30'), + ('0x1.89d8c423ea0c6p+29', '0x1.d35dcfe902bc3p+29'), + ]: + x = float.fromhex(hx) + y = float.fromhex(hy) + with self.subTest(hx=hx, hy=hy, x=x, y=y): + with decimal.localcontext(high_precision): + z = float((Decimal(x)**2 + Decimal(y)**2).sqrt()) + self.assertEqual(hypot(x, y), z) + def testDist(self): from decimal import Decimal as D from fractions import Fraction as F @@ -904,8 +970,8 @@ class MathTests(unittest.TestCase): for n in range(32): p = (fourthmax,) * n q = (0.0,) * n - self.assertEqual(dist(p, q), fourthmax * math.sqrt(n)) - self.assertEqual(dist(q, p), fourthmax * math.sqrt(n)) + self.assertTrue(math.isclose(dist(p, q), fourthmax * math.sqrt(n))) + self.assertTrue(math.isclose(dist(q, p), fourthmax * math.sqrt(n))) # Verify scaling for extremely small values for exp in range(32): @@ -968,8 +1034,7 @@ class MathTests(unittest.TestCase): with self.assertRaises(TypeError): math.isqrt(value) - # Python 3.9 - def testlcm(self): + def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) self.assertEqual(lcm(1, 0), 0) @@ -1011,7 +1076,7 @@ class MathTests(unittest.TestCase): self.assertRaises(TypeError, lcm, 120.0, 84) self.assertRaises(TypeError, lcm, 120, 84.0) self.assertRaises(TypeError, lcm, 120, 0, 84.0) - # self.assertEqual(lcm(MyIndexable(120), MyIndexable(84)), 840) # TODO: RUSTPYTHON + self.assertEqual(lcm(MyIndexable(120), MyIndexable(84)), 840) def testLdexp(self): self.assertRaises(TypeError, math.ldexp) @@ -1987,7 +2052,7 @@ class MathTests(unittest.TestCase): # min and max self.assertEqual(math.ulp(0.0), - sys.float_info.min * sys.float_info.epsilon) + sys.float_info.min * sys.float_info.epsilon) self.assertEqual(math.ulp(FLOAT_MAX), FLOAT_MAX - math.nextafter(FLOAT_MAX, -INF)) @@ -2024,6 +2089,7 @@ class MathTests(unittest.TestCase): def assertEqualSign(self, x, y): """Similar to assertEqual(), but compare also the sign with copysign(). + Function useful to compare signed zeros. """ self.assertEqual(x, y) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 68d0bfa0f..9ddddbb8e 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -343,8 +343,6 @@ class AbstractMemoryTests: m = self._view(b) self.assertRaises(ValueError, hash, m) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weakref(self): # Check memoryviews are weakrefable for tp in self._types: diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index ba1810c24..690009fde 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -318,8 +318,6 @@ class PositionalOnlyTestCase(unittest.TestCase): with self.assertRaisesRegex(TypeError, r"g\(\) takes 2 positional arguments but 3 were given"): f(1,2)(3,4,5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_same_keyword_as_positional_with_kwargs(self): def f(something,/,**kwargs): return (something, kwargs) diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index ecdd92b6c..9e79775ac 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -234,8 +234,6 @@ class SliceTest(unittest.TestCase): x[1:2] = 42 self.assertEqual(tmp, [(slice(1, 2), 42)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle(self): s = slice(10, 20, 3) for protocol in (0,1,2): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 3f8a6f0fd..c24f9a633 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1454,8 +1454,6 @@ class TestTemporaryDirectory(BaseTestCase): self.assertNotIn("Exception ", err) self.assertIn("ResourceWarning: Implicitly cleaning up", err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warnings_on_cleanup(self): # ResourceWarning will be triggered by __del__ with self.do_create() as dir: diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 45f3480ad..cb1c875eb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3731,8 +3731,6 @@ class TypedDictTests(BaseTestCase): self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) self.assertEqual(Emp.__total__, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_typeddict_special_keyword_names(self): TD = TypedDict("TD", cls=type, self=object, typename=str, _typename=int, fields=list, _fields=dict) self.assertEqual(TD.__name__, 'TD') diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 05eeb846d..bad28bba0 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -661,8 +661,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertFalse("\u20ac".isascii()) self.assertFalse("\U0010ffff".isascii()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_isdecimal(self): self.checkequalnofix(False, '', 'isdecimal') self.checkequalnofix(False, 'a', 'isdecimal') diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 97efd9e74..3b783c456 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -113,8 +113,6 @@ class ReferencesTestCase(TestBase): del o repr(wr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_callback(self): self.check_basic_callback(C) self.check_basic_callback(create_function) @@ -132,8 +130,6 @@ class ReferencesTestCase(TestBase): self.check_basic_ref(create_cfunction) self.check_basic_callback(create_cfunction) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiple_callbacks(self): o = C() ref1 = weakref.ref(o, self.callback) @@ -166,7 +162,8 @@ class ReferencesTestCase(TestBase): c = C() self.assertRaises(TypeError, weakref.ref, c, callback=None) - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'Passed a callback to weakproxy, but weakproxy does not yet support proxies.'") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_proxy_ref(self): o = C() o.bar = 1 @@ -201,8 +198,6 @@ class ReferencesTestCase(TestBase): self.assertIsNone(ref(), "ref2 should be dead after deleting object reference") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ref_reuse(self): o = C() ref1 = weakref.ref(o) @@ -332,7 +327,8 @@ class ReferencesTestCase(TestBase): def test_shared_ref_without_callback(self): self.check_shared_without_callback(weakref.ref) - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'Passed a callback to weakproxy, but weakproxy does not yet support proxies.'") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_shared_proxy_without_callback(self): self.check_shared_without_callback(weakref.proxy) @@ -439,7 +435,6 @@ class ReferencesTestCase(TestBase): # can be killed in the middle of the call "blech" in p - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'Passed a callback to weakproxy, but weakproxy does not yet support proxies.'") def test_getweakrefcount(self): o = C() ref1 = weakref.ref(o) @@ -461,8 +456,6 @@ class ReferencesTestCase(TestBase): self.assertEqual(weakref.getweakrefcount(1), 0, "got wrong number of weak reference objects for int") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getweakrefs(self): o = C() ref1 = weakref.ref(o, self.callback) @@ -828,8 +821,6 @@ class ReferencesTestCase(TestBase): # No exception should be raised here gc.collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_classes(self): # Check that classes are weakrefable. class A(object): @@ -1003,8 +994,6 @@ class SubclassableWeakrefTestCase(TestBase): self.assertIn(r1, refs[1:]) self.assertIn(r3, refs[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs_dont_conflate_callbacks(self): class MyRef(weakref.ref): pass @@ -1107,8 +1096,6 @@ class WeakMethodTestCase(unittest.TestCase): gc.collect() self.assertIs(r(), None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_callback_when_object_dead(self): # Test callback behaviour when object dies first. C = self._subclass() @@ -1125,8 +1112,6 @@ class WeakMethodTestCase(unittest.TestCase): gc.collect() self.assertEqual(calls, [r]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_callback_when_method_dead(self): # Test callback behaviour when method dies first. C = self._subclass() @@ -1154,8 +1139,6 @@ class WeakMethodTestCase(unittest.TestCase): del r self.assertIs(wr(), None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_equality(self): def _eq(a, b): self.assertTrue(a == b) @@ -1282,8 +1265,6 @@ class MappingTestCase(TestBase): def test_weak_valued_len_race(self): self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_values(self): # # This exercises d.copy(), d.items(), d[], del d[], len(d). @@ -1313,8 +1294,6 @@ class MappingTestCase(TestBase): dict[2] = C() self.assertRaises(KeyError, dict.__getitem__, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keys(self): # # This exercises d.copy(), d.items(), d[] = v, d[], del d[], @@ -1492,8 +1471,6 @@ class MappingTestCase(TestBase): self.assertEqual(len(dict), 0) self.assertEqual(len(dict), 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keys_destroy_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed dict, objects = self.make_weak_keyed_dict() @@ -1520,8 +1497,6 @@ class MappingTestCase(TestBase): dict, objects = self.make_weak_keyed_dict() self.check_weak_del_and_len_while_iterating(dict, testcontext) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_values_destroy_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed dict, objects = self.make_weak_valued_dict() @@ -1724,8 +1699,6 @@ class MappingTestCase(TestBase): self.assertRaises(TypeError, d.__getitem__, 13) self.assertRaises(TypeError, d.__setitem__, 13, 13) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_cascading_deletes(self): # SF bug 742860. For some reason, before 2.3 __delitem__ iterated # over the keys via self.data.iterkeys(). If things vanished from @@ -1921,8 +1894,6 @@ class FinalizeTestCase(unittest.TestCase): if sys.implementation.name != 'cpython': support.gc_collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_finalize(self): def add(x,y,z): res.append(x + y + z) @@ -1995,8 +1966,6 @@ class FinalizeTestCase(unittest.TestCase): self.assertRaises(TypeError, weakref.finalize, a) self.assertRaises(TypeError, weakref.finalize) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_order(self): a = self.A() res = [] @@ -2036,8 +2005,6 @@ class FinalizeTestCase(unittest.TestCase): expected = ['A', 'f3', 'B', 'C', 'f4', 'f2', 'f1', 'D'] self.assertEqual(res, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_all_freed(self): # we want a weakrefable subclass of weakref.finalize class MyFinalizer(weakref.finalize): diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 40a3bee49..307203850 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -43,8 +43,6 @@ class TestWeakSet(unittest.TestCase): def test_new_or_init(self): self.assertRaises(TypeError, WeakSet, [], 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_len(self): self.assertEqual(len(self.s), len(self.d)) self.assertEqual(len(self.fs), 1) @@ -60,8 +58,6 @@ class TestWeakSet(unittest.TestCase): del self.obj self.assertNotIn(ustr('F'), self.fs) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union(self): u = self.s.union(self.items2) for c in self.letters: @@ -84,8 +80,6 @@ class TestWeakSet(unittest.TestCase): self.assertEqual(self.s | set(self.items2), i) self.assertEqual(self.s | frozenset(self.items2), i) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_intersection(self): s = WeakSet(self.letters) i = s.intersection(self.items2) @@ -123,8 +117,6 @@ class TestWeakSet(unittest.TestCase): self.assertEqual(self.s - set(self.items2), i) self.assertEqual(self.s - frozenset(self.items2), i) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_symmetric_difference(self): i = self.s.symmetric_difference(self.items2) for c in self.letters: @@ -215,8 +207,6 @@ class TestWeakSet(unittest.TestCase): self.assertEqual(self.s, dup) self.assertNotEqual(id(self.s), id(dup)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_add(self): x = ustr('Q') self.s.add(x) @@ -352,8 +342,6 @@ class TestWeakSet(unittest.TestCase): s2 = WeakSet() self.assertFalse(s1 != s2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_destroy_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed # Create new items to be sure no-one else holds a reference @@ -370,8 +358,6 @@ class TestWeakSet(unittest.TestCase): # The removal has been committed self.assertEqual(len(s), len(items)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_destroy_and_mutate_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed items = [ustr(c) for c in string.ascii_letters] diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 6b2426d70..6950f8ba5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2344,8 +2344,6 @@ class BasicElementTest(ElementTestCase, unittest.TestCase): gc_collect() self.assertIsNone(wref()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weakref(self): flag = False def wref_cb(w): diff --git a/Lib/weakref.py b/Lib/weakref.py index 99de2eab7..994ea8aa3 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -2,7 +2,7 @@ This module is an implementation of PEP 205: -http://www.python.org/dev/peps/pep-0205/ +https://www.python.org/dev/peps/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; @@ -33,6 +33,9 @@ __all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", "WeakSet", "WeakMethod", "finalize"] +_collections_abc.Set.register(WeakSet) +_collections_abc.MutableSet.register(WeakSet) + class WeakMethod(ref): """ A custom `weakref.ref` subclass which simulates a weak reference to @@ -75,14 +78,14 @@ class WeakMethod(ref): if not self._alive or not other._alive: return self is other return ref.__eq__(self, other) and self._func_ref == other._func_ref - return False + return NotImplemented def __ne__(self, other): if isinstance(other, WeakMethod): if not self._alive or not other._alive: return self is not other return ref.__ne__(self, other) or self._func_ref != other._func_ref - return True + return NotImplemented __hash__ = ref.__hash__ @@ -99,13 +102,7 @@ class WeakValueDictionary(_collections_abc.MutableMapping): # objects are unwrapped on the way out, and we always wrap on the # way in). - def __init__(*args, **kw): - if not args: - raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " - "object needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) + def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: @@ -114,22 +111,25 @@ class WeakValueDictionary(_collections_abc.MutableMapping): else: # Atomic removal is necessary since this function # can be called asynchronously by the GC - _atomic_removal(d, wr.key) + _atomic_removal(self.data, wr.key) self._remove = remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() - self.data = d = {} - self.update(*args, **kw) + self.data = {} + self.update(other, **kw) - def _commit_removals(self): - l = self._pending_removals + def _commit_removals(self, _atomic_removal=_remove_dead_weakref): + pop = self._pending_removals.pop d = self.data # We shouldn't encounter any KeyError, because this method should # always be called *before* mutating the dict. - while l: - key = l.pop() - _remove_dead_weakref(d, key) + while True: + try: + key = pop() + except IndexError: + return + _atomic_removal(d, key) def __getitem__(self, key): if self._pending_removals: @@ -171,10 +171,11 @@ class WeakValueDictionary(_collections_abc.MutableMapping): if self._pending_removals: self._commit_removals() new = WeakValueDictionary() - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + with _IterationGuard(self): + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy @@ -184,10 +185,11 @@ class WeakValueDictionary(_collections_abc.MutableMapping): if self._pending_removals: self._commit_removals() new = self.__class__() - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + with _IterationGuard(self): + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): @@ -285,24 +287,17 @@ class WeakValueDictionary(_collections_abc.MutableMapping): else: return o - def update(*args, **kwargs): - if not args: - raise TypeError("descriptor 'update' of 'WeakValueDictionary' " - "object needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - dict = args[0] if args else None + def update(self, other=None, /, **kwargs): if self._pending_removals: self._commit_removals() d = self.data - if dict is not None: - if not hasattr(dict, "items"): - dict = type({})(dict) - for key, o in dict.items(): + if other is not None: + if not hasattr(other, "items"): + other = dict(other) + for key, o in other.items(): d[key] = KeyedRef(o, self._remove, key) - if len(kwargs): - self.update(kwargs) + for key, o in kwargs.items(): + d[key] = KeyedRef(o, self._remove, key) def valuerefs(self): """Return a list of weak references to the values. @@ -318,6 +313,25 @@ class WeakValueDictionary(_collections_abc.MutableMapping): self._commit_removals() return list(self.data.values()) + def __ior__(self, other): + self.update(other) + return self + + def __or__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.copy() + c.update(other) + return c + return NotImplemented + + def __ror__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.__class__() + c.update(other) + c.update(self) + return c + return NotImplemented + class KeyedRef(ref): """Specialized reference that includes a key corresponding to the value. @@ -359,7 +373,10 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): if self._iterating: self._pending_removals.append(k) else: - del self.data[k] + try: + del self.data[k] + except KeyError: + pass self._remove = remove # A list of dead weakrefs (keys to be removed) self._pending_removals = [] @@ -373,11 +390,16 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): # because a dead weakref never compares equal to a live weakref, # even if they happened to refer to equal objects. # However, it means keys may already have been removed. - l = self._pending_removals + pop = self._pending_removals.pop d = self.data - while l: + while True: try: - del d[l.pop()] + key = pop() + except IndexError: + return + + try: + del d[key] except KeyError: pass @@ -408,10 +430,11 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def copy(self): new = WeakKeyDictionary() - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -419,10 +442,11 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -484,7 +508,7 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) - def update(self, dict=None, **kwargs): + def update(self, dict=None, /, **kwargs): d = self.data if dict is not None: if not hasattr(dict, "items"): @@ -494,6 +518,25 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): if len(kwargs): self.update(kwargs) + def __ior__(self, other): + self.update(other) + return self + + def __or__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.copy() + c.update(other) + return c + return NotImplemented + + def __ror__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.__class__() + c.update(other) + c.update(self) + return c + return NotImplemented + class finalize: """Class for finalization of weakrefable objects @@ -523,7 +566,7 @@ class finalize: class _Info: __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") - def __init__(self, obj, func, *args, **kwargs): + def __init__(self, obj, func, /, *args, **kwargs): if not self._registered_with_atexit: # We may register the exit function more than once because # of a thread race, but that is harmless diff --git a/common/Cargo.toml b/common/Cargo.toml index e74de0855..4b845c737 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -20,7 +20,7 @@ once_cell = "1.4.1" siphasher = "0.3" rand = "0.8" volatile = "0.3" -radium = "0.6" +radium = "0.7" libc = "0.2.101" ascii = "1.0" unic-ucd-category = "0.9" diff --git a/common/src/atomic.rs b/common/src/atomic.rs index 5b4d77a74..afe4afb44 100644 --- a/common/src/atomic.rs +++ b/common/src/atomic.rs @@ -1,3 +1,4 @@ +use core::ptr::{self, NonNull}; pub use core::sync::atomic::*; pub use radium::Radium; @@ -50,3 +51,77 @@ impl sealed::Sealed for *mut T {} impl PyAtomicScalar for *mut T { type Radium = atomic_ty!(*mut T, AtomicPtr); } + +pub struct OncePtr { + inner: PyAtomic<*mut T>, +} + +impl Default for OncePtr { + fn default() -> Self { + Self::new() + } +} + +impl OncePtr { + #[inline] + pub fn new() -> Self { + OncePtr { + inner: Radium::new(ptr::null_mut()), + } + } + + pub fn get(&self) -> Option> { + NonNull::new(self.inner.load(Ordering::Acquire)) + } + + pub fn set(&self, value: NonNull) -> Result<(), NonNull> { + let exchange = self.inner.compare_exchange( + ptr::null_mut(), + value.as_ptr(), + Ordering::AcqRel, + Ordering::Acquire, + ); + match exchange { + Ok(_) => Ok(()), + Err(_) => Err(value), + } + } + + pub fn get_or_init(&self, f: F) -> NonNull + where + F: FnOnce() -> Box, + { + enum Void {} + match self.get_or_try_init(|| Ok::<_, Void>(f())) { + Ok(val) => val, + Err(void) => match void {}, + } + } + + pub fn get_or_try_init(&self, f: F) -> Result, E> + where + F: FnOnce() -> Result, E>, + { + if let Some(val) = self.get() { + return Ok(val); + } + + Ok(self.initialize(f()?)) + } + + #[cold] + fn initialize(&self, val: Box) -> NonNull { + let ptr = Box::into_raw(val); + let exchange = + self.inner + .compare_exchange(ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire); + let ptr = match exchange { + Ok(_) => ptr, + Err(winner) => { + drop(unsafe { Box::from_raw(ptr) }); + winner + } + }; + unsafe { NonNull::new_unchecked(ptr) } + } +} diff --git a/common/src/hash.rs b/common/src/hash.rs index 20dabf127..e3ed7d1b8 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -169,7 +169,7 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { } #[inline(always)] -fn fix_sentinel(x: PyHash) -> PyHash { +pub fn fix_sentinel(x: PyHash) -> PyHash { if x == SENTINEL { -2 } else { diff --git a/common/src/lib.rs b/common/src/lib.rs index 0bd2b2c36..348733b3e 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -9,8 +9,10 @@ pub mod cmp; pub mod encodings; pub mod float_ops; pub mod hash; +pub mod linked_list; pub mod lock; pub mod rc; +pub mod refcount; pub mod static_cell; pub mod str; diff --git a/common/src/linked_list.rs b/common/src/linked_list.rs new file mode 100644 index 000000000..5834a1443 --- /dev/null +++ b/common/src/linked_list.rs @@ -0,0 +1,375 @@ +//! This module is modified from tokio::util::linked_list: https://github.com/tokio-rs/tokio/blob/master/tokio/src/util/linked_list.rs +//! Tokio is licensed under the MIT license: +//! +//! Copyright (c) 2021 Tokio Contributors +//! +//! Permission is hereby granted, free of charge, to any +//! person obtaining a copy of this software and associated +//! documentation files (the "Software"), to deal in the +//! Software without restriction, including without +//! limitation the rights to use, copy, modify, merge, +//! publish, distribute, sublicense, and/or sell copies of +//! the Software, and to permit persons to whom the Software +//! is furnished to do so, subject to the following +//! conditions: +//! +//! The above copyright notice and this permission notice +//! shall be included in all copies or substantial portions +//! of the Software. +//! +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +//! ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +//! TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +//! PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +//! SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +//! CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +//! OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +//! IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +//! DEALINGS IN THE SOFTWARE. +//! +//! Original header: +//! +//! An intrusive double linked list of data. +//! +//! The data structure supports tracking pinned nodes. Most of the data +//! structure's APIs are `unsafe` as they require the caller to ensure the +//! specified node is actually contained by the list. + +#![allow(clippy::new_without_default, clippy::missing_safety_doc)] + +use core::cell::UnsafeCell; +use core::fmt; +use core::marker::{PhantomData, PhantomPinned}; +use core::mem::ManuallyDrop; +use core::ptr::{self, NonNull}; + +/// An intrusive linked list. +/// +/// Currently, the list is not emptied on drop. It is the caller's +/// responsibility to ensure the list is empty before dropping it. +pub struct LinkedList { + /// Linked list head + head: Option>, + + // /// Linked list tail + // tail: Option>, + /// Node type marker. + _marker: PhantomData<*const L>, +} + +unsafe impl Send for LinkedList where L::Target: Send {} +unsafe impl Sync for LinkedList where L::Target: Sync {} + +/// Defines how a type is tracked within a linked list. +/// +/// In order to support storing a single type within multiple lists, accessing +/// the list pointers is decoupled from the entry type. +/// +/// # Safety +/// +/// Implementations must guarantee that `Target` types are pinned in memory. In +/// other words, when a node is inserted, the value will not be moved as long as +/// it is stored in the list. +pub unsafe trait Link { + /// Handle to the list entry. + /// + /// This is usually a pointer-ish type. + type Handle; + + /// Node type. + type Target; + + /// Convert the handle to a raw pointer without consuming the handle. + #[allow(clippy::wrong_self_convention)] + fn as_raw(handle: &Self::Handle) -> NonNull; + + /// Convert the raw pointer to a handle + unsafe fn from_raw(ptr: NonNull) -> Self::Handle; + + /// Return the pointers for a node + unsafe fn pointers(target: NonNull) -> NonNull>; +} + +/// Previous / next pointers. +pub struct Pointers { + inner: UnsafeCell>, +} +/// We do not want the compiler to put the `noalias` attribute on mutable +/// references to this type, so the type has been made `!Unpin` with a +/// `PhantomPinned` field. +/// +/// Additionally, we never access the `prev` or `next` fields directly, as any +/// such access would implicitly involve the creation of a reference to the +/// field, which we want to avoid since the fields are not `!Unpin`, and would +/// hence be given the `noalias` attribute if we were to do such an access. +/// As an alternative to accessing the fields directly, the `Pointers` type +/// provides getters and setters for the two fields, and those are implemented +/// using raw pointer casts and offsets, which is valid since the struct is +/// #[repr(C)]. +/// +/// See this link for more information: +/// +#[repr(C)] +struct PointersInner { + /// The previous node in the list. null if there is no previous node. + /// + /// This field is accessed through pointer manipulation, so it is not dead code. + #[allow(dead_code)] + prev: Option>, + + /// The next node in the list. null if there is no previous node. + /// + /// This field is accessed through pointer manipulation, so it is not dead code. + #[allow(dead_code)] + next: Option>, + + /// This type is !Unpin due to the heuristic from: + /// + _pin: PhantomPinned, +} + +unsafe impl Send for Pointers {} +unsafe impl Sync for Pointers {} + +// ===== impl LinkedList ===== + +impl LinkedList { + /// Creates an empty linked list. + pub const fn new() -> LinkedList { + LinkedList { + head: None, + // tail: None, + _marker: PhantomData, + } + } +} + +impl LinkedList { + /// Adds an element first in the list. + pub fn push_front(&mut self, val: L::Handle) { + // The value should not be dropped, it is being inserted into the list + let val = ManuallyDrop::new(val); + let ptr = L::as_raw(&*val); + assert_ne!(self.head, Some(ptr)); + unsafe { + L::pointers(ptr).as_mut().set_next(self.head); + L::pointers(ptr).as_mut().set_prev(None); + + if let Some(head) = self.head { + L::pointers(head).as_mut().set_prev(Some(ptr)); + } + + self.head = Some(ptr); + + // if self.tail.is_none() { + // self.tail = Some(ptr); + // } + } + } + + // /// Removes the last element from a list and returns it, or None if it is + // /// empty. + // pub fn pop_back(&mut self) -> Option { + // unsafe { + // let last = self.tail?; + // self.tail = L::pointers(last).as_ref().get_prev(); + + // if let Some(prev) = L::pointers(last).as_ref().get_prev() { + // L::pointers(prev).as_mut().set_next(None); + // } else { + // self.head = None + // } + + // L::pointers(last).as_mut().set_prev(None); + // L::pointers(last).as_mut().set_next(None); + + // Some(L::from_raw(last)) + // } + // } + + /// Returns whether the linked list does not contain any node + pub fn is_empty(&self) -> bool { + self.head.is_none() + // if self.head.is_some() { + // return false; + // } + + // assert!(self.tail.is_none()); + // true + } + + /// Removes the specified node from the list + /// + /// # Safety + /// + /// The caller **must** ensure that `node` is currently contained by + /// `self` or not contained by any other list. + pub unsafe fn remove(&mut self, node: NonNull) -> Option { + if let Some(prev) = L::pointers(node).as_ref().get_prev() { + debug_assert_eq!(L::pointers(prev).as_ref().get_next(), Some(node)); + L::pointers(prev) + .as_mut() + .set_next(L::pointers(node).as_ref().get_next()); + } else { + if self.head != Some(node) { + return None; + } + + self.head = L::pointers(node).as_ref().get_next(); + } + + if let Some(next) = L::pointers(node).as_ref().get_next() { + debug_assert_eq!(L::pointers(next).as_ref().get_prev(), Some(node)); + L::pointers(next) + .as_mut() + .set_prev(L::pointers(node).as_ref().get_prev()); + } else { + // // This might be the last item in the list + // if self.tail != Some(node) { + // return None; + // } + + // self.tail = L::pointers(node).as_ref().get_prev(); + } + + L::pointers(node).as_mut().set_next(None); + L::pointers(node).as_mut().set_prev(None); + + Some(L::from_raw(node)) + } + + // pub fn last(&self) -> Option<&L::Target> { + // let tail = self.tail.as_ref()?; + // unsafe { Some(&*tail.as_ptr()) } + // } + + // === rustpython additions === + + pub fn iter(&self) -> impl Iterator { + std::iter::successors(self.head, |node| unsafe { + L::pointers(*node).as_ref().get_next() + }) + .map(|ptr| unsafe { ptr.as_ref() }) + } +} + +impl fmt::Debug for LinkedList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LinkedList") + .field("head", &self.head) + // .field("tail", &self.tail) + .finish() + } +} + +impl Default for LinkedList { + fn default() -> Self { + Self::new() + } +} + +// ===== impl DrainFilter ===== + +pub struct DrainFilter<'a, T: Link, F> { + list: &'a mut LinkedList, + filter: F, + curr: Option>, +} + +impl LinkedList { + pub fn drain_filter(&mut self, filter: F) -> DrainFilter<'_, T, F> + where + F: FnMut(&mut T::Target) -> bool, + { + let curr = self.head; + DrainFilter { + curr, + filter, + list: self, + } + } +} + +impl<'a, T, F> Iterator for DrainFilter<'a, T, F> +where + T: Link, + F: FnMut(&mut T::Target) -> bool, +{ + type Item = T::Handle; + + fn next(&mut self) -> Option { + while let Some(curr) = self.curr { + // safety: the pointer references data contained by the list + self.curr = unsafe { T::pointers(curr).as_ref() }.get_next(); + + // safety: the value is still owned by the linked list. + if (self.filter)(unsafe { &mut *curr.as_ptr() }) { + return unsafe { self.list.remove(curr) }; + } + } + + None + } +} + +// ===== impl Pointers ===== + +impl Pointers { + /// Create a new set of empty pointers + pub fn new() -> Pointers { + Pointers { + inner: UnsafeCell::new(PointersInner { + prev: None, + next: None, + _pin: PhantomPinned, + }), + } + } + + fn get_prev(&self) -> Option> { + // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. + unsafe { + let inner = self.inner.get(); + let prev = inner as *const Option>; + ptr::read(prev) + } + } + fn get_next(&self) -> Option> { + // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. + unsafe { + let inner = self.inner.get(); + let prev = inner as *const Option>; + let next = prev.add(1); + ptr::read(next) + } + } + + fn set_prev(&mut self, value: Option>) { + // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. + unsafe { + let inner = self.inner.get(); + let prev = inner as *mut Option>; + ptr::write(prev, value); + } + } + fn set_next(&mut self, value: Option>) { + // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. + unsafe { + let inner = self.inner.get(); + let prev = inner as *mut Option>; + let next = prev.add(1); + ptr::write(next, value); + } + } +} + +impl fmt::Debug for Pointers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let prev = self.get_prev(); + let next = self.get_next(); + f.debug_struct("Pointers") + .field("prev", &prev) + .field("next", &next) + .finish() + } +} diff --git a/common/src/refcount.rs b/common/src/refcount.rs new file mode 100644 index 000000000..6db81e5ab --- /dev/null +++ b/common/src/refcount.rs @@ -0,0 +1,60 @@ +use crate::atomic::{Ordering::*, PyAtomic, Radium}; + +/// from alloc::sync +/// A soft limit on the amount of references that may be made to an `Arc`. +/// +/// Going above this limit will abort your program (although not +/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references. +const MAX_REFCOUNT: usize = (isize::MAX) as usize; + +pub struct RefCount { + strong: PyAtomic, +} + +impl Default for RefCount { + fn default() -> Self { + Self::new() + } +} + +impl RefCount { + pub fn new() -> Self { + RefCount { + strong: Radium::new(1), + } + } + + #[inline] + pub fn get(&self) -> usize { + self.strong.load(SeqCst) + } + + #[inline] + pub fn inc(&self) { + let old_size = self.strong.fetch_add(1, Relaxed); + + if old_size > MAX_REFCOUNT { + std::process::abort(); + } + } + + /// Returns true if successful + #[inline] + pub fn safe_inc(&self) -> bool { + self.strong + .fetch_update(AcqRel, Acquire, |prev| (prev != 0).then(|| prev + 1)) + .is_ok() + } + + /// Decrement the reference count. Returns true when the refcount drops to 0. + #[inline] + pub fn dec(&self) -> bool { + if self.strong.fetch_sub(1, Release) != 1 { + return false; + } + + PyAtomic::::fence(Acquire); + + true + } +} diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 81a7a6aff..b148e951a 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -667,7 +667,7 @@ impl ToTokens for GetSetNursery { #( #cfgs )* class.set_str_attr( #name, - ::rustpython_vm::PyGenericObject::new( + ::rustpython_vm::PyRef::new_ref( ::rustpython_vm::builtins::PyGetSet::new(#name.into(), class.clone()) .with_get(&Self::#getter) #setter #deleter, diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index d706da7c6..28124aeca 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -1,7 +1,7 @@ use crate::error::Diagnostic; use crate::util::{ - iter_use_idents, pyclass_ident_and_attrs, text_signature, AttributeExt, ClassItemMeta, - ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery, SimpleItemMeta, + iter_use_idents, pyclass_ident_and_attrs, text_signature, AttrItemMeta, AttributeExt, + ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES, }; use proc_macro2::TokenStream; @@ -241,7 +241,7 @@ impl ContentItem for AttributeItem { } struct ModuleItemArgs<'a> { - item: &'a Item, + item: &'a mut Item, attrs: &'a mut Vec, context: &'a mut ModuleContext, cfgs: &'a [Attribute], @@ -370,15 +370,36 @@ impl ModuleItem for AttributeItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { let cfgs = args.cfgs.to_vec(); let attr = args.attrs.remove(self.index()); - let get_py_name = |attr: &Attribute, ident: &Ident| -> Result<_> { - let item_meta = SimpleItemMeta::from_attr(ident.clone(), attr)?; - let py_name = item_meta.simple_name()?; - Ok(py_name) - }; let (py_name, tokens) = match args.item { - Item::Fn(syn::ItemFn { sig, .. }) => { + Item::Fn(syn::ItemFn { sig, block, .. }) => { let ident = &sig.ident; - let py_name = get_py_name(&attr, ident)?; + // If `once` keyword is in #[pyattr], + // wrapping it with static_cell for preventing it from using it as function + let attr_meta = AttrItemMeta::from_attr(ident.clone(), &attr)?; + if attr_meta.inner()._bool("once")? { + let stmts = &block.stmts; + let return_type = match &sig.output { + syn::ReturnType::Default => { + unreachable!("#[pyattr] attached function must have return type.") + } + syn::ReturnType::Type(_, ty) => ty, + }; + let stmt: syn::Stmt = parse_quote! { + { + rustpython_common::static_cell! { + static ERROR: #return_type; + } + ERROR + .get_or_init(|| { + #(#stmts)* + }) + .clone() + } + }; + block.stmts = vec![stmt]; + } + + let py_name = attr_meta.simple_name()?; ( py_name.clone(), quote_spanned! { ident.span() => @@ -387,7 +408,8 @@ impl ModuleItem for AttributeItem { ) } Item::Const(syn::ItemConst { ident, .. }) => { - let py_name = get_py_name(&attr, ident)?; + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr)?; + let py_name = item_meta.simple_name()?; ( py_name.clone(), quote_spanned! { ident.span() => diff --git a/derive/src/pyvalue.rs b/derive/src/pyvalue.rs index 64878f071..23636e511 100644 --- a/derive/src/pyvalue.rs +++ b/derive/src/pyvalue.rs @@ -8,8 +8,7 @@ pub(crate) fn impl_pyvalue(input: DeriveInput) -> Result { let ret = quote! { impl ::rustpython_vm::PyValue for #ty { fn class(_vm: &::rustpython_vm::VirtualMachine) -> &rustpython_vm::builtins::PyTypeRef { - use ::rustpython_vm::StaticType; - Self::static_type() + ::static_type() } } }; diff --git a/derive/src/util.rs b/derive/src/util.rs index ed790d810..f104c03c9 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -236,6 +236,19 @@ impl ItemMeta for SimpleItemMeta { } } +pub(crate) struct AttrItemMeta(pub ItemMetaInner); + +impl ItemMeta for AttrItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "once"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + pub(crate) struct ClassItemMeta(ItemMetaInner); impl ItemMeta for ClassItemMeta { diff --git a/extra_tests/not_impl_gen.py b/extra_tests/not_impl_gen.py index 21d8c9158..3162bc0ec 100644 --- a/extra_tests/not_impl_gen.py +++ b/extra_tests/not_impl_gen.py @@ -72,8 +72,6 @@ def attr_is_not_inherited(type_, attr): def extra_info(obj): - # RustPython doesn't support __text_signature__ for getting signatures of builtins - # https://github.com/RustPython/RustPython/issues/2410 if callable(obj) and not inspect._signature_is_builtin(obj): try: sig = str(inspect.signature(obj)) diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index d041a6fcb..ca29a956c 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -47,4 +47,18 @@ assert () not in d.keys() assert 10 not in d.keys() assert (1, 10) not in d.keys() assert "abc" not in d.keys() -assert ((3,4),5) not in d.keys() \ No newline at end of file +assert ((3,4),5) not in d.keys() + +d1 = {"a": 1, "b": 2} +d2 = {"c": 3, "d": 4} +assert d1.items().isdisjoint(d2.items()) +assert d1.keys().isdisjoint(d2.keys()) +d2 = {"b": 3, "d": 4} +assert d1.items().isdisjoint(d2.items()) +assert not d1.keys().isdisjoint(d2.keys()) +d2 = {"c": 2, "d": 4} +assert d1.items().isdisjoint(d2.items()) +assert d1.keys().isdisjoint(d2.keys()) +d2 = {"b": 2, "d": 4} +assert not d1.items().isdisjoint(d2.items()) +assert not d1.keys().isdisjoint(d2.keys()) \ No newline at end of file diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index def45bfaa..0b75f78cc 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -505,26 +505,3 @@ if "win" not in sys.platform: for arg in [None, 1, 1.0, TabError]: assert_raises(TypeError, os.system, arg) - -if sys.platform.startswith("win"): - winver = sys.getwindowsversion() - - # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members). - all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \ - 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 - - # We really can't test if the results are correct, so it just checks for meaningful value - assert winver.major > 0 - assert winver.minor >= 0 - assert winver.build > 0 - assert winver.platform == 2 - assert isinstance(winver.service_pack, str) - assert 0 <= winver.suite_mask <= all_masks - assert 1 <= winver.product_type <= 3 - - # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest - assert winver.major == winver.platform_version[0] - assert winver.minor == winver.platform_version[1] - assert winver.build == winver.platform_version[2] - - diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 53df9bf60..25e3428c5 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -51,11 +51,11 @@ assert ("demo", "call", None) in events assert sys.exc_info() == (None, None, None) try: - 1/0 + 1/0 except ZeroDivisionError as exc: - exc_info = sys.exc_info() - assert exc_info[0] == type(exc) == ZeroDivisionError - assert exc_info[1] == exc + exc_info = sys.exc_info() + assert exc_info[0] == type(exc) == ZeroDivisionError + assert exc_info[1] == exc # Recursion: @@ -69,3 +69,26 @@ assert sys.getrecursionlimit() == 200 with assert_raises(RecursionError): recursive_call(300) + +if sys.platform.startswith("win"): + winver = sys.getwindowsversion() + print(f'winver: {winver} {winver.platform_version}') + + # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members). + all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \ + 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 + + # We really can't test if the results are correct, so it just checks for meaningful value + assert winver.major > 0 + assert winver.minor >= 0 + assert winver.build > 0 + assert winver.platform == 2 + assert isinstance(winver.service_pack, str) + assert 0 <= winver.suite_mask <= all_masks + assert 1 <= winver.product_type <= 3 + + # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest + # these fields can mismatch in CPython + # assert winver.major == winver.platform_version[0] + # assert winver.minor == winver.platform_version[1] + # assert winver.build == winver.platform_version[2] diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 941e9e204..360ae6057 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -90,7 +90,8 @@ impl<'vm> ShellHelper<'vm> { // we need to get a variable based off of globals/builtins let globals = str_iter_method(self.globals.as_object().to_owned(), "keys").ok()?; - let builtins = str_iter_method(self.vm.builtins.clone(), "__dir__").ok()?; + let builtins = + str_iter_method(self.vm.builtins.as_object().to_owned(), "__dir__").ok()?; (first, globals, Some(builtins)) }; Some((word_start, iter1.chain(iter2.into_iter().flatten()))) diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index 1ce1c9a18..22189cfda 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -23,6 +23,7 @@ mod array { BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods, }, + sequence::{SequenceMutOp, SequenceOp}, sliceable::{PySliceableSequence, PySliceableSequenceMut, SaturatedSlice, SequenceIndex}, types::{ AsBuffer, AsMapping, Comparable, Constructor, IterNext, IterNextIterable, Iterable, @@ -387,34 +388,24 @@ mod array { } } - fn mul(&self, counter: usize) -> Self { + fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult { match self { $(ArrayContentType::$n(v) => { - let elements = v.repeat(counter); - ArrayContentType::$n(elements) + // MemoryError instead Overflow Error, hard to says it is right + // but it is how cpython doing right now + let elements = v.mul(vm, value).map_err(|_| vm.new_memory_error("".to_owned()))?; + Ok(ArrayContentType::$n(elements)) })* } } - fn clear(&mut self) { + fn imul(&mut self, value: isize, vm: &VirtualMachine) -> PyResult<()> { match self { - $(ArrayContentType::$n(v) => v.clear(),)* - } - } - - fn imul(&mut self, counter: usize) { - if counter == 0 { - self.clear(); - } else if counter != 1 { - match self { - $(ArrayContentType::$n(v) => { - let old = v.clone(); - v.reserve((counter - 1) * old.len()); - for _ in 1..counter { - v.extend(&old); - } - })* - } + $(ArrayContentType::$n(v) => { + // MemoryError instead Overflow Error, hard to says it is right + // but it is how cpython doing right now + v.imul(vm, value).map_err(|_| vm.new_memory_error("".to_owned())) + })* } } @@ -742,8 +733,7 @@ mod array { fn extend(zelf: PyRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let mut w = zelf.try_resizable(vm)?; if zelf.is(&obj) { - w.imul(2); - Ok(()) + w.imul(2, vm) } else if let Some(array) = obj.payload::() { w.iadd(&*array.read(), vm) } else { @@ -856,7 +846,7 @@ mod array { if n < 0 { return Err(vm.new_value_error("negative count".to_owned())); } - let n = vm.check_repeat_or_memory_error(itemsize, n)?; + let n = vm.check_repeat_or_overflow_error(itemsize, n)?; let nbytes = n * itemsize; let b = vm.call_method(&f, "read", (nbytes,))?; @@ -1044,7 +1034,7 @@ mod array { vm: &VirtualMachine, ) -> PyResult> { if zelf.is(&other) { - zelf.try_resizable(vm)?.imul(2); + zelf.try_resizable(vm)?.imul(2, vm)?; } else if let Some(other) = other.payload::() { zelf.try_resizable(vm)?.iadd(&*other.read(), vm)?; } else { @@ -1059,14 +1049,14 @@ mod array { #[pymethod(name = "__rmul__")] #[pymethod(magic)] fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult> { - let value = vm.check_repeat_or_memory_error(self.len(), value)?; - Ok(Self::from(self.read().mul(value)).into_ref(vm)) + self.read() + .mul(value, vm) + .map(|x| Self::from(x).into_ref(vm)) } #[pymethod(magic)] fn imul(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { - let value = vm.check_repeat_or_memory_error(zelf.len(), value)?; - zelf.try_resizable(vm)?.imul(value); + zelf.try_resizable(vm)?.imul(value, vm)?; Ok(zelf) } diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index 4e3a53635..2130b5ddc 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -11,38 +11,18 @@ mod decl { }; use itertools::Itertools; - #[pyattr(name = "Error")] + #[pyattr(name = "Error", once)] fn error_type(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static BINASCII_ERROR: PyTypeRef; - } - BINASCII_ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("binascii"), - "Error", - &vm.ctx.exceptions.value_error, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type( + "binascii", + "Error", + Some(vec![vm.ctx.exceptions.value_error.clone()]), + ) } - #[pyattr(name = "Incomplete")] + #[pyattr(name = "Incomplete", once)] fn incomplete_type(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static BINASCII_INCOMPLTE: PyTypeRef; - } - BINASCII_INCOMPLTE - .get_or_init(|| { - vm.ctx.new_class( - Some("binascii"), - "Incomplete", - &vm.ctx.exceptions.exception_type, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type("binascii", "Incomplete", None) } fn hex_nibble(n: u8) -> u8 { diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index f89238ed1..a9fc1f1c9 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -11,7 +11,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { mod _csv { use crate::common::lock::PyMutex; use crate::vm::{ - builtins::{PyStr, PyStrRef, PyType, PyTypeRef}, + builtins::{PyStr, PyStrRef, PyTypeRef}, function::{ArgIterable, ArgumentError, FromArgs, FuncArgs}, match_class, protocol::{PyIter, PyIterReturn}, @@ -30,9 +30,13 @@ mod _csv { #[pyattr] const QUOTE_NONE: i32 = QuoteStyle::None as i32; - #[pyattr(name = "Error")] + #[pyattr(name = "Error", once)] fn error(vm: &VirtualMachine) -> PyTypeRef { - PyType::new_simple_ref("_csv.Error", &vm.ctx.exceptions.exception_type).unwrap() + vm.ctx.new_exception_type( + "_csv", + "Error", + Some(vec![vm.ctx.exceptions.exception_type.clone()]), + ) } #[pyfunction] diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 4de71fdd0..9543c02b2 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -80,53 +80,29 @@ mod _socket { vm.ctx.exceptions.os_error.clone() } - #[pyattr] + #[pyattr(once)] fn timeout(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("socket"), - "timeout", - &vm.ctx.exceptions.os_error, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type( + "socket", + "timeout", + Some(vec![vm.ctx.exceptions.os_error.clone()]), + ) } - #[pyattr] + #[pyattr(once)] fn herror(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("socket"), - "herror", - &vm.ctx.exceptions.os_error, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type( + "socket", + "herror", + Some(vec![vm.ctx.exceptions.os_error.clone()]), + ) } - #[pyattr] + #[pyattr(once)] fn gaierror(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("socket"), - "gaierror", - &vm.ctx.exceptions.os_error, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type( + "socket", + "gaierror", + Some(vec![vm.ctx.exceptions.os_error.clone()]), + ) } #[pyfunction] diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 8709839d2..ad5578b1d 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -30,7 +30,7 @@ mod _ssl { }, socket::{self, PySocket}, vm::{ - builtins::{PyBaseException, PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef, PyWeak}, + builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef}, exceptions, function::{ ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, IntoPyException, @@ -39,7 +39,7 @@ mod _ssl { stdlib::os::PyPathLike, types::Constructor, utils::{Either, ToCString}, - ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, VirtualMachine, + ItemProtocol, PyObjectRef, PyObjectWeak, PyRef, PyResult, PyValue, VirtualMachine, }, }; use crossbeam_utils::atomic::AtomicCell; @@ -174,90 +174,58 @@ mod _ssl { parse_version_info(openssl_api_version) } - #[pyattr(name = "SSLError")] + /// An error occurred in the SSL implementation. + #[pyattr(name = "SSLError", once)] fn ssl_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - PyType::new_simple_ref("ssl.SSLError", &vm.ctx.exceptions.os_error).unwrap() - }) - .clone() + vm.ctx.new_exception_type( + "ssl", + "SSLError", + Some(vec![vm.ctx.exceptions.os_error.clone()]), + ) } - #[pyattr(name = "SSLCertVerificationError")] + /// A certificate could not be verified. + #[pyattr(name = "SSLCertVerificationError", once)] fn ssl_cert_verification_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - let ssl_error = ssl_error(vm); - PyType::new_ref( - "ssl.SSLCertVerificationError", - vec![ssl_error, vm.ctx.exceptions.value_error.clone()], - Default::default(), - PyBaseException::make_slots(), - vm.ctx.types.type_type.clone(), - ) - .unwrap() - }) - .clone() + vm.ctx.new_exception_type( + "ssl", + "SSLCertVerificationError", + Some(vec![ssl_error(vm), vm.ctx.exceptions.value_error.clone()]), + ) } - #[pyattr(name = "SSLZeroReturnError")] + /// SSL/TLS session closed cleanly. + #[pyattr(name = "SSLZeroReturnError", once)] fn ssl_zero_return_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - PyType::new_simple_ref("ssl.SSLZeroReturnError", &ssl_error(vm)).unwrap() - }) - .clone() + vm.ctx + .new_exception_type("ssl", "SSLZeroReturnError", Some(vec![ssl_error(vm)])) } - #[pyattr(name = "SSLWantReadError")] + /// Non-blocking SSL socket needs to read more data before the requested operation can be completed. + #[pyattr(name = "SSLWantReadError", once)] fn ssl_want_read_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| PyType::new_simple_ref("ssl.SSLWantReadError", &ssl_error(vm)).unwrap()) - .clone() + vm.ctx + .new_exception_type("ssl", "SSLWantReadError", Some(vec![ssl_error(vm)])) } - #[pyattr(name = "SSLWantWriteError")] + /// Non-blocking SSL socket needs to write more data before the requested operation can be completed. + #[pyattr(name = "SSLWantWriteError", once)] fn ssl_want_write_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| { - PyType::new_simple_ref("ssl.SSLWantWriteError", &ssl_error(vm)).unwrap() - }) - .clone() + vm.ctx + .new_exception_type("ssl", "SSLWantWriteError", Some(vec![ssl_error(vm)])) } - #[pyattr(name = "SSLSyscallError")] + /// System error when attempting SSL operation. + #[pyattr(name = "SSLSyscallError", once)] fn ssl_syscall_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| PyType::new_simple_ref("ssl.SSLSyscallError", &ssl_error(vm)).unwrap()) - .clone() + vm.ctx + .new_exception_type("ssl", "SSLSyscallError", Some(vec![ssl_error(vm)])) } - #[pyattr(name = "SSLEOFError")] + /// SSL/TLS connection terminated abruptly. + #[pyattr(name = "SSLEOFError", once)] fn ssl_eof_error(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static ERROR: PyTypeRef; - } - ERROR - .get_or_init(|| PyType::new_simple_ref("ssl.SSLEOFError", &ssl_error(vm)).unwrap()) - .clone() + PyType::new_simple_ref("ssl.SSLEOFError", &ssl_error(vm)).unwrap() } type OpensslVersionInfo = (u8, u8, u8, u8, u8); @@ -805,7 +773,7 @@ mod _ssl { stream: PyRwLock::new(stream), socket_type, server_hostname: args.server_hostname, - owner: PyRwLock::new(args.owner.as_ref().map(|o| PyWeak::downgrade(o))), + owner: PyRwLock::new(args.owner.map(|o| o.downgrade(None, vm)).transpose()?), }) } } @@ -922,7 +890,7 @@ mod _ssl { stream: PyRwLock>, socket_type: SslServerOrClient, server_hostname: Option, - owner: PyRwLock>, + owner: PyRwLock>, } impl fmt::Debug for PySslSocket { @@ -935,11 +903,14 @@ mod _ssl { impl PySslSocket { #[pyproperty] fn owner(&self) -> Option { - self.owner.read().as_ref().and_then(PyWeak::upgrade) + self.owner.read().as_ref().and_then(|weak| weak.upgrade()) } #[pyproperty(setter)] - fn set_owner(&self, owner: PyObjectRef) { - *self.owner.write() = Some(PyWeak::downgrade(&owner)) + fn set_owner(&self, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut lock = self.owner.write(); + lock.take(); + *lock = Some(owner.downgrade(None, vm)?); + Ok(()) } #[pyproperty] fn server_side(&self) -> bool { diff --git a/stdlib/src/termios.rs b/stdlib/src/termios.rs index 97ccdac29..507a655fa 100644 --- a/stdlib/src/termios.rs +++ b/stdlib/src/termios.rs @@ -100,20 +100,12 @@ mod termios { ) } - #[pyattr(name = "error")] + #[pyattr(name = "error", once)] fn error_type(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static TERMIOS_ERROR: PyTypeRef; - } - TERMIOS_ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("termios"), - "error", - &vm.ctx.exceptions.os_error, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type( + "termios", + "error", + Some(vec![vm.ctx.exceptions.os_error.clone()]), + ) } } diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 492fc60e3..d2f44de20 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -4,7 +4,7 @@ pub(crate) use zlib::make_module; mod zlib { use crate::common::lock::PyMutex; use crate::vm::{ - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyType, PyTypeRef}, + builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyTypeRef}, function::{ArgBytesLike, OptionalArg}, IntoPyRef, PyResult, PyValue, VirtualMachine, }; @@ -53,9 +53,13 @@ mod zlib { #[pyattr] const DEF_MEM_LEVEL: u8 = 8; - #[pyattr] + #[pyattr(once)] fn error(vm: &VirtualMachine) -> PyTypeRef { - PyType::new_simple_ref("zlib.error", &vm.ctx.exceptions.exception_type).unwrap() + vm.ctx.new_exception_type( + "zlib", + "error", + Some(vec![vm.ctx.exceptions.exception_type.clone()]), + ) } /// Compute an Adler-32 checksum of data. diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 914baf292..3989b5648 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -68,6 +68,8 @@ memchr = "2" adler32 = "1.0.3" flate2 = "1.0.20" once_cell = "1" +memoffset = "0.6" +optional = "0.5" # RustPython crates implementing functionality based on CPython sre-engine = "0.1.2" diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index 4dc7efe71..c7eb74165 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -35,17 +35,6 @@ use crate::{ use bstr::ByteSlice; use std::mem::size_of; -/// "bytearray(iterable_of_ints) -> bytearray\n\ -/// bytearray(string, encoding[, errors]) -> bytearray\n\ -/// bytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\n\ -/// bytearray(int) -> bytes array of size given by the parameter initialized with null bytes\n\ -/// bytearray() -> empty bytes array\n\n\ -/// Construct a mutable bytearray object from:\n \ -/// - an iterable yielding integers in range(256)\n \ -/// - a text string encoded using the specified encoding\n \ -/// - a bytes or a buffer object\n \ -/// - any object implementing the buffer API.\n \ -/// - an integer"; #[pyclass(module = false, name = "bytearray")] #[derive(Debug, Default)] pub struct PyByteArray { @@ -296,7 +285,7 @@ impl PyByteArray { } } - fn irepeat(zelf: &crate::PyObjectView, n: usize, vm: &VirtualMachine) -> PyResult<()> { + fn irepeat(zelf: &crate::PyObjectView, n: isize, vm: &VirtualMachine) -> PyResult<()> { if n == 1 { return Ok(()); } @@ -311,19 +300,8 @@ impl PyByteArray { }; } }; - let elements = &mut w.elements; - if n == 0 { - elements.clear(); - } else if n != 1 { - let old = elements.clone(); - - elements.reserve((n - 1) * old.len()); - for _ in 1..n { - elements.extend(&old); - } - } - Ok(()) + w.imul(n, vm) } #[pymethod] @@ -646,14 +624,13 @@ impl PyByteArray { #[pymethod(name = "__rmul__")] #[pymethod(magic)] fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult { - vm.check_repeat_or_memory_error(self.len(), value) - .map(|value| self.inner().repeat(value).into()) + self.inner().mul(value, vm).map(|x| x.into()) } #[pymethod(magic)] fn imul(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { - vm.check_repeat_or_memory_error(zelf.len(), value) - .and_then(|value| Self::irepeat(&zelf, value, vm).map(|_| zelf)) + Self::irepeat(&zelf, value, vm)?; + Ok(zelf) } #[pymethod(name = "__mod__")] diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 0b0ee5228..750a91425 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -23,15 +23,6 @@ use bstr::ByteSlice; use std::mem::size_of; use std::ops::Deref; -/// "bytes(iterable_of_ints) -> bytes\n\ -/// bytes(string, encoding[, errors]) -> bytes\n\ -/// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ -/// bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ -/// bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ -/// - an iterable yielding integers in range(256)\n \ -/// - a text string encoded using the specified encoding\n \ -/// - any object implementing the buffer API.\n \ -/// - an integer"; #[pyclass(module = false, name = "bytes")] #[derive(Clone, Debug)] pub struct PyBytes { @@ -470,14 +461,9 @@ impl PyBytes { // This only works for `bytes` itself, not its subclasses. return Ok(zelf); } - // todo: map err to overflow. - vm.check_repeat_or_memory_error(zelf.inner.len(), value) - .map(|value| { - let bytes: PyBytes = zelf.inner.repeat(value).into(); - bytes.into_ref(vm) - }) - // see issue 45044 on b.p.o. - .map_err(|_| vm.new_overflow_error("repeated bytes are too long".to_owned())) + zelf.inner + .mul(value, vm) + .map(|x| Self::from(x).into_ref(vm)) } #[pymethod(name = "__mod__")] diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index 0ee51973d..ec1e7e99f 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -63,6 +63,12 @@ impl PyDict { #[allow(clippy::len_without_is_empty)] #[pyimpl(with(AsMapping, Hashable, Comparable, Iterable), flags(BASETYPE))] impl PyDict { + /// escape hatch to access the underlying data structure directly. prefer adding a method on + /// PyDict instead of using this + pub(crate) fn _as_dict_inner(&self) -> &DictContentType { + &self.entries + } + #[pyslot] fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { PyDict::default().into_pyresult_with_type(vm, cls) @@ -987,6 +993,14 @@ trait ViewSetOps: DictView { } }) } + + #[pymethod] + fn isdisjoint(zelf: PyRef, other: ArgIterable, vm: &VirtualMachine) -> PyResult { + // TODO: to_set is an expensive operation. After merging #3316 rewrite implementation using PySequence_Contains. + let zelf = Self::to_set(zelf, vm)?; + let result = zelf.isdisjoint(other, vm)?; + Ok(result) + } } impl ViewSetOps for PyDictKeys {} diff --git a/vm/src/builtins/enumerate.rs b/vm/src/builtins/enumerate.rs index 34809fa45..699d2d3cb 100644 --- a/vm/src/builtins/enumerate.rs +++ b/vm/src/builtins/enumerate.rs @@ -4,7 +4,8 @@ use crate::{ function::{IntoPyObject, OptionalArg}, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, IterNextIterable}, - ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult, PyValue, VirtualMachine, + ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + VirtualMachine, }; use num_bigint::BigInt; use num_traits::Zero; @@ -52,6 +53,13 @@ impl PyEnumerate { fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { PyGenericAlias::new(cls, args, vm) } + #[pymethod(magic)] + fn reduce(zelf: PyRef) -> (PyTypeRef, (PyIter, BigInt)) { + ( + zelf.clone_class(), + (zelf.iterator.clone(), zelf.counter.read().clone()), + ) + } } impl IterNextIterable for PyEnumerate {} diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 766bf5b36..e579b7f69 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -5,9 +5,9 @@ use super::{ tuple::PyTupleTyped, PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStrRef, PyTupleRef, PyTypeRef, }; +use crate::common::lock::PyMutex; use crate::{ bytecode, - common::lock::PyMutex, frame::Frame, function::{FuncArgs, OptionalArg}, protocol::PyMapping, @@ -136,10 +136,10 @@ impl PyFunction { ); } *slot = Some(value); - } else if argpos(0..code.posonlyarg_count, &name).is_some() { - posonly_passed_as_kwarg.push(name); } else if let Some(kwargs) = kwargs.as_ref() { kwargs.set_item(name, value, vm)?; + } else if argpos(0..code.posonlyarg_count, &name).is_some() { + posonly_passed_as_kwarg.push(name); } else { return Err( vm.new_type_error(format!("got an unexpected keyword argument '{}'", name)) @@ -299,7 +299,7 @@ impl PyFunction { let frame = Frame::new( code.clone(), Scope::new(Some(locals), self.globals.clone()), - vm.builtins.dict().unwrap(), + vm.builtins.dict(), self.closure.as_ref().map_or(&[], |c| c.as_slice()), vm, ) diff --git a/vm/src/builtins/list.rs b/vm/src/builtins/list.rs index 7a9abac7c..835d84acb 100644 --- a/vm/src/builtins/list.rs +++ b/vm/src/builtins/list.rs @@ -2,24 +2,24 @@ use super::{PositionIterInternal, PyGenericAlias, PySlice, PyTupleRef, PyTypeRef use crate::common::lock::{ PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, }; +use crate::sequence::MutObjectSequenceOp; use crate::{ function::{ArgIterable, FuncArgs, IntoPyObject, OptionalArg}, protocol::{PyIterReturn, PyMappingMethods}, - sequence::{self, SimpleSeq}, + sequence::{ObjectSequenceOp, SequenceMutOp, SequenceOp}, sliceable::{saturate_index, PySliceableSequence, PySliceableSequenceMut, SequenceIndex}, - stdlib::sys, types::{ - richcompare_wrapper, AsMapping, Comparable, Constructor, Hashable, IterNext, - IterNextIterable, Iterable, PyComparisonOp, RichCompareFunc, Unconstructible, Unhashable, + AsMapping, Comparable, Constructor, Hashable, IterNext, IterNextIterable, Iterable, + PyComparisonOp, Unconstructible, Unhashable, }, utils::Either, vm::{ReprGuard, VirtualMachine}, - IdProtocol, PyClassDef, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef, - PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, + PyClassDef, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef, PyRef, PyResult, + PyValue, TryFromObject, TypeProtocol, }; use std::fmt; use std::mem::size_of; -use std::ops::{DerefMut, Range}; +use std::ops::DerefMut; /// Built-in mutable sequence. /// @@ -240,189 +240,26 @@ impl PyList { #[pymethod(magic)] #[pymethod(name = "__rmul__")] - fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult> { - let new_elements = sequence::seq_mul(vm, &self.borrow_vec(), value)? - .cloned() - .collect(); - Ok(Self::new_ref(new_elements, &vm.ctx)) + fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult> { + let elements = &*self.borrow_vec(); + let v = elements.mul(vm, n)?; + Ok(Self::new_ref(v, &vm.ctx)) } #[pymethod(magic)] - fn imul(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { - let mut elements = zelf.borrow_vec_mut(); - let mut new_elements: Vec = - sequence::seq_mul(vm, &*elements, value)?.cloned().collect(); - std::mem::swap(elements.deref_mut(), &mut new_elements); - Ok(zelf.clone()) - } - - fn _iter_equal( - &self, - needle: &PyObject, - range: Range, - mut f: F, - vm: &VirtualMachine, - ) -> PyResult { - let needle_cls = needle.class(); - let needle_cmp = needle_cls - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); - - let mut borrower = None; - let mut i = range.start; - - let index = loop { - if i >= range.end { - break usize::MAX; - } - let guard = if let Some(x) = borrower.take() { - x - } else { - self.borrow_vec() - }; - - let elem = if let Some(x) = guard.get(i) { - x - } else { - break usize::MAX; - }; - - if elem.is(needle) { - f(); - if SHORT { - break i; - } - borrower = Some(guard); - } else { - let elem_cls = elem.class(); - let reverse_first = !elem_cls.is(&needle_cls) && elem_cls.issubclass(&needle_cls); - - let eq = if reverse_first { - let elem_cmp = elem_cls - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); - drop(elem_cls); - - fn cmp( - elem: &PyObject, - needle: &PyObject, - elem_cmp: RichCompareFunc, - needle_cmp: RichCompareFunc, - vm: &VirtualMachine, - ) -> PyResult { - match elem_cmp(elem, needle, PyComparisonOp::Eq, vm)? { - Either::B(PyComparisonValue::Implemented(value)) => Ok(value), - Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { - obj.try_to_bool(vm) - } - _ => match needle_cmp(needle, elem, PyComparisonOp::Eq, vm)? { - Either::B(PyComparisonValue::Implemented(value)) => Ok(value), - Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { - obj.try_to_bool(vm) - } - _ => Ok(false), - }, - } - } - - if elem_cmp as usize == richcompare_wrapper as usize { - let elem = elem.clone(); - drop(guard); - cmp(&elem, needle, elem_cmp, needle_cmp, vm)? - } else { - let eq = cmp(elem, needle, elem_cmp, needle_cmp, vm)?; - borrower = Some(guard); - eq - } - } else { - match needle_cmp(needle, elem, PyComparisonOp::Eq, vm)? { - Either::B(PyComparisonValue::Implemented(value)) => { - drop(elem_cls); - borrower = Some(guard); - value - } - Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { - drop(elem_cls); - borrower = Some(guard); - obj.try_to_bool(vm)? - } - _ => { - let elem_cmp = elem_cls - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); - drop(elem_cls); - - fn cmp( - elem: &PyObject, - needle: &PyObject, - elem_cmp: RichCompareFunc, - vm: &VirtualMachine, - ) -> PyResult { - match elem_cmp(elem, needle, PyComparisonOp::Eq, vm)? { - Either::B(PyComparisonValue::Implemented(value)) => Ok(value), - Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { - obj.try_to_bool(vm) - } - _ => Ok(false), - } - } - - if elem_cmp as usize == richcompare_wrapper as usize { - let elem = elem.clone(); - drop(guard); - cmp(&elem, needle, elem_cmp, vm)? - } else { - let eq = cmp(elem, needle, elem_cmp, vm)?; - borrower = Some(guard); - eq - } - } - } - }; - - if eq { - f(); - if SHORT { - break i; - } - } - } - i += 1; - }; - - // TODO: Optioned - Ok(index) - } - - fn foreach_equal( - &self, - needle: &PyObject, - f: F, - vm: &VirtualMachine, - ) -> PyResult<()> { - self._iter_equal::<_, false>(needle, 0..usize::MAX, f, vm) - .map(|_| ()) - } - - fn find_equal( - &self, - needle: &PyObject, - range: Range, - vm: &VirtualMachine, - ) -> PyResult { - self._iter_equal::<_, true>(needle, range, || {}, vm) + fn imul(zelf: PyRef, n: isize, vm: &VirtualMachine) -> PyResult> { + zelf.borrow_vec_mut().imul(vm, n)?; + Ok(zelf) } #[pymethod] fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut count = 0; - self.foreach_equal(&needle, || count += 1, vm)?; - Ok(count) + self.mut_count(vm, &needle) } #[pymethod(magic)] pub(crate) fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Ok(self.find_equal(&needle, 0..usize::MAX, vm)? != usize::MAX) + self.mut_contains(vm, &needle) } #[pymethod] @@ -437,12 +274,12 @@ impl PyList { let start = start.map(|i| saturate_index(i, len)).unwrap_or(0); let stop = stop .map(|i| saturate_index(i, len)) - .unwrap_or(sys::MAXSIZE as usize); - let index = self.find_equal(&needle, start..stop, vm)?; - if index == usize::MAX { - Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) - } else { + .unwrap_or(isize::MAX as usize); + let index = self.mut_index_range(vm, &needle, start..stop)?; + if let Some(index) = index.into() { Ok(index) + } else { + Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) } } @@ -464,9 +301,9 @@ impl PyList { #[pymethod] fn remove(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let index = self.find_equal(&needle, 0..usize::MAX, vm)?; + let index = self.mut_index(vm, &needle)?; - if index != usize::MAX { + if let Some(index) = index.into() { // defer delete out of borrow Ok(self.borrow_vec_mut().remove(index)) } else { @@ -540,6 +377,18 @@ impl PyList { } } +impl<'a> MutObjectSequenceOp<'a> for PyList { + type Guard = PyMappedRwLockReadGuard<'a, [PyObjectRef]>; + + fn do_get(index: usize, guard: &Self::Guard) -> Option<&PyObjectRef> { + guard.get(index) + } + + fn do_lock(&'a self) -> Self::Guard { + self.borrow_vec() + } +} + impl AsMapping for PyList { fn as_mapping(_zelf: &crate::PyObjectView, _vm: &VirtualMachine) -> PyMappingMethods { PyMappingMethods { @@ -593,9 +442,9 @@ impl Comparable for PyList { return Ok(res.into()); } let other = class_or_notimplemented!(Self, other); - let a = zelf.borrow_vec(); - let b = other.borrow_vec(); - sequence::cmp(vm, a.boxed_iter(), b.boxed_iter(), op).map(PyComparisonValue::Implemented) + let a = &*zelf.borrow_vec(); + let b = &*other.borrow_vec(); + a.cmp(vm, b, op).map(PyComparisonValue::Implemented) } } diff --git a/vm/src/builtins/module.rs b/vm/src/builtins/module.rs index a2300b271..79a9399e3 100644 --- a/vm/src/builtins/module.rs +++ b/vm/src/builtins/module.rs @@ -1,13 +1,15 @@ +use super::pystr::IntoPyStrRef; use super::{PyDictRef, PyStr, PyStrRef, PyTypeRef}; use crate::{ function::{FuncArgs, IntoPyObject}, types::GetAttr, - ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, VirtualMachine, + ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyObjectView, PyRef, PyResult, PyValue, + VirtualMachine, }; #[pyclass(module = false, name = "module")] #[derive(Debug)] -pub struct PyModule; +pub struct PyModule {} impl PyValue for PyModule { fn class(vm: &VirtualMachine) -> &PyTypeRef { @@ -15,29 +17,6 @@ impl PyValue for PyModule { } } -pub fn init_module_dict( - vm: &VirtualMachine, - module_dict: &PyDictRef, - name: PyObjectRef, - doc: PyObjectRef, -) { - module_dict - .set_item("__name__", name, vm) - .expect("Failed to set __name__ on module"); - module_dict - .set_item("__doc__", doc, vm) - .expect("Failed to set __doc__ on module"); - module_dict - .set_item("__package__", vm.ctx.none(), vm) - .expect("Failed to set __package__ on module"); - module_dict - .set_item("__loader__", vm.ctx.none(), vm) - .expect("Failed to set __loader__ on module"); - module_dict - .set_item("__spec__", vm.ctx.none(), vm) - .expect("Failed to set __spec__ on module"); -} - #[derive(FromArgs)] struct ModuleInitArgs { name: PyStrRef, @@ -47,6 +26,15 @@ struct ModuleInitArgs { #[pyimpl(with(GetAttr), flags(BASETYPE, HAS_DICT))] impl PyModule { + // pub(crate) fn new(d: PyDictRef) -> Self { + // PyModule { dict: d.into() } + // } + + // #[inline] + // pub fn dict(&self) -> PyDictRef { + // self.dict.get() + // } + #[pyslot] fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { PyModule {}.into_pyresult_with_type(vm, cls) @@ -58,22 +46,30 @@ impl PyModule { .slots .flags .has_feature(crate::types::PyTypeFlags::HAS_DICT)); - init_module_dict( - vm, - &zelf.as_object().dict().unwrap(), - args.name.into(), - args.doc.into_pyobject(vm), - ); + zelf.init_module_dict(args.name.into(), args.doc.into_pyobject(vm), vm); } - fn name(zelf: PyRef, vm: &VirtualMachine) -> Option { - vm.generic_getattribute_opt( - zelf.as_object().to_owned(), - PyStr::from("__name__").into_ref(vm), - None, - ) - .unwrap_or(None) - .and_then(|obj| obj.payload::().map(|s| s.as_str().to_owned())) + fn getattr_inner(zelf: &PyObjectView, name: PyStrRef, vm: &VirtualMachine) -> PyResult { + if let Some(attr) = + vm.generic_getattribute_opt(zelf.as_object().to_owned(), name.clone(), None)? + { + return Ok(attr); + } + if let Ok(getattr) = zelf.dict().get_item("__getattr__", vm) { + return vm.invoke(&getattr, (name,)); + } + let module_name = if let Some(name) = Self::name(zelf.to_owned(), vm) { + format!(" '{}'", name) + } else { + "".to_owned() + }; + Err(vm.new_attribute_error(format!("module{} has no attribute '{}'", module_name, name))) + } + + fn name(zelf: PyRef, vm: &VirtualMachine) -> Option { + vm.generic_getattribute_opt(zelf.into(), PyStr::from("__name__").into_ref(vm), None) + .unwrap_or(None) + .and_then(|obj| obj.downcast::().ok()) } #[pymethod(magic)] @@ -94,26 +90,47 @@ impl PyModule { } } +impl PyObjectView { + // TODO: to be replaced by the commented-out dict method above once dictoffsets land + pub fn dict(&self) -> PyDictRef { + self.as_object().dict().unwrap() + } + // TODO: should be on PyModule, not PyObjectView + pub(crate) fn init_module_dict( + &self, + name: PyObjectRef, + doc: PyObjectRef, + vm: &VirtualMachine, + ) { + let dict = self.dict(); + dict.set_item("__name__", name, vm) + .expect("Failed to set __name__ on module"); + dict.set_item("__doc__", doc, vm) + .expect("Failed to set __doc__ on module"); + dict.set_item("__package__", vm.ctx.none(), vm) + .expect("Failed to set __package__ on module"); + dict.set_item("__loader__", vm.ctx.none(), vm) + .expect("Failed to set __loader__ on module"); + dict.set_item("__spec__", vm.ctx.none(), vm) + .expect("Failed to set __spec__ on module"); + } + + pub fn get_attr(&self, attr_name: impl IntoPyStrRef, vm: &VirtualMachine) -> PyResult { + PyModule::getattr_inner(self, attr_name.into_pystr_ref(vm), vm) + } + pub fn set_attr( + &self, + attr_name: impl IntoPyStrRef, + attr_value: impl Into, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.as_object().set_attr(attr_name, attr_value, vm) + } +} + impl GetAttr for PyModule { fn getattro(zelf: PyRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - if let Some(attr) = - vm.generic_getattribute_opt(zelf.as_object().to_owned(), name.clone(), None)? - { - return Ok(attr); - } - if let Some(getattr) = zelf - .as_object() - .dict() - .and_then(|d| d.get_item("__getattr__", vm).ok()) - { - return vm.invoke(&getattr, (name,)); - } - let module_name = if let Some(name) = Self::name(zelf, vm) { - format!(" '{}'", name) - } else { - "".to_owned() - }; - Err(vm.new_attribute_error(format!("module{} has no attribute '{}'", module_name, name))) + Self::getattr_inner(&zelf, name, vm) } } diff --git a/vm/src/builtins/namespace.rs b/vm/src/builtins/namespace.rs index 77a5e4c07..8e7443e5d 100644 --- a/vm/src/builtins/namespace.rs +++ b/vm/src/builtins/namespace.rs @@ -13,7 +13,7 @@ use crate::{ /// SimpleNamespace(**kwargs) #[pyclass(module = false, name = "SimpleNamespace")] #[derive(Debug)] -pub struct PyNamespace; +pub struct PyNamespace {} impl PyValue for PyNamespace { fn class(vm: &VirtualMachine) -> &PyTypeRef { @@ -31,7 +31,11 @@ impl Constructor for PyNamespace { impl PyNamespace { pub fn new_ref(ctx: &PyContext) -> PyRef { - PyRef::new_ref(Self, ctx.types.namespace_type.clone(), Some(ctx.new_dict())) + PyRef::new_ref( + Self {}, + ctx.types.namespace_type.clone(), + Some(ctx.new_dict()), + ) } } @@ -58,18 +62,14 @@ impl PyNamespace { }; let repr = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let parts = if let Some(dict) = zelf.as_object().dict() { - let mut parts = Vec::with_capacity(dict.len()); - for (key, value) in dict { - let k = &key.repr(vm)?; - let key_str = k.as_str(); - let value_repr = value.repr(vm)?; - parts.push(format!("{}={}", &key_str[1..key_str.len() - 1], value_repr)); - } - parts - } else { - vec![] - }; + let dict = zelf.as_object().dict().unwrap(); + let mut parts = Vec::with_capacity(dict.len()); + for (key, value) in dict { + let k = &key.repr(vm)?; + let key_str = k.as_str(); + let value_repr = value.repr(vm)?; + parts.push(format!("{}={}", &key_str[1..key_str.len() - 1], value_repr)); + } format!("{}({})", name, parts.join(", ")) } else { format!("{}(...)", name) @@ -86,10 +86,11 @@ impl Comparable for PyNamespace { vm: &VirtualMachine, ) -> PyResult { let other = class_or_notimplemented!(Self, other); - match (zelf.as_object().dict(), other.as_object().dict()) { - (Some(d1), Some(d2)) => PyDict::cmp(&d1, d2.as_object(), op, vm), - _ => Ok(PyComparisonValue::NotImplemented), - } + let (d1, d2) = ( + zelf.as_object().dict().unwrap(), + other.as_object().dict().unwrap(), + ); + PyDict::cmp(&d1, d2.as_object(), op, vm) } } diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index 871a0b935..14f801986 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -2,8 +2,8 @@ use super::{PyDict, PyDictRef, PyList, PyStr, PyStrRef, PyType, PyTypeRef}; use crate::common::hash::PyHash; use crate::{ function::FuncArgs, types::PyComparisonOp, utils::Either, IdProtocol, ItemProtocol, - PyArithmeticValue, PyAttributes, PyClassImpl, PyComparisonValue, PyContext, PyGenericObject, - PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, VirtualMachine, + PyArithmeticValue, PyAttributes, PyClassImpl, PyComparisonValue, PyContext, PyObject, + PyObjectRef, PyResult, PyValue, TypeProtocol, VirtualMachine, }; /// object() @@ -46,7 +46,7 @@ impl PyBaseObject { } } - Ok(PyGenericObject::new(PyBaseObject, cls, dict)) + Ok(crate::PyRef::new_ref(PyBaseObject, cls, dict).into()) } #[pyslot] @@ -160,13 +160,13 @@ impl PyBaseObject { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - setattr(&obj, name, Some(value), vm) + generic_setattr(&obj, name, Some(value), vm) } /// Implement delattr(self, name). #[pymethod] fn __delattr__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - setattr(&obj, name, None, vm) + generic_setattr(&obj, name, None, vm) } #[pyslot] @@ -176,7 +176,7 @@ impl PyBaseObject { value: Option, vm: &VirtualMachine, ) -> PyResult<()> { - setattr(&*obj, attr_name, value, vm) + generic_setattr(&*obj, attr_name, value, vm) } /// Return str(self). @@ -328,8 +328,12 @@ pub fn object_set_dict(obj: PyObjectRef, dict: PyDictRef, vm: &VirtualMachine) - .map_err(|_| vm.new_attribute_error("This object has no __dict__".to_owned())) } +pub fn generic_getattr(obj: PyObjectRef, attr_name: PyStrRef, vm: &VirtualMachine) -> PyResult { + vm.generic_getattribute(obj, attr_name) +} + #[cfg_attr(feature = "flame-it", flame)] -pub(crate) fn setattr( +pub fn generic_setattr( obj: &PyObject, attr_name: PyStrRef, value: Option, diff --git a/vm/src/builtins/pystr.rs b/vm/src/builtins/pystr.rs index 9f0d1d436..080f4f9bd 100644 --- a/vm/src/builtins/pystr.rs +++ b/vm/src/builtins/pystr.rs @@ -8,6 +8,7 @@ use crate::{ format::{FormatSpec, FormatString, FromTemplate}, function::{ArgIterable, FuncArgs, IntoPyException, IntoPyObject, OptionalArg, OptionalOption}, protocol::{PyIterReturn, PyMappingMethods}, + sequence::SequenceOp, sliceable::PySliceableSequence, types::{ AsMapping, Comparable, Constructor, Hashable, IterNext, IterNextIterable, Iterable, @@ -89,13 +90,22 @@ impl PyStrKindData { /// encoding defaults to sys.getdefaultencoding(). /// errors defaults to 'strict'." #[pyclass(module = false, name = "str")] -#[derive(Debug)] pub struct PyStr { bytes: Box<[u8]>, kind: PyStrKindData, hash: PyAtomic, } +impl fmt::Debug for PyStr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PyStr") + .field("value", &self.as_str()) + .field("kind", &self.kind) + .field("hash", &self.hash) + .finish() + } +} + impl AsRef for PyStr { fn as_ref(&self) -> &str { self.as_str() @@ -484,11 +494,10 @@ impl PyStr { // This only works for `str` itself, not its subclasses. return Ok(zelf); } - // todo: map err to overflow. - vm.check_repeat_or_memory_error(zelf.len(), value) - .map(|value| Self::from(zelf.as_str().repeat(value)).into_ref(vm)) - // see issue 45044 on b.p.o. - .map_err(|_| vm.new_overflow_error("repeated bytes are too long".to_owned())) + zelf.as_str() + .as_bytes() + .mul(vm, value) + .map(|x| Self::from(unsafe { String::from_utf8_unchecked(x) }).into_ref(vm)) } #[pymethod(magic)] @@ -709,7 +718,8 @@ impl PyStr { #[pymethod] fn isdecimal(&self) -> bool { - !self.bytes.is_empty() && self.char_all(|c| c.is_ascii_digit()) + !self.bytes.is_empty() + && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) } #[pymethod(name = "__mod__")] diff --git a/vm/src/builtins/pytype.rs b/vm/src/builtins/pytype.rs index 867a6c587..3ed6f77bc 100644 --- a/vm/src/builtins/pytype.rs +++ b/vm/src/builtins/pytype.rs @@ -1,6 +1,6 @@ use super::{ mappingproxy::PyMappingProxy, object, PyClassMethod, PyDictRef, PyList, PyStaticMethod, PyStr, - PyStrRef, PyTuple, PyTupleRef, PyWeak, + PyStrRef, PyTuple, PyTupleRef, }; use crate::common::{ ascii, @@ -10,8 +10,8 @@ use crate::common::{ use crate::{ function::{FuncArgs, KwArgs, OptionalArg}, types::{Callable, GetAttr, PyTypeFlags, PyTypeSlots, SetAttr}, - IdProtocol, PyAttributes, PyClassImpl, PyContext, PyLease, PyObjectRef, PyRef, PyResult, - PyValue, StaticType, TypeProtocol, VirtualMachine, + IdProtocol, PyAttributes, PyClassImpl, PyContext, PyLease, PyObjectRef, PyObjectWeak, PyRef, + PyResult, PyValue, StaticType, TypeProtocol, VirtualMachine, }; use itertools::Itertools; use std::collections::HashSet; @@ -26,7 +26,7 @@ pub struct PyType { pub base: Option, pub bases: Vec, pub mro: Vec, - pub subclasses: PyRwLock>, + pub subclasses: PyRwLock>, pub attributes: PyRwLock, pub slots: PyTypeSlots, } @@ -96,7 +96,7 @@ impl PyType { slots.flags |= PyTypeFlags::HAS_DICT } - *slots.name.write() = Some(String::from(name)); + *slots.name.get_mut() = Some(String::from(name)); let new_type = PyRef::new_ref( PyType { @@ -116,10 +116,14 @@ impl PyType { new_type.update_slot(attr_name, true); } } + let weakref_type = super::PyWeak::static_type(); for base in &new_type.bases { - base.subclasses - .write() - .push(PyWeak::downgrade(new_type.as_object())); + base.subclasses.write().push( + new_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); } Ok(new_type) diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index cc238b8c3..e07145412 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -195,7 +195,7 @@ impl PySetInner { self.compare(&other_set, PyComparisonOp::Le, vm) } - fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { + pub(super) fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { for item in other.iter(vm)? { if self.contains(&*item?, vm)? { return Ok(false); diff --git a/vm/src/builtins/slice.rs b/vm/src/builtins/slice.rs index 99bcec732..68e5d6886 100644 --- a/vm/src/builtins/slice.rs +++ b/vm/src/builtins/slice.rs @@ -194,6 +194,20 @@ impl PySlice { let (start, stop, step) = self.inner_indices(length, vm)?; Ok(vm.new_tuple((start, stop, step))) } + + #[allow(clippy::type_complexity)] + #[pymethod(magic)] + fn reduce( + zelf: PyRef, + ) -> PyResult<( + PyTypeRef, + (Option, PyObjectRef, Option), + )> { + Ok(( + zelf.clone_class(), + (zelf.start.clone(), zelf.stop.clone(), zelf.step.clone()), + )) + } } impl Comparable for PySlice { diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index 2dc699465..b69b520a2 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -3,7 +3,7 @@ use crate::common::hash::PyHash; use crate::{ function::{IntoPyObject, OptionalArg}, protocol::{PyIterReturn, PyMappingMethods}, - sequence::{self, SimpleSeq}, + sequence::{ObjectSequenceOp, SequenceOp}, sliceable::PySliceableSequence, stdlib::sys, types::{ @@ -228,10 +228,8 @@ impl PyTuple { // This only works for `tuple` itself, not its subclasses. zelf } else { - let elements = sequence::seq_mul(vm, &zelf.elements, value)? - .cloned() - .collect::>() - .into_boxed_slice(); + let v = zelf.elements.mul(vm, value)?; + let elements = v.into_boxed_slice(); Self { elements }.into_ref(vm) }) } @@ -360,7 +358,7 @@ impl Comparable for PyTuple { let other = class_or_notimplemented!(Self, other); let a = zelf.as_slice(); let b = other.as_slice(); - sequence::cmp(vm, a.boxed_iter(), b.boxed_iter(), op).map(PyComparisonValue::Implemented) + a.cmp(vm, b, op).map(PyComparisonValue::Implemented) } } diff --git a/vm/src/builtins/weakproxy.rs b/vm/src/builtins/weakproxy.rs index e7446ad65..8cf4ab404 100644 --- a/vm/src/builtins/weakproxy.rs +++ b/vm/src/builtins/weakproxy.rs @@ -1,14 +1,14 @@ -use super::{PyStrRef, PyTypeRef, PyWeak}; +use super::{PyStrRef, PyTypeRef}; use crate::{ function::OptionalArg, types::{Constructor, SetAttr}, - PyClassImpl, PyContext, PyObjectRef, PyResult, PyValue, VirtualMachine, + PyClassImpl, PyContext, PyObjectRef, PyObjectWeak, PyResult, PyValue, VirtualMachine, }; #[pyclass(module = false, name = "weakproxy")] #[derive(Debug)] pub struct PyWeakProxy { - weak: PyWeak, + weak: PyObjectWeak, } impl PyValue for PyWeakProxy { @@ -33,16 +33,28 @@ impl Constructor for PyWeakProxy { Self::Args { referent, callback }: Self::Args, vm: &VirtualMachine, ) -> PyResult { - if callback.is_present() { - panic!("Passed a callback to weakproxy, but weakproxy does not yet support proxies."); - } + // using an internal subclass as the class prevents us from getting the generic weakref, + // which would mess up the weakref count + let weak_cls = WEAK_SUBCLASS.get_or_init(|| { + vm.ctx.new_class( + None, + "__weakproxy", + &vm.ctx.types.weakref_type, + super::PyWeak::make_slots(), + ) + }); + // TODO: PyWeakProxy should use the same payload as PyWeak PyWeakProxy { - weak: PyWeak::downgrade(&referent), + weak: referent.downgrade_with_typ(callback.into_option(), weak_cls.clone(), vm)?, } .into_pyresult_with_type(vm, cls) } } +crate::common::static_cell! { + static WEAK_SUBCLASS: PyTypeRef; +} + #[pyimpl(with(SetAttr, Constructor))] impl PyWeakProxy { // TODO: callbacks diff --git a/vm/src/builtins/weakref.rs b/vm/src/builtins/weakref.rs index 5b80f301c..841681444 100644 --- a/vm/src/builtins/weakref.rs +++ b/vm/src/builtins/weakref.rs @@ -1,40 +1,21 @@ use super::{PyGenericAlias, PyTypeRef}; -use crate::common::hash::PyHash; +use crate::common::atomic::{Ordering, Radium}; +use crate::common::hash::{self, PyHash}; use crate::{ function::OptionalArg, types::{Callable, Comparable, Constructor, Hashable, PyComparisonOp}, - IdProtocol, PyClassImpl, PyContext, PyObject, PyObjectRef, PyObjectWeak, PyRef, PyResult, - PyValue, TypeProtocol, VirtualMachine, + IdProtocol, PyClassImpl, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, VirtualMachine, }; -use crossbeam_utils::atomic::AtomicCell; - -#[pyclass(module = false, name = "weakref")] -#[derive(Debug)] -pub struct PyWeak { - referent: PyObjectWeak, - hash: AtomicCell>, -} - -impl PyWeak { - pub fn downgrade(obj: &PyObject) -> PyWeak { - PyWeak { - referent: obj.downgrade(), - hash: AtomicCell::new(None), - } - } - - pub fn upgrade(&self) -> Option { - self.referent.upgrade() - } -} +pub use crate::pyobjectrc::PyWeak; #[derive(FromArgs)] pub struct WeakNewArgs { #[pyarg(positional)] referent: PyObjectRef, #[pyarg(positional, optional)] - _callback: OptionalArg, + callback: OptionalArg, } impl PyValue for PyWeak { @@ -54,16 +35,13 @@ impl Callable for PyWeak { impl Constructor for PyWeak { type Args = WeakNewArgs; - // TODO callbacks fn py_new( cls: PyTypeRef, - Self::Args { - referent, - _callback, - }: Self::Args, + Self::Args { referent, callback }: Self::Args, vm: &VirtualMachine, ) -> PyResult { - PyWeak::downgrade(&referent).into_pyresult_with_type(vm, cls) + let weak = referent.downgrade_with_typ(callback.into_option(), cls, vm)?; + Ok(weak.into_object()) } } @@ -92,17 +70,26 @@ impl PyWeak { impl Hashable for PyWeak { fn hash(zelf: &crate::PyObjectView, vm: &VirtualMachine) -> PyResult { - match zelf.hash.load() { - Some(hash) => Ok(hash), - None => { + let hash = match zelf.hash.load(Ordering::Relaxed) { + hash::SENTINEL => { let obj = zelf .upgrade() .ok_or_else(|| vm.new_type_error("weak object has gone away".to_owned()))?; let hash = obj.hash(vm)?; - zelf.hash.store(Some(hash)); - Ok(hash) + match Radium::compare_exchange( + &zelf.hash, + hash::SENTINEL, + hash::fix_sentinel(hash), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => hash, + Err(prev_stored) => prev_stored, + } } - } + hash => hash, + }; + Ok(hash) } } @@ -118,7 +105,7 @@ impl Comparable for PyWeak { let both = zelf.upgrade().and_then(|s| other.upgrade().map(|o| (s, o))); let eq = match both { Some((a, b)) => vm.bool_eq(&a, &b)?, - None => false, + None => zelf.is(other), }; Ok(eq.into()) }) diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index e79c83227..31abb5d85 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -4,8 +4,8 @@ use crate::{ pystr, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, PyStrRef, PyTypeRef, }, cformat::CFormatBytes, - function::ArgIterable, - function::{OptionalArg, OptionalOption}, + function::{ArgIterable, OptionalArg, OptionalOption}, + sequence::{SequenceMutOp, SequenceOp}, sliceable::PySliceableSequence, types::PyComparisonOp, utils::Either, @@ -909,8 +909,12 @@ impl PyBytesInner { .format(vm, values) } - pub fn repeat(&self, n: usize) -> Vec { - self.elements.repeat(n) + pub fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult> { + self.elements.mul(vm, n) + } + + pub fn imul(&mut self, n: isize, vm: &VirtualMachine) -> PyResult<()> { + self.elements.imul(vm, n) } } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index dbbcfddde..4fe19d014 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -13,6 +13,7 @@ use crate::{ }; use std::fmt; use std::mem::size_of; +use std::ops::ControlFlow; // HashIndex is intended to be same size with hash::PyHash // but it doesn't mean the values are compatible with actual pyhash value @@ -223,6 +224,8 @@ impl DictInner { } } +type PopInnerResult = ControlFlow>>; + impl Dict { fn read(&self) -> PyRwLockReadGuard<'_, DictInner> { self.inner.read() @@ -348,18 +351,23 @@ impl Dict { pub fn delete_if_exists(&self, vm: &VirtualMachine, key: &K) -> PyResult where K: DictKey, + { + self.delete_if(vm, key, |_| Ok(true)) + } + + /// pred should be VERY CAREFUL about what it does as it is called while + /// the dict's internal mutex is held + pub(crate) fn delete_if(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult + where + K: DictKey, + F: Fn(&T) -> PyResult, { let hash = key.key_hash(vm)?; let deleted = loop { let lookup = self.lookup(vm, key, hash, None)?; - if let IndexEntry::Index(_) = lookup.0 { - if let Ok(Some(entry)) = self.pop_inner(lookup) { - break Some(entry); - } else { - // The dict was changed since we did lookup. Let's try again. - } - } else { - break None; + match self.pop_inner_if(lookup, &pred)? { + ControlFlow::Break(entry) => break entry, + ControlFlow::Continue(()) => continue, } }; Ok(deleted.is_some()) @@ -371,10 +379,9 @@ impl Dict { let lookup = self.lookup(vm, key, hash, None)?; let (entry, index_index) = lookup; if let IndexEntry::Index(_) = entry { - if let Ok(Some(entry)) = self.pop_inner(lookup) { - break Some(entry); - } else { - // The dict was changed since we did lookup. Let's try again. + match self.pop_inner(lookup) { + ControlFlow::Break(Some(entry)) => break Some(entry), + _ => continue, } } else { let mut inner = self.write(); @@ -573,25 +580,42 @@ impl Dict { } // returns Err(()) if changed since lookup - fn pop_inner(&self, lookup: LookupResult) -> Result>, ()> { + fn pop_inner(&self, lookup: LookupResult) -> PopInnerResult { + self.pop_inner_if(lookup, |_| Ok::<_, std::convert::Infallible>(true)) + .unwrap_or_else(|x| match x {}) + } + + fn pop_inner_if( + &self, + lookup: LookupResult, + pred: impl Fn(&T) -> Result, + ) -> Result, E> { let (entry_index, index_index) = lookup; let entry_index = if let IndexEntry::Index(entry_index) = entry_index { entry_index } else { - return Ok(None); + return Ok(ControlFlow::Break(None)); }; - let mut inner = self.write(); - if matches!(inner.entries.get(entry_index), Some(Some(entry)) if entry.index == index_index) - { - // all good + let inner = &mut *self.write(); + let slot = if let Some(slot) = inner.entries.get_mut(entry_index) { + slot } else { // The dict was changed since we did lookup. Let's try again. - return Err(()); + return Ok(ControlFlow::Continue(())); }; + match slot { + Some(entry) if entry.index == index_index => { + if !pred(&entry.value)? { + return Ok(ControlFlow::Break(None)); + } + } + // The dict was changed since we did lookup. Let's try again. + _ => return Ok(ControlFlow::Continue(())), + } inner.indices[index_index] = IndexEntry::DUMMY; inner.used -= 1; - let removed = std::mem::take(&mut inner.entries[entry_index]); - Ok(removed) + let removed = slot.take(); + Ok(ControlFlow::Break(removed)) } /// Retrieve and delete a key @@ -599,10 +623,9 @@ impl Dict { let hash_value = key.key_hash(vm)?; let removed = loop { let lookup = self.lookup(vm, key, hash_value, None)?; - if let Ok(ret) = self.pop_inner(lookup) { - break ret.map(|e| e.value); - } else { - // changed since lookup, loop again + match self.pop_inner(lookup) { + ControlFlow::Break(entry) => break entry.map(|e| e.value), + ControlFlow::Continue(()) => continue, } }; Ok(removed) diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 804b31344..3d897fe8b 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -66,7 +66,7 @@ mod pyobject; mod pyobjectrc; pub mod readline; pub mod scope; -mod sequence; +pub mod sequence; pub mod signal; pub mod sliceable; pub mod stdlib; diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d7cff5d60..e6f0abd5e 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -4,17 +4,16 @@ use crate::common::{ static_cell, }; pub use crate::pyobjectrc::{ - PyGenericObject, PyObject, PyObjectRef, PyObjectView, PyObjectWeak, PyObjectWrap, PyRef, - PyWeakRef, + PyObject, PyObjectRef, PyObjectView, PyObjectWeak, PyObjectWrap, PyRef, PyWeakRef, }; use crate::{ builtins::{ builtinfunc::{PyBuiltinFunction, PyBuiltinMethod, PyNativeFuncDef}, bytes, getset::{IntoPyGetterFunc, IntoPySetterFunc, PyGetSet}, - object, pystr, PyBaseExceptionRef, PyBoundMethod, PyDict, PyDictRef, PyEllipsis, PyFloat, - PyFrozenSet, PyGenericAlias, PyInt, PyIntRef, PyList, PyListRef, PyNone, PyNotImplemented, - PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef, + object, pystr, PyBaseException, PyBaseExceptionRef, PyBoundMethod, PyDict, PyDictRef, + PyEllipsis, PyFloat, PyFrozenSet, PyGenericAlias, PyInt, PyIntRef, PyList, PyListRef, + PyNone, PyNotImplemented, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef, }, dictdatatype::Dict, exceptions, @@ -94,6 +93,7 @@ impl PyContext { let types = TypeZoo::init(); let exceptions = exceptions::ExceptionZoo::init(); + #[inline] fn create_object(payload: T, cls: &PyTypeRef) -> PyRef { PyRef::new_ref(payload, cls.clone(), None) } @@ -242,6 +242,30 @@ impl PyContext { .unwrap() } + pub fn new_exception_type( + &self, + module: &str, + name: &str, + bases: Option>, + ) -> PyTypeRef { + let bases = if let Some(bases) = bases { + bases + } else { + vec![self.exceptions.exception_type.clone()] + }; + let mut attrs = PyAttributes::default(); + attrs.insert("__module__".to_string(), self.new_str(module).into()); + + PyType::new_ref( + name, + bases, + attrs, + PyBaseException::make_slots(), + self.types.type_type.clone(), + ) + .unwrap() + } + #[inline] pub fn make_funcdef(&self, name: impl Into, f: F) -> PyNativeFuncDef where @@ -309,7 +333,7 @@ impl PyContext { class.slots.flags.contains(PyTypeFlags::HAS_DICT), dict.is_some() ); - PyGenericObject::new(object::PyBaseObject, class, dict) + PyRef::new_ref(object::PyBaseObject, class, dict).into() } } diff --git a/vm/src/pyobjectrc.rs b/vm/src/pyobjectrc.rs index 0e7832983..d6ebc3ccd 100644 --- a/vm/src/pyobjectrc.rs +++ b/vm/src/pyobjectrc.rs @@ -1,18 +1,21 @@ -use crate::common::{ - lock::PyRwLock, - rc::{PyRc, PyWeak}, -}; +use crate::common::atomic::{OncePtr, PyAtomic, Radium}; +use crate::common::linked_list::{Link, LinkedList, Pointers}; +use crate::common::lock::{PyMutex, PyMutexGuard, PyRwLock}; +use crate::common::refcount::RefCount; use crate::{ builtins::{PyBaseExceptionRef, PyDictRef, PyTypeRef}, - IdProtocol, PyObjectPayload, TypeProtocol, VirtualMachine, + IdProtocol, PyObjectPayload, PyResult, TypeProtocol, VirtualMachine, }; use std::any::TypeId; use std::borrow::Borrow; +use std::cell::UnsafeCell; use std::fmt; +use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ops::Deref; +use std::ptr::{self, NonNull}; -// so, PyObjectRef is basically equivalent to `PyRc>`, except it's +// so, PyObjectRef is basically equivalent to `PyRc>`, except it's // only one pointer in width rather than 2. We do that by manually creating a vtable, and putting // a &'static reference to it inside the `PyRc` rather than adjacent to it, like trait objects do. // This can lead to faster code since there's just less data to pass around, as well as because of @@ -29,18 +32,18 @@ use std::ops::Deref; // There has to be padding in the space between the 2 fields. But, if that field is a trait object // (like `dyn PyObjectPayload`) we don't *know* how much padding there is between the `payload` // field and the previous field. So, Rust has to consult the vtable to know the exact offset of -// `payload` in `PyGenericObject`, which has a huge performance impact when *every +// `payload` in `PyInner`, which has a huge performance impact when *every // single payload access* requires a vtable lookup. Thankfully, we're able to avoid that because of // the way we use PyObjectRef, in that whenever we want to access the payload we (almost) always // access it from a generic function. So, rather than doing // // - check vtable for payload offset -// - get offset in PyGenericObject struct +// - get offset in PyInner struct // - call as_any() method of PyObjectPayload // - call downcast_ref() method of Any // we can just do // - check vtable that typeid matches -// - pointer cast directly to *const PyGenericObject +// - pointer cast directly to *const PyInner // // and at that point the compiler can know the offset of `payload` for us because **we've given it a // concrete type to work with before we ever access the `payload` field** @@ -49,36 +52,46 @@ use std::ops::Deref; struct Erased; struct PyObjVTable { - drop: unsafe fn(*mut PyInner), - debug: unsafe fn(*const PyInner, &mut fmt::Formatter) -> fmt::Result, + drop_dealloc: unsafe fn(*mut PyObject), + debug: unsafe fn(&PyObject, &mut fmt::Formatter) -> fmt::Result, } -unsafe fn drop_obj(x: *mut PyInner) { - std::ptr::drop_in_place(x as *mut PyInner) +unsafe fn drop_dealloc_obj(x: *mut PyObject) { + Box::from_raw(x as *mut PyInner); } -unsafe fn debug_obj( - x: *const PyInner, - f: &mut fmt::Formatter, -) -> fmt::Result { - let x = &*x.cast::>(); +unsafe fn debug_obj(x: &PyObject, f: &mut fmt::Formatter) -> fmt::Result { + let x = &*(x as *const PyObject as *const PyInner); fmt::Debug::fmt(x, f) } + impl PyObjVTable { pub fn of() -> &'static Self { - &PyObjVTable { - drop: drop_obj::, - debug: debug_obj::, + struct Helper(PhantomData); + trait VtableHelper { + const VTABLE: PyObjVTable; } + impl VtableHelper for Helper { + const VTABLE: PyObjVTable = PyObjVTable { + drop_dealloc: drop_dealloc_obj::, + debug: debug_obj::, + }; + } + &Helper::::VTABLE } } +/// This is an actual python object. It consists of a `typ` which is the +/// python class, and carries some rust payload optionally. This rust +/// payload can be a rust float or rust int in case of float and int objects. #[repr(C)] struct PyInner { + ref_count: RefCount, // TODO: move typeid into vtable once TypeId::of is const typeid: TypeId, vtable: &'static PyObjVTable, - typ: PyRwLock, // __class__ member - dict: Option>, // __dict__ member + typ: PyRwLock, // __class__ member + dict: Option, + weak_list: WeakRefList, payload: T, } @@ -89,36 +102,311 @@ impl fmt::Debug for PyInner { } } -/// This is an actual python object. It consists of a `typ` which is the -/// python class, and carries some rust payload optionally. This rust -/// payload can be a rust float or rust int in case of float and int objects. -#[repr(transparent)] -pub struct PyGenericObject { - inner: ManuallyDrop>, +struct WeakRefList { + inner: OncePtr>, } -impl PyGenericObject { - #[allow(clippy::new_ret_no_self)] - pub fn new(payload: T, typ: PyTypeRef, dict: Option) -> PyObjectRef { - let inner = PyInner { - typeid: TypeId::of::(), - vtable: PyObjVTable::of::(), - typ: PyRwLock::new(typ), - dict: dict.map(PyRwLock::new), - payload, - }; - PyObjectRef::new(PyGenericObject { - inner: ManuallyDrop::new(inner), - }) +impl fmt::Debug for WeakRefList { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("WeakRefList").finish_non_exhaustive() } } -impl Drop for PyGenericObject { +struct WeakListInner { + list: LinkedList>, + generic_weakref: Option>>, + obj: Option>, + // one for each live PyWeak with a reference to this, + 1 for the referent object if it's not dead + ref_count: usize, +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for WeakListInner {} + unsafe impl Sync for WeakListInner {} + } +} + +impl WeakRefList { + pub fn new() -> Self { + WeakRefList { + inner: OncePtr::new(), + } + } + + /// returns None if there have never been any weakrefs in this list + fn try_lock(&self) -> Option> { + self.inner.get().map(|mu| unsafe { mu.as_ref().lock() }) + } + + fn add( + &self, + obj: &PyObject, + cls: PyTypeRef, + cls_is_weakref: bool, + callback: Option, + dict: Option, + ) -> PyObjectWeak { + let is_generic = cls_is_weakref && callback.is_none(); + let inner_ptr = self.inner.get_or_init(|| { + Box::new(PyMutex::new(WeakListInner { + list: LinkedList::default(), + generic_weakref: None, + obj: Some(NonNull::from(obj)), + ref_count: 1, + })) + }); + let mut inner = unsafe { inner_ptr.as_ref().lock() }; + if is_generic { + if let Some(generic_weakref) = inner.generic_weakref { + let generic_weakref = unsafe { generic_weakref.as_ref() }; + if generic_weakref.0.ref_count.get() != 0 { + return PyObjectWeak { + weak: generic_weakref.to_owned(), + }; + } + } + } + let obj = PyWeak { + pointers: Pointers::new(), + parent: inner_ptr, + callback: UnsafeCell::new(callback), + hash: Radium::new(crate::common::hash::SENTINEL), + }; + let weak = PyRef::new_ref(obj, cls, dict); + // SAFETY: we don't actually own the PyObjectWeaks inside `list`, and every time we take + // one out of the list we immediately wrap it in ManuallyDrop or forget it + inner.list.push_front(unsafe { ptr::read(&weak) }); + inner.ref_count += 1; + if is_generic { + inner.generic_weakref = Some(NonNull::from(&*weak)); + } + PyObjectWeak { weak } + } + + fn clear(&self) { + let to_dealloc = { + let ptr = match self.inner.get() { + Some(ptr) => ptr, + None => return, + }; + let mut inner = unsafe { ptr.as_ref().lock() }; + inner.obj = None; + // TODO: can be an arrayvec + let mut v = Vec::with_capacity(16); + loop { + let inner2 = &mut *inner; + let iter = inner2 + .list + .drain_filter(|_| true) + .filter_map(|wr| { + // we don't have actual ownership of the reference counts in the list. + // but, now we do want ownership (and so incref these *while the lock + // is held*) to avoid weird things if PyWeakObj::drop happens after + // this but before we reach the loop body below + let wr = ManuallyDrop::new(wr); + + if Some(NonNull::from(&**wr)) == inner2.generic_weakref { + inner2.generic_weakref = None + } + + // if strong_count == 0 there's some reentrancy going on. we don't + // want to call the callback + (wr.as_object().strong_count() > 0).then(|| (*wr).clone()) + }) + .take(16); + v.extend(iter); + if v.is_empty() { + break; + } + PyMutexGuard::unlocked(&mut inner, || { + for wr in v.drain(..) { + let cb = unsafe { wr.callback.get().replace(None) }; + if let Some(cb) = cb { + crate::vm::thread::with_vm(&cb, |vm| { + // TODO: handle unraisable exception + let _ = vm.invoke(&cb, (wr.clone(),)); + }); + } + } + }) + } + inner.ref_count -= 1; + (inner.ref_count == 0).then(|| ptr) + }; + if let Some(ptr) = to_dealloc { + unsafe { WeakRefList::dealloc(ptr) } + } + } + + fn count(&self) -> usize { + self.try_lock() + // we assume the object is still alive (and this is only + // called from PyObject::weak_count so it should be) + .map(|inner| inner.ref_count - 1) + .unwrap_or(0) + } + + unsafe fn dealloc(ptr: NonNull>) { + Box::from_raw(ptr.as_ptr()); + } + + fn get_weak_references(&self) -> Vec { + let inner = match self.try_lock() { + Some(inner) => inner, + None => return vec![], + }; + let mut v = Vec::with_capacity(inner.ref_count - 1); + v.extend(inner.iter().map(|wr| PyObjectWeak { + weak: wr.to_owned(), + })); + v + } +} + +impl WeakListInner { + fn iter(&self) -> impl Iterator> { + self.list.iter().filter(|wr| wr.0.ref_count.get() > 0) + } +} + +impl Default for WeakRefList { + fn default() -> Self { + Self::new() + } +} + +struct WeakLink; +unsafe impl Link for WeakLink { + type Handle = PyRef; + + type Target = PyObjectView; + + fn as_raw(handle: &PyRef) -> NonNull { + NonNull::from(&**handle) + } + + unsafe fn from_raw(ptr: NonNull) -> Self::Handle { + PyRef::from_raw(ptr.as_ptr()) + } + + unsafe fn pointers(target: NonNull) -> NonNull> { + NonNull::new_unchecked(ptr::addr_of_mut!((*target.as_ptr()).0.payload.pointers)) + } +} + +#[pyclass(name = "weakref", module = false)] +#[derive(Debug)] +pub struct PyWeak { + pointers: Pointers>, + parent: NonNull>, + // this is treated as part of parent's mutex - you must hold that lock to access it + callback: UnsafeCell>, + pub(crate) hash: PyAtomic, +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyWeak {} + unsafe impl Sync for PyWeak {} + } +} + +impl PyWeak { + pub(crate) fn upgrade(&self) -> Option { + let guard = unsafe { self.parent.as_ref().lock() }; + let obj_ptr = guard.obj?; + unsafe { + if !obj_ptr.as_ref().0.ref_count.safe_inc() { + return None; + } + Some(PyObjectRef::from_raw(obj_ptr.as_ptr())) + } + } + + pub(crate) fn is_dead(&self) -> bool { + let guard = unsafe { self.parent.as_ref().lock() }; + guard.obj.is_none() + } + + #[inline(always)] + fn drop_inner(&self) { + let dealloc = { + let mut guard = unsafe { self.parent.as_ref().lock() }; + let offset = memoffset::offset_of!(PyInner, payload); + let pyinner = (self as *const Self as usize - offset) as *const PyInner; + let node_ptr = unsafe { NonNull::new_unchecked(pyinner as *mut PyObjectView) }; + // the list doesn't have ownership over its PyRef! we're being dropped + // right now so that should be obvious!! + std::mem::forget(unsafe { guard.list.remove(node_ptr) }); + guard.ref_count -= 1; + if Some(node_ptr) == guard.generic_weakref { + guard.generic_weakref = None; + } + guard.ref_count == 0 + }; + if dealloc { + unsafe { WeakRefList::dealloc(self.parent) } + } + } +} + +impl Drop for PyWeak { fn drop(&mut self) { - let erased = &mut *self.inner as *mut _ as *mut PyInner; - // SAFETY: the vtable contains functions that accept payload types that always match up - // with the payload of the object - unsafe { (self.inner.vtable.drop)(erased) } + // we do NOT have actual exclusive access! + // no clue if doing this actually reduces chance of UB + let me: &Self = self; + me.drop_inner(); + } +} + +#[derive(Debug)] +struct InstanceDict { + d: PyRwLock, +} + +impl From for InstanceDict { + #[inline] + fn from(d: PyDictRef) -> Self { + Self::new(d) + } +} + +impl InstanceDict { + #[inline] + pub fn new(d: PyDictRef) -> Self { + Self { + d: PyRwLock::new(d), + } + } + + #[inline] + pub fn get(&self) -> PyDictRef { + self.d.read().clone() + } + + #[inline] + pub fn set(&self, d: PyDictRef) { + self.replace(d); + } + + #[inline] + pub fn replace(&self, d: PyDictRef) -> PyDictRef { + std::mem::replace(&mut self.d.write(), d) + } +} + +impl PyInner { + fn new(payload: T, typ: PyTypeRef, dict: Option) -> Box { + Box::new(PyInner { + ref_count: RefCount::new(), + typeid: TypeId::of::(), + vtable: PyObjVTable::of::(), + typ: PyRwLock::new(typ), + dict: dict.map(InstanceDict::new), + weak_list: WeakRefList::new(), + payload, + }) } } @@ -127,34 +415,50 @@ impl Drop for PyGenericObject { /// this reference counting is accounted for by this type. Use the `.clone()` /// method to create a new reference and increment the amount of references /// to the python object by 1. -#[derive(Clone)] #[repr(transparent)] pub struct PyObjectRef { - rc: PyRc, + ptr: NonNull, +} + +impl Clone for PyObjectRef { + #[inline(always)] + fn clone(&self) -> Self { + (**self).to_owned() + } +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyObjectRef {} + unsafe impl Sync for PyObjectRef {} + } } #[derive(Clone)] #[repr(transparent)] pub struct PyObjectWeak { - weak: PyWeak, + weak: PyRef, } #[repr(transparent)] -pub struct PyObject(PyGenericObject); +pub struct PyObject(PyInner); impl Deref for PyObjectRef { type Target = PyObject; fn deref(&self) -> &PyObject { - &self.rc + unsafe { self.ptr.as_ref() } } } impl ToOwned for PyObject { type Owned = PyObjectRef; - #[inline] + #[inline(always)] fn to_owned(&self) -> Self::Owned { - self.with_pyobjectref(Clone::clone) + self.0.ref_count.inc(); + PyObjectRef { + ptr: NonNull::from(self), + } } } @@ -184,16 +488,10 @@ impl PyObjectRef { /// too many times. pub unsafe fn from_raw(ptr: *const PyObject) -> Self { Self { - rc: PyRc::from_raw(ptr.cast()), + ptr: NonNull::new_unchecked(ptr as *mut PyObject), } } - fn new(value: PyGenericObject) -> Self { - let inner: *const PyGenericObject = PyRc::into_raw(PyRc::new(value)); - let rc = unsafe { PyRc::from_raw(inner as *const PyObject) }; - Self { rc } - } - /// Attempt to downcast this reference to a subclass. /// /// If the downcast fails, the original ref is returned in as `Err` so @@ -255,29 +553,67 @@ impl PyObjectRef { impl PyObject { #[inline] - fn with_pyobjectref(&self, f: impl FnOnce(&PyObjectRef) -> R) -> R { - // SAFETY: we don't use obj after the real lifetime (self) goes out of scope, we wrap it - // in a ManuallyDrop to prevent double free, and we only pass it by reference - let obj = unsafe { ManuallyDrop::new(PyObjectRef::from_raw(self)) }; - f(&obj) + fn weak_ref_list(&self) -> Option<&WeakRefList> { + Some(&self.0.weak_list) } - pub fn downgrade(&self) -> PyObjectWeak { - PyObjectWeak { - weak: self.with_pyobjectref(|obj| PyRc::downgrade(&obj.rc)), - } + pub(crate) fn downgrade_with_weakref_typ_opt( + &self, + callback: Option, + // a reference to weakref_type **specifically** + typ: PyTypeRef, + ) -> Option { + self.weak_ref_list() + .map(|wrl| wrl.add(self, typ, true, callback, None)) + } + + pub(crate) fn downgrade_with_typ( + &self, + callback: Option, + typ: PyTypeRef, + vm: &VirtualMachine, + ) -> PyResult { + let dict = if typ + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + { + Some(vm.ctx.new_dict()) + } else { + None + }; + let cls_is_weakref = typ.is(&vm.ctx.types.weakref_type); + self.weak_ref_list() + .map(|wrl| wrl.add(self, typ, cls_is_weakref, callback, dict)) + .ok_or_else(|| { + vm.new_type_error(format!( + "cannot create weak reference to '{}' object", + self.class().name() + )) + }) + } + + pub fn downgrade( + &self, + callback: Option, + vm: &VirtualMachine, + ) -> PyResult { + self.downgrade_with_typ(callback, vm.ctx.types.weakref_type.clone(), vm) + } + + pub fn get_weak_references(&self) -> Option> { + self.weak_ref_list().map(|wrl| wrl.get_weak_references()) } pub fn payload_is(&self) -> bool { - self.0.inner.typeid == TypeId::of::() + self.0.typeid == TypeId::of::() } pub fn payload(&self) -> Option<&T> { if self.payload_is::() { // we cast to a PyInner first because we don't know T's exact offset because of // varying alignment, but once we get a PyInner the compiler can get it for us - let inner = - unsafe { &*(&*self.0.inner as *const PyInner as *const PyInner) }; + let inner = unsafe { &*(&self.0 as *const PyInner as *const PyInner) }; Some(&inner.payload) } else { None @@ -285,7 +621,7 @@ impl PyObject { } pub(crate) fn class_lock(&self) -> &PyRwLock { - &self.0.inner.typ + &self.0.typ } #[inline] @@ -300,15 +636,20 @@ impl PyObject { } } + #[inline] + fn instance_dict(&self) -> Option<&InstanceDict> { + self.0.dict.as_ref() + } + pub fn dict(&self) -> Option { - self.0.inner.dict.as_ref().map(|mu| mu.read().clone()) + self.instance_dict().map(|d| d.get()) } /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field /// in the first place. pub fn set_dict(&self, dict: PyDictRef) -> Result<(), PyDictRef> { - match self.0.inner.dict { - Some(ref mu) => { - *mu.write() = dict; + match self.instance_dict() { + Some(d) => { + d.set(dict); Ok(()) } None => Err(dict), @@ -343,18 +684,59 @@ impl PyObject { #[inline] pub fn strong_count(&self) -> usize { - self.with_pyobjectref(|obj| PyRc::strong_count(&obj.rc)) + self.0.ref_count.get() } #[inline] - pub fn weak_count(&self) -> usize { - self.with_pyobjectref(|obj| PyRc::weak_count(&obj.rc)) + pub fn weak_count(&self) -> Option { + self.weak_ref_list().map(|wrl| wrl.count()) } #[inline] pub fn as_raw(&self) -> *const PyObject { self } + + #[inline] + fn drop_slow_inner(&self) -> Result<(), ()> { + // CPython-compatible drop implementation + if let Some(slot_del) = self.class().mro_find_map(|cls| cls.slots.del.load()) { + let ret = crate::vm::thread::with_vm(self, |vm| { + self.0.ref_count.inc(); + if let Err(e) = slot_del(self, vm) { + print_del_error(e, self, vm); + } + self.0.ref_count.dec() + }); + match ret { + // the decref right above set ref_count back to 0 + Some(true) => {} + // we've been resurrected by __del__ + Some(false) => return Err(()), + None => { + warn!("couldn't run __del__ method for object") + } + } + } + if let Some(wrl) = self.weak_ref_list() { + wrl.clear(); + } + + Ok(()) + } + + /// Can only be called when ref_count has dropped to zero. `ptr` must be valid + #[inline(never)] + #[cold] + unsafe fn drop_slow(ptr: NonNull) { + if let Err(()) = ptr.as_ref().drop_slow_inner() { + // abort drop for whatever reason + return; + } + let drop_dealloc = ptr.as_ref().0.vtable.drop_dealloc; + // call drop only when there are no references in scope - stacked borrows stuff + drop_dealloc(ptr.as_ptr()) + } } impl Borrow for PyObjectRef { @@ -377,7 +759,7 @@ impl AsRef for PyObject { impl IdProtocol for PyObjectRef { fn get_id(&self) -> usize { - self.rc.get_id() + self.ptr.as_ptr() as usize } } @@ -403,33 +785,21 @@ where } impl PyObjectWeak { + #[inline] pub fn upgrade(&self) -> Option { - self.weak.upgrade().map(|rc| PyObjectRef { rc }) + self.weak.upgrade() + } + + pub fn into_object(self) -> PyObjectRef { + self.weak.into_object() } } impl Drop for PyObjectRef { fn drop(&mut self) { - // PyObjectRef will drop the value when its count goes to 0 - if PyRc::strong_count(&self.rc) != 1 { - return; + if self.0.ref_count.dec() { + unsafe { PyObject::drop_slow(self.ptr) } } - - // CPython-compatible drop implementation - let zelf = self.clone(); - if let Some(slot_del) = self.class().mro_find_map(|cls| cls.slots.del.load()) { - let ret = crate::vm::thread::with_vm(&zelf, |vm| { - if let Err(e) = slot_del(&zelf, vm) { - print_del_error(e, &zelf, vm); - } - }); - if ret.is_none() { - warn!("couldn't run __del__ method for object") - } - } - - // __del__ might have resurrected the object at this point, but that's fine, - // inner.strong_count would be >1 now and it'll maybe get dropped the next time } } @@ -457,8 +827,7 @@ impl fmt::Debug for PyObjectRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: the vtable contains functions that accept payload types that always match up // with the payload of the object - let inner = &*self.rc.0.inner; - unsafe { (inner.vtable.debug)(inner, f) } + unsafe { ((*self).0.vtable.debug)(self, f) } } } @@ -477,19 +846,14 @@ impl PyObjectView { unsafe { &*(&self.0 as *const PyInner as *const PyObject) } } - #[inline] - fn with_pyref(&self, f: impl FnOnce(&PyRef) -> R) -> R { - unsafe { - // SAFETY: we don't use obj after the real lifetime (self) goes out of scope, we wrap it - // in a ManuallyDrop to prevent double free, and we only pass it by reference - let obj = ManuallyDrop::new(PyRef::from_raw(self)); - f(&obj) - } - } - - pub fn downgrade(&self) -> PyWeakRef { - self.with_pyref(|r| PyWeakRef { - weak: PyRc::downgrade(&r.obj), + pub fn downgrade( + &self, + callback: Option, + vm: &VirtualMachine, + ) -> PyResult> { + Ok(PyWeakRef { + weak: self.as_object().downgrade(callback, vm)?, + _marker: PhantomData, }) } } @@ -497,8 +861,12 @@ impl PyObjectView { impl ToOwned for PyObjectView { type Owned = PyRef; + #[inline(always)] fn to_owned(&self) -> Self::Owned { - self.with_pyref(Clone::clone) + self.0.ref_count.inc(); + PyRef { + ptr: NonNull::from(self), + } } } @@ -536,47 +904,61 @@ impl fmt::Debug for PyObjectView { /// where a reference to the same object must be returned. #[repr(transparent)] pub struct PyRef { - // invariant: this obj must always have payload of type T - obj: PyRc>, + ptr: NonNull>, +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyRef {} + unsafe impl Sync for PyRef {} + } } impl fmt::Debug for PyRef { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.obj.fmt(f) + (**self).fmt(f) + } +} + +impl Drop for PyRef { + #[inline] + fn drop(&mut self) { + if self.0.ref_count.dec() { + unsafe { PyObject::drop_slow(self.ptr.cast::()) } + } } } impl Clone for PyRef { + #[inline(always)] fn clone(&self) -> Self { - Self { - obj: self.obj.clone(), - } + (**self).to_owned() } } impl PyRef { unsafe fn from_raw(raw: *const PyObjectView) -> Self { Self { - obj: PyRc::from_raw(raw), + ptr: NonNull::new_unchecked(raw as *mut _), } } /// Safety: payload type of `obj` must be `T` + #[inline] unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self { debug_assert!(obj.payload_is::()); - let obj = PyRc::from_raw(obj.into_raw() as *const PyObjectView); - Self { obj } + let obj = ManuallyDrop::new(obj); + Self { + ptr: obj.ptr.cast(), + } } - // ideally we'd be able to define this in pyobject.rs, but method visibility rules are weird - pub fn new_ref( - payload: T, - typ: crate::builtins::PyTypeRef, - dict: Option, - ) -> Self { - let obj = PyGenericObject::new(payload, typ, dict); - // SAFETY: we just created the object from a payload of type T - unsafe { Self::from_obj_unchecked(obj) } + #[inline] + pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { + let inner = Box::into_raw(PyInner::new(payload, typ, dict)); + Self { + ptr: unsafe { NonNull::new_unchecked(inner.cast::>()) }, + } } } @@ -584,8 +966,10 @@ impl PyObjectWrap for PyRef where T: PyObjectPayload, { + #[inline] fn into_object(self) -> PyObjectRef { - unsafe { PyObjectRef::from_raw(PyRc::into_raw(self.obj) as *const PyObject) } + let me = ManuallyDrop::new(self); + PyObjectRef { ptr: me.ptr.cast() } } } @@ -593,6 +977,7 @@ impl AsRef for PyRef where T: PyObjectPayload, { + #[inline(always)] fn as_ref(&self) -> &PyObject { (**self).as_object() } @@ -613,19 +998,24 @@ where { type Target = PyObjectView; + #[inline(always)] fn deref(&self) -> &PyObjectView { - &self.obj + unsafe { self.ptr.as_ref() } } } #[repr(transparent)] pub struct PyWeakRef { - weak: PyWeak>, + weak: PyObjectWeak, + _marker: PhantomData, } impl PyWeakRef { pub fn upgrade(&self) -> Option> { - self.weak.upgrade().map(|obj| PyRef { obj }) + self.weak + .upgrade() + // SAFETY: PyWeakRef was always created from a PyRef, so the object is T + .map(|obj| unsafe { PyRef::from_obj_unchecked(obj) }) } } @@ -653,18 +1043,17 @@ macro_rules! partially_init { }}; } -pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef) { - use crate::builtins::{object, PyType, PyWeak}; +pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { + use crate::builtins::{object, PyType}; use crate::{PyAttributes, PyClassImpl}; use std::mem::MaybeUninit; - use std::ptr; // `type` inherits from `object` // and both `type` and `object are instances of `type`. // to produce this circular dependency, we need an unsafe block. // (and yes, this will never get dropped. TODO?) let (type_type, object_type) = { - type UninitRef = PyRwLock>>>; + type UninitRef = PyRwLock>>; // We cast between these 2 types, so make sure (at compile time) that there's no change in // layout when we wrap PyInner in MaybeUninit<> @@ -687,59 +1076,83 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef) { attributes: PyRwLock::new(PyAttributes::default()), slots: object::PyBaseObject::make_slots(), }; - let type_type = PyRc::new(partially_init!( + let type_type_ptr = Box::into_raw(Box::new(partially_init!( PyInner:: { + ref_count: RefCount::new(), typeid: TypeId::of::(), vtable: PyObjVTable::of::(), dict: None, + weak_list: WeakRefList::new(), payload: type_payload, }, Uninit { typ } - )); - let object_type = PyRc::new(partially_init!( + ))); + let object_type_ptr = Box::into_raw(Box::new(partially_init!( PyInner:: { + ref_count: RefCount::new(), typeid: TypeId::of::(), vtable: PyObjVTable::of::(), dict: None, + weak_list: WeakRefList::new(), payload: object_payload, }, Uninit { typ }, - )); + ))); - let object_type_ptr = PyRc::into_raw(object_type) as *mut MaybeUninit> - as *mut PyInner; - let type_type_ptr = PyRc::into_raw(type_type.clone()) as *mut MaybeUninit> - as *mut PyInner; + let object_type_ptr = + object_type_ptr as *mut MaybeUninit> as *mut PyInner; + let type_type_ptr = + type_type_ptr as *mut MaybeUninit> as *mut PyInner; unsafe { + (*type_type_ptr).ref_count.inc(); ptr::write( &mut (*object_type_ptr).typ as *mut PyRwLock as *mut UninitRef, - PyRwLock::new(type_type.clone()), + PyRwLock::new(NonNull::new_unchecked(type_type_ptr)), ); + (*type_type_ptr).ref_count.inc(); ptr::write( &mut (*type_type_ptr).typ as *mut PyRwLock as *mut UninitRef, - PyRwLock::new(type_type), + PyRwLock::new(NonNull::new_unchecked(type_type_ptr)), ); - let type_type = - PyTypeRef::from_obj_unchecked(PyObjectRef::from_raw(type_type_ptr.cast())); - let object_type = - PyTypeRef::from_obj_unchecked(PyObjectRef::from_raw(object_type_ptr.cast())); + let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); (*type_type_ptr).payload.mro = vec![object_type.clone()]; (*type_type_ptr).payload.bases = vec![object_type.clone()]; (*type_type_ptr).payload.base = Some(object_type.clone()); + let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + (type_type, object_type) } }; - object_type - .subclasses - .write() - .push(PyWeak::downgrade(type_type.as_object())); + let weakref_type = PyType { + base: Some(object_type.clone()), + bases: vec![object_type.clone()], + mro: vec![object_type.clone()], + subclasses: PyRwLock::default(), + attributes: PyRwLock::default(), + slots: PyWeak::make_slots(), + }; + let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); - (type_type, object_type) + object_type.subclasses.write().push( + type_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); + + object_type.subclasses.write().push( + weakref_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); + + (type_type, object_type, weakref_type) } #[cfg(test)] @@ -749,4 +1162,11 @@ mod tests { fn miri_test_type_initialization() { let _ = init_type_hierarchy(); } + + #[test] + fn miri_test_drop() { + let ctx = crate::PyContext::new(); + let obj = ctx.new_bytes(b"dfghjkl".to_vec()); + drop(obj); + } } diff --git a/vm/src/scope.rs b/vm/src/scope.rs index 0ab2587d3..30d833e4a 100644 --- a/vm/src/scope.rs +++ b/vm/src/scope.rs @@ -33,7 +33,7 @@ impl Scope { ) -> Scope { if !globals.contains_key("__builtins__", vm) { globals - .set_item("__builtins__", vm.builtins.clone(), vm) + .set_item("__builtins__", vm.builtins.clone().into(), vm) .unwrap(); } Scope::new(locals, globals) diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index d12be98f3..d06e192f8 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -1,107 +1,294 @@ -use crate::types::PyComparisonOp; -use crate::vm::VirtualMachine; -use crate::{PyObjectRef, PyResult}; +use itertools::Itertools; +use optional::Optioned; +use std::{collections::VecDeque, ops::Range}; -pub(super) type DynPyIter<'a> = Box + 'a>; +use crate::{ + types::{richcompare_wrapper, PyComparisonOp, RichCompareFunc}, + utils::Either, + vm::VirtualMachine, + IdProtocol, PyComparisonValue, PyObject, PyObjectRef, PyResult, TypeProtocol, +}; -#[allow(clippy::len_without_is_empty)] -pub(crate) trait SimpleSeq { - fn len(&self) -> usize; - fn boxed_iter(&self) -> DynPyIter; -} +pub trait ObjectSequenceOp<'a> { + type Iter: ExactSizeIterator; -impl<'a, D> SimpleSeq for D -where - D: 'a + std::ops::Deref, -{ - fn len(&self) -> usize { - self.deref().len() - } + fn iter(&'a self) -> Self::Iter; - fn boxed_iter(&self) -> DynPyIter { - Box::new(self.deref().iter()) - } -} - -pub(crate) fn eq(vm: &VirtualMachine, zelf: DynPyIter, other: DynPyIter) -> PyResult { - if zelf.len() == other.len() { - for (a, b) in Iterator::zip(zelf, other) { + fn eq(&'a self, vm: &VirtualMachine, other: &'a Self) -> PyResult { + let lhs = self.iter(); + let rhs = other.iter(); + if lhs.len() != rhs.len() { + return Ok(false); + } + for (a, b) in lhs.zip_eq(rhs) { if !vm.identical_or_equal(a, b)? { return Ok(false); } } Ok(true) - } else { - Ok(false) } -} -pub fn cmp( - vm: &VirtualMachine, - zelf: DynPyIter, - other: DynPyIter, - op: PyComparisonOp, -) -> PyResult { - let less = match op { - PyComparisonOp::Eq => return eq(vm, zelf, other), - PyComparisonOp::Ne => return eq(vm, zelf, other).map(|eq| !eq), - PyComparisonOp::Lt | PyComparisonOp::Le => true, - PyComparisonOp::Gt | PyComparisonOp::Ge => false, - }; - let (lhs_len, rhs_len) = (zelf.len(), other.len()); - for (a, b) in Iterator::zip(zelf, other) { - let ret = if less { - vm.bool_seq_lt(a, b)? - } else { - vm.bool_seq_gt(a, b)? + fn cmp(&'a self, vm: &VirtualMachine, other: &'a Self, op: PyComparisonOp) -> PyResult { + let less = match op { + PyComparisonOp::Eq => return self.eq(vm, other), + PyComparisonOp::Ne => return self.eq(vm, other).map(|eq| !eq), + PyComparisonOp::Lt | PyComparisonOp::Le => true, + PyComparisonOp::Gt | PyComparisonOp::Ge => false, }; - if let Some(v) = ret { - return Ok(v); - } - } - Ok(op.eval_ord(lhs_len.cmp(&rhs_len))) -} -pub(crate) struct SeqMul<'a> { - seq: &'a dyn SimpleSeq, - repetitions: usize, - iter: Option>, -} - -impl ExactSizeIterator for SeqMul<'_> {} - -impl<'a> Iterator for SeqMul<'a> { - type Item = &'a PyObjectRef; - fn next(&mut self) -> Option { - match self.iter.as_mut().and_then(Iterator::next) { - Some(item) => Some(item), - None => { - if self.repetitions == 0 { - None - } else { - self.repetitions -= 1; - self.iter = Some(self.seq.boxed_iter()); - self.next() - } + let lhs = self.iter(); + let rhs = other.iter(); + let lhs_len = lhs.len(); + let rhs_len = rhs.len(); + for (a, b) in lhs.zip(rhs) { + let ret = if less { + vm.bool_seq_lt(a, b)? + } else { + vm.bool_seq_gt(a, b)? + }; + if let Some(v) = ret { + return Ok(v); } } - } - fn size_hint(&self) -> (usize, Option) { - let size = self.iter.as_ref().map_or(0, ExactSizeIterator::len) - + (self.repetitions * self.seq.len()); - (size, Some(size)) + Ok(op.eval_ord(lhs_len.cmp(&rhs_len))) } } -pub(crate) fn seq_mul<'a>( - vm: &VirtualMachine, - seq: &'a impl SimpleSeq, - repetitions: isize, -) -> PyResult> { - vm.check_repeat_or_memory_error(seq.len(), repetitions) - .map(|repetitions| SeqMul { - seq, - repetitions, - iter: None, - }) +impl<'a> ObjectSequenceOp<'a> for [PyObjectRef] { + type Iter = core::slice::Iter<'a, PyObjectRef>; + + fn iter(&'a self) -> Self::Iter { + self.iter() + } +} + +impl<'a> ObjectSequenceOp<'a> for VecDeque { + type Iter = std::collections::vec_deque::Iter<'a, PyObjectRef>; + + fn iter(&'a self) -> Self::Iter { + self.iter() + } +} + +pub trait MutObjectSequenceOp<'a> { + type Guard; + + fn do_get(index: usize, guard: &Self::Guard) -> Option<&PyObjectRef>; + fn do_lock(&'a self) -> Self::Guard; + + fn mut_count(&'a self, vm: &VirtualMachine, needle: &PyObject) -> PyResult { + let mut count = 0; + self._mut_iter_equal_skeleton::<_, false>(vm, needle, 0..isize::MAX as usize, || { + count += 1 + })?; + Ok(count) + } + + fn mut_index_range( + &'a self, + vm: &VirtualMachine, + needle: &PyObject, + range: Range, + ) -> PyResult> { + self._mut_iter_equal_skeleton::<_, true>(vm, needle, range, || {}) + } + + fn mut_index(&'a self, vm: &VirtualMachine, needle: &PyObject) -> PyResult> { + self.mut_index_range(vm, needle, 0..isize::MAX as usize) + } + + fn mut_contains(&'a self, vm: &VirtualMachine, needle: &PyObject) -> PyResult { + self.mut_index(vm, needle).map(|x| x.is_some()) + } + + fn _mut_iter_equal_skeleton( + &'a self, + vm: &VirtualMachine, + needle: &PyObject, + range: Range, + mut f: F, + ) -> PyResult> + where + F: FnMut(), + { + let needle_cls = needle.class(); + let needle_cmp = needle_cls + .mro_find_map(|cls| cls.slots.richcompare.load()) + .unwrap(); + + let mut borrower = None; + let mut i = range.start; + + let index = loop { + if i >= range.end { + break Optioned::::none(); + } + let guard = if let Some(x) = borrower.take() { + x + } else { + self.do_lock() + }; + + let elem = if let Some(x) = Self::do_get(i, &guard) { + x + } else { + break Optioned::::none(); + }; + + if elem.is(needle) { + f(); + if SHORT { + break Optioned::::some(i); + } + borrower = Some(guard); + } else { + let elem_cls = elem.class(); + let reverse_first = !elem_cls.is(&needle_cls) && elem_cls.issubclass(&needle_cls); + + let eq = if reverse_first { + let elem_cmp = elem_cls + .mro_find_map(|cls| cls.slots.richcompare.load()) + .unwrap(); + drop(elem_cls); + + fn cmp( + elem: &PyObject, + needle: &PyObject, + elem_cmp: RichCompareFunc, + needle_cmp: RichCompareFunc, + vm: &VirtualMachine, + ) -> PyResult { + match elem_cmp(elem, needle, PyComparisonOp::Eq, vm)? { + Either::B(PyComparisonValue::Implemented(value)) => Ok(value), + Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { + obj.try_to_bool(vm) + } + _ => match needle_cmp(needle, elem, PyComparisonOp::Eq, vm)? { + Either::B(PyComparisonValue::Implemented(value)) => Ok(value), + Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { + obj.try_to_bool(vm) + } + _ => Ok(false), + }, + } + } + + if elem_cmp as usize == richcompare_wrapper as usize { + let elem = elem.clone(); + drop(guard); + cmp(&elem, needle, elem_cmp, needle_cmp, vm)? + } else { + let eq = cmp(elem, needle, elem_cmp, needle_cmp, vm)?; + borrower = Some(guard); + eq + } + } else { + match needle_cmp(needle, elem, PyComparisonOp::Eq, vm)? { + Either::B(PyComparisonValue::Implemented(value)) => { + drop(elem_cls); + borrower = Some(guard); + value + } + Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { + drop(elem_cls); + borrower = Some(guard); + obj.try_to_bool(vm)? + } + _ => { + let elem_cmp = elem_cls + .mro_find_map(|cls| cls.slots.richcompare.load()) + .unwrap(); + drop(elem_cls); + + fn cmp( + elem: &PyObject, + needle: &PyObject, + elem_cmp: RichCompareFunc, + vm: &VirtualMachine, + ) -> PyResult { + match elem_cmp(elem, needle, PyComparisonOp::Eq, vm)? { + Either::B(PyComparisonValue::Implemented(value)) => Ok(value), + Either::A(obj) if !obj.is(&vm.ctx.not_implemented) => { + obj.try_to_bool(vm) + } + _ => Ok(false), + } + } + + if elem_cmp as usize == richcompare_wrapper as usize { + let elem = elem.clone(); + drop(guard); + cmp(&elem, needle, elem_cmp, vm)? + } else { + let eq = cmp(elem, needle, elem_cmp, vm)?; + borrower = Some(guard); + eq + } + } + } + }; + + if eq { + f(); + if SHORT { + break Optioned::::some(i); + } + } + } + i += 1; + }; + + Ok(index) + } +} + +pub trait SequenceOp { + fn as_slice(&self) -> &[T]; + + fn mul(&self, vm: &VirtualMachine, n: isize) -> PyResult> { + let n = vm.check_repeat_or_overflow_error(self.as_slice().len(), n)?; + let mut v = Vec::with_capacity(n * self.as_slice().len()); + for _ in 0..n { + v.extend_from_slice(self.as_slice()); + } + Ok(v) + } +} + +impl SequenceOp for [T] { + fn as_slice(&self) -> &[T] { + self + } +} + +pub trait SequenceMutOp { + fn as_slice(&self) -> &[T]; + fn as_vec_mut(&mut self) -> &mut Vec; + + fn imul(&mut self, vm: &VirtualMachine, n: isize) -> PyResult<()> { + let n = vm.check_repeat_or_overflow_error(self.as_slice().len(), n)?; + if n == 0 { + self.as_vec_mut().clear(); + } else if n != 1 { + let mut sample = self.as_slice().to_vec(); + if n != 2 { + self.as_vec_mut().reserve(sample.len() * (n - 1)); + for _ in 0..n - 2 { + self.as_vec_mut().extend_from_slice(&sample); + } + } + self.as_vec_mut().append(&mut sample); + } + Ok(()) + } +} + +impl SequenceMutOp for Vec { + fn as_slice(&self) -> &[T] { + self.as_slice() + } + + fn as_vec_mut(&mut self) -> &mut Vec { + self + } } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 4dace44c4..630aaa772 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -214,7 +214,7 @@ mod builtins { let (globals, locals) = match self.globals { Some(globals) => { if !globals.contains_key("__builtins__", vm) { - let builtins_dict = vm.builtins.dict().unwrap().into(); + let builtins_dict = vm.builtins.dict().into(); globals.set_item("__builtins__", builtins_dict, vm)?; } ( diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index 5565596bc..5a3da10fa 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -2,16 +2,17 @@ pub(crate) use _collections::make_module; #[pymodule] mod _collections { - use crate::builtins::{PositionIterInternal, PyGenericAlias}; - use crate::common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::{ builtins::{ IterStatus::{Active, Exhausted}, - PyInt, PyTypeRef, + PositionIterInternal, PyGenericAlias, PyInt, PyTypeRef, }, + common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, function::{FuncArgs, KwArgs, OptionalArg}, protocol::PyIterReturn, - sequence, sliceable, + sequence::{MutObjectSequenceOp, ObjectSequenceOp}, + sliceable, + sliceable::saturate_index, types::{ Comparable, Constructor, Hashable, IterNext, IterNextIterable, Iterable, PyComparisonOp, Unhashable, @@ -22,7 +23,7 @@ mod _collections { }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; - use std::cmp::{max, min}; + use std::cmp::max; use std::collections::VecDeque; #[pyattr] @@ -54,24 +55,6 @@ mod _collections { } } - struct SimpleSeqDeque<'a>(PyRwLockReadGuard<'a, VecDeque>); - - impl sequence::SimpleSeq for SimpleSeqDeque<'_> { - fn len(&self) -> usize { - self.0.len() - } - - fn boxed_iter(&self) -> sequence::DynPyIter { - Box::new(self.0.iter()) - } - } - - impl<'a> From>> for SimpleSeqDeque<'a> { - fn from(from: PyRwLockReadGuard<'a, VecDeque>) -> Self { - Self(from) - } - } - #[pyimpl(flags(BASETYPE), with(Comparable, Hashable, Iterable))] impl PyDeque { #[pyslot] @@ -178,17 +161,11 @@ mod _collections { #[pymethod] fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut count = 0; let start_state = self.state.load(); - let deque = self.borrow_deque().clone(); - for elem in deque.iter() { - if vm.identical_or_equal(elem, &obj)? { - count += 1; - } + let count = self.mut_count(vm, &obj)?; - if start_state != self.state.load() { - return Err(vm.new_runtime_error("deque mutated during iteration".to_owned())); - } + if start_state != self.state.load() { + return Err(vm.new_runtime_error("deque mutated during iteration".to_owned())); } Ok(count) } @@ -235,14 +212,6 @@ mod _collections { Ok(()) } - fn adjust_negative_index(&self, index: isize) -> usize { - if index.is_negative() { - max(index + self.borrow_deque().len() as isize, 0) as usize - } else { - index as usize - } - } - #[pymethod] fn index( &self, @@ -251,35 +220,25 @@ mod _collections { stop: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let deque = self.borrow_deque().clone(); let start_state = self.state.load(); - let start = self.adjust_negative_index(start.unwrap_or(0)); - let stop = min( - self.adjust_negative_index(stop.unwrap_or_else(|| deque.len() as isize)), - deque.len(), - ); - - for (i, elem) in deque - .iter() - .skip(start) - .take(stop.saturating_sub(start)) - .enumerate() - { - let is_element = vm.identical_or_equal(elem, &obj)?; - - if start_state != self.state.load() { - return Err(vm.new_runtime_error("deque mutated during iteration".to_owned())); - } - if is_element { - return Ok(i + start); - } + let len = self.len(); + let start = start.map(|i| saturate_index(i, len)).unwrap_or(0); + let stop = stop + .map(|i| saturate_index(i, len)) + .unwrap_or(isize::MAX as usize); + let index = self.mut_index_range(vm, &obj, start..stop)?; + if start_state != self.state.load() { + Err(vm.new_runtime_error("deque mutated during iteration".to_owned())) + } else if let Some(index) = index.into() { + Ok(index) + } else { + Err(vm.new_value_error( + obj.repr(vm) + .map(|repr| format!("{} is not in deque", repr)) + .unwrap_or_else(|_| String::new()), + )) } - Err(vm.new_value_error( - obj.repr(vm) - .map(|repr| format!("{} is not in deque", repr)) - .unwrap_or_else(|_| String::new()), - )) } #[pymethod] @@ -326,23 +285,17 @@ mod _collections { #[pymethod] fn remove(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let deque = self.borrow_deque().clone(); let start_state = self.state.load(); + let index = self.mut_index(vm, &obj)?; - let mut idx = None; - for (i, elem) in deque.iter().enumerate() { - if vm.identical_or_equal(elem, &obj)? { - idx = Some(i); - break; - } - } - let mut deque = self.borrow_deque_mut(); if start_state != self.state.load() { Err(vm.new_index_error("deque mutated during remove().".to_owned())) - } else { + } else if let Some(index) = index.into() { + let mut deque = self.borrow_deque_mut(); self.state.fetch_add(1); - idx.map(|idx| deque.remove(idx).unwrap()) - .ok_or_else(|| vm.new_value_error("deque.remove(x): x not in deque".to_owned())) + Ok(deque.remove(index).unwrap()) + } else { + Err(vm.new_value_error("deque.remove(x): x not in deque".to_owned())) } } @@ -431,32 +384,32 @@ mod _collections { #[pymethod(magic)] fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let start_state = self.state.load(); - let deque = self.borrow_deque().clone(); - for element in deque.iter() { - let is_element = vm.identical_or_equal(element, &needle)?; - - if start_state != self.state.load() { - return Err(vm.new_runtime_error("deque mutated during iteration".to_owned())); - } - if is_element { - return Ok(true); - } + let ret = self.mut_contains(vm, &needle)?; + if start_state != self.state.load() { + Err(vm.new_runtime_error("deque mutated during iteration".to_owned())) + } else { + Ok(ret) } + } - Ok(false) + fn _mul(&self, n: isize, vm: &VirtualMachine) -> PyResult> { + let deque = self.borrow_deque(); + let n = vm.check_repeat_or_overflow_error(deque.len(), n)?; + let mul_len = n * deque.len(); + let iter = deque.iter().cycle().take(mul_len); + let skipped = self + .maxlen + .and_then(|maxlen| mul_len.checked_sub(maxlen)) + .unwrap_or(0); + + let deque = iter.skip(skipped).cloned().collect(); + Ok(deque) } #[pymethod(magic)] #[pymethod(name = "__rmul__")] - fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult { - let deque: SimpleSeqDeque = self.borrow_deque().into(); - let mul = sequence::seq_mul(vm, &deque, value)?; - let skipped = self - .maxlen - .and_then(|maxlen| mul.len().checked_sub(maxlen)) - .unwrap_or(0); - - let deque = mul.skip(skipped).cloned().collect(); + fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult { + let deque = self._mul(n, vm)?; Ok(PyDeque { deque: PyRwLock::new(deque), maxlen: self.maxlen, @@ -465,12 +418,9 @@ mod _collections { } #[pymethod(magic)] - fn imul(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { - let mul_deque = zelf.mul(value, vm)?; - std::mem::swap( - &mut *zelf.borrow_deque_mut(), - &mut *mul_deque.borrow_deque_mut(), - ); + fn imul(zelf: PyRef, n: isize, vm: &VirtualMachine) -> PyResult> { + let mul_deque = zelf._mul(n, vm)?; + *zelf.borrow_deque_mut() = mul_deque; Ok(zelf) } @@ -536,6 +486,18 @@ mod _collections { } } + impl<'a> MutObjectSequenceOp<'a> for PyDeque { + type Guard = PyRwLockReadGuard<'a, VecDeque>; + + fn do_get(index: usize, guard: &Self::Guard) -> Option<&PyObjectRef> { + guard.get(index) + } + + fn do_lock(&'a self) -> Self::Guard { + self.borrow_deque() + } + } + impl Comparable for PyDeque { fn cmp( zelf: &crate::PyObjectView, @@ -547,9 +509,9 @@ mod _collections { return Ok(res.into()); } let other = class_or_notimplemented!(Self, other); - let (lhs, rhs) = (zelf.borrow_deque(), other.borrow_deque()); - sequence::cmp(vm, Box::new(lhs.iter()), Box::new(rhs.iter()), op) - .map(PyComparisonValue::Implemented) + let lhs = zelf.borrow_deque(); + let rhs = other.borrow_deque(); + lhs.cmp(vm, &rhs, op).map(PyComparisonValue::Implemented) } } diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index e58112743..8b611540a 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -638,7 +638,7 @@ mod decl { } .into_ref(vm); - state.grouper = Some(grouper.downgrade()); + state.grouper = Some(grouper.downgrade(None, vm).unwrap()); Ok(PyIterReturn::Return( (state.current_key.as_ref().unwrap().clone(), grouper).into_pyobject(vm), )) diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 7530a303c..bec416b06 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -960,21 +960,9 @@ pub(crate) mod _struct { #[pyfunction] fn _clearcache() {} - #[pyattr(name = "error")] + #[pyattr(name = "error", once)] fn error_type(vm: &VirtualMachine) -> PyTypeRef { - rustpython_common::static_cell! { - static STRUCT_ERROR: PyTypeRef; - } - STRUCT_ERROR - .get_or_init(|| { - vm.ctx.new_class( - Some("struct"), - "error", - &vm.ctx.exceptions.exception_type, - Default::default(), - ) - }) - .clone() + vm.ctx.new_exception_type("struct", "error", None) } fn new_struct_error(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index e13aadfb3..ffeb03628 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -8,7 +8,8 @@ pub(crate) use _weakref::make_module; #[pymodule] mod _weakref { - use crate::{builtins::PyTypeRef, PyObjectRef, VirtualMachine}; + use crate::builtins::{PyDictRef, PyTypeRef, PyWeak}; + use crate::{PyObjectRef, PyResult, VirtualMachine}; #[pyattr(name = "ref")] fn ref_(vm: &VirtualMachine) -> PyTypeRef { @@ -33,17 +34,30 @@ mod _weakref { #[pyfunction] fn getweakrefcount(obj: PyObjectRef) -> usize { - obj.weak_count() + obj.weak_count().unwrap_or(0) } #[pyfunction] - fn getweakrefs(_obj: PyObjectRef) -> Vec { - // TODO: implement this, may require a different gc - Vec::new() + fn getweakrefs(obj: PyObjectRef) -> Vec { + match obj.get_weak_references() { + Some(v) => v.into_iter().map(|weak| weak.into_object()).collect(), + None => vec![], + } } #[pyfunction] - fn _remove_dead_weakref(_obj: PyObjectRef, _key: PyObjectRef) { - // TODO + fn _remove_dead_weakref( + dict: PyDictRef, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + dict._as_dict_inner() + .delete_if(vm, &key, |wr| { + let wr = wr + .payload::() + .ok_or_else(|| vm.new_type_error("not a weakref".to_owned()))?; + Ok(wr.is_dead()) + }) + .map(drop) } } diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index 1a9c21e63..d5fb06800 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -150,6 +150,8 @@ pub(crate) type DescrSetFunc = pub(crate) type NewFunc = fn(PyTypeRef, FuncArgs, &VirtualMachine) -> PyResult; pub(crate) type DelFunc = fn(&PyObject, &VirtualMachine) -> PyResult<()>; +pub use crate::builtins::object::{generic_getattr, generic_setattr}; + fn as_mapping_wrapper(zelf: &PyObject, _vm: &VirtualMachine) -> PyMappingMethods { macro_rules! then_some_closure { ($cond:expr, $closure:expr) => { diff --git a/vm/src/types/zoo.rs b/vm/src/types/zoo.rs index f780343f4..df3e048a8 100644 --- a/vm/src/types/zoo.rs +++ b/vm/src/types/zoo.rs @@ -85,11 +85,12 @@ pub struct TypeZoo { impl TypeZoo { pub(crate) fn init() -> Self { - let (type_type, object_type) = crate::pyobjectrc::init_type_hierarchy(); + let (type_type, object_type, weakref_type) = crate::pyobjectrc::init_type_hierarchy(); Self { - // the order matters for type, object and int + // the order matters for type, object, weakref, and int type_type: pytype::PyType::init_manually(type_type).clone(), object_type: object::PyBaseObject::init_manually(object_type).clone(), + weakref_type: weakref::PyWeak::init_manually(weakref_type).clone(), int_type: int::PyInt::init_bare_type().clone(), // types exposed as builtins @@ -162,7 +163,6 @@ impl TypeZoo { traceback_type: traceback::PyTraceback::init_bare_type().clone(), tuple_iterator_type: tuple::PyTupleIterator::init_bare_type().clone(), weakproxy_type: weakproxy::PyWeakProxy::init_bare_type().clone(), - weakref_type: weakref::PyWeak::init_bare_type().clone(), method_descriptor_type: builtinfunc::PyBuiltinMethod::init_bare_type().clone(), none_type: singletons::PyNone::init_bare_type().clone(), not_implemented_type: singletons::PyNotImplemented::init_bare_type().clone(), diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 5c4d27ad1..dbf7acf03 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -9,7 +9,7 @@ use crate::compile::{self, CompileError, CompileErrorType, CompileOpts}; use crate::{ builtins::{ code::{self, PyCode}, - module, object, + object, pystr::IntoPyStrRef, tuple::{IntoPyTuple, PyTuple, PyTupleRef, PyTupleTyped}, PyBaseException, PyBaseExceptionRef, PyDictRef, PyInt, PyIntRef, PyList, PyModule, PyStr, @@ -26,9 +26,8 @@ use crate::{ scope::Scope, signal, stdlib, types::PyComparisonOp, - IdProtocol, ItemProtocol, PyArithmeticValue, PyContext, PyGenericObject, PyLease, PyMethod, - PyObject, PyObjectRef, PyObjectWrap, PyRef, PyRefExact, PyResult, PyValue, TryFromObject, - TypeProtocol, + IdProtocol, ItemProtocol, PyArithmeticValue, PyContext, PyLease, PyMethod, PyObject, + PyObjectRef, PyObjectWrap, PyRef, PyRefExact, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crossbeam_utils::atomic::AtomicCell; use num_traits::{Signed, ToPrimitive}; @@ -46,8 +45,8 @@ use std::fmt; /// /// To construct this, please refer to the [`Interpreter`](Interpreter) pub struct VirtualMachine { - pub builtins: PyObjectRef, - pub sys_module: PyObjectRef, + pub builtins: PyRef, + pub sys_module: PyRef, pub ctx: PyRc, pub frames: RefCell>, pub wasm_id: Option, @@ -248,14 +247,17 @@ impl VirtualMachine { // make a new module without access to the vm; doesn't // set __spec__, __loader__, etc. attributes - let new_module = - |dict| PyGenericObject::new(PyModule {}, ctx.types.module_type.clone(), Some(dict)); + let new_module = || { + PyRef::new_ref( + PyModule {}, + ctx.types.module_type.clone(), + Some(ctx.new_dict()), + ) + }; // Hard-core modules: - let builtins_dict = ctx.new_dict(); - let builtins = new_module(builtins_dict.clone()); - let sysmod_dict = ctx.new_dict(); - let sysmod = new_module(sysmod_dict.clone()); + let builtins = new_module(); + let sys_module = new_module(); let import_func = ctx.none(); let profile_func = RefCell::new(ctx.none()); @@ -279,7 +281,7 @@ impl VirtualMachine { let mut vm = VirtualMachine { builtins, - sys_module: sysmod, + sys_module, ctx: PyRc::new(ctx), frames: RefCell::new(vec![]), wasm_id: None, @@ -309,18 +311,14 @@ impl VirtualMachine { let frozen = frozen::map_frozen(&vm, frozen::get_module_inits()).collect(); PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen; - module::init_module_dict( - &vm, - &builtins_dict, + vm.builtins.init_module_dict( vm.ctx.new_str(ascii!("builtins")).into(), vm.ctx.none(), - ); - module::init_module_dict( &vm, - &sysmod_dict, - vm.ctx.new_str(ascii!("sys")).into(), - vm.ctx.none(), ); + vm.sys_module + .init_module_dict(vm.ctx.new_str(ascii!("sys")).into(), vm.ctx.none(), &vm); + vm } @@ -331,8 +329,8 @@ impl VirtualMachine { panic!("Double Initialize Error"); } - stdlib::builtins::make_module(self, self.builtins.clone()); - stdlib::sys::init_module(self, &self.sys_module, &self.builtins); + stdlib::builtins::make_module(self, self.builtins.clone().into()); + stdlib::sys::init_module(self, self.sys_module.as_ref(), self.builtins.as_ref()); let mut inner_init = || -> PyResult<()> { #[cfg(not(target_arch = "wasm32"))] @@ -511,8 +509,7 @@ impl VirtualMachine { } pub fn run_code_obj(&self, code: PyRef, scope: Scope) -> PyResult { - let frame = - Frame::new(code, scope, self.builtins.dict().unwrap(), &[], self).into_ref(self); + let frame = Frame::new(code, scope, self.builtins.dict(), &[], self).into_ref(self); self.run_frame_full(frame) } @@ -624,14 +621,14 @@ impl VirtualMachine { } pub fn new_module(&self, name: &str, dict: PyDictRef, doc: Option<&str>) -> PyObjectRef { - module::init_module_dict( - self, - &dict, + let module = PyRef::new_ref(PyModule {}, self.ctx.types.module_type.clone(), Some(dict)); + module.init_module_dict( self.new_pyobj(name.to_owned()), doc.map(|doc| self.new_pyobj(doc.to_owned())) .unwrap_or_else(|| self.ctx.none()), + self, ); - PyGenericObject::new(PyModule {}, self.ctx.types.module_type.clone(), Some(dict)) + module.into_object() } /// Instantiate an exception with arguments. @@ -1748,13 +1745,16 @@ impl VirtualMachine { /// Checks that the multiplication is able to be performed. On Ok returns the /// index as a usize for sequences to be able to use immediately. - pub fn check_repeat_or_memory_error(&self, length: usize, n: isize) -> PyResult { - let n = n.to_usize().unwrap_or(0); - if n > 0 && length > stdlib::sys::MAXSIZE as usize / n { - // Empty message is currently used in CPython. - Err(self.new_memory_error("".to_owned())) + pub fn check_repeat_or_overflow_error(&self, length: usize, n: isize) -> PyResult { + if n <= 0 { + Ok(0) } else { - Ok(n) + let n = n as usize; + if length > stdlib::sys::MAXSIZE as usize / n { + Err(self.new_overflow_error("repeated value are too long".to_owned())) + } else { + Ok(n) + } } } @@ -1898,7 +1898,7 @@ impl VirtualMachine { attr_value: impl Into, ) -> PyResult<()> { let val = attr_value.into(); - object::setattr(module, attr_name.into_pystr_ref(self), Some(val), self) + object::generic_setattr(module, attr_name.into_pystr_ref(self), Some(val), self) } } @@ -1939,7 +1939,7 @@ impl<'vm> ReprGuard<'vm> { let mut guards = vm.repr_guards.borrow_mut(); // Should this be a flag on the obj itself? putting it in a global variable for now until it - // decided the form of the PyGenericObject. https://github.com/RustPython/RustPython/issues/371 + // decided the form of PyObject. https://github.com/RustPython/RustPython/issues/371 let id = obj.get_id(); if guards.contains(&id) { return None; diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 49996bd0f..2a4f1ae62 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -10,7 +10,7 @@ mod _browser { builtins::{PyDictRef, PyStrRef}, function::{ArgCallable, IntoPyObject, OptionalArg}, import::import_file, - PyClassImpl, PyGenericObject, PyObjectRef, PyResult, PyValue, VirtualMachine, + PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, VirtualMachine, }; use wasm_bindgen::{prelude::*, JsCast}; use wasm_bindgen_futures::JsFuture; @@ -179,8 +179,8 @@ mod _browser { } #[pyattr] - fn document(vm: &VirtualMachine) -> PyObjectRef { - PyGenericObject::new( + fn document(vm: &VirtualMachine) -> PyRef { + PyRef::new_ref( Document { doc: window().document().expect("Document missing from window"), }, diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index b90276901..c71c4e1ec 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -84,7 +84,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { let wasm_vm = WASMVirtualMachine { id: wasm_id.clone(), }; - let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap(); + let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); let closure = move |args: Option>, kwargs: Option| diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index 8617e6a90..a3273c4f1 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -123,12 +123,12 @@ mod _js { } #[pymethod] - fn new_closure(&self, obj: PyObjectRef, vm: &VirtualMachine) -> JsClosure { + fn new_closure(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { JsClosure::new(obj, false, vm) } #[pymethod] - fn new_closure_once(&self, obj: PyObjectRef, vm: &VirtualMachine) -> JsClosure { + fn new_closure_once(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { JsClosure::new(obj, true, vm) } @@ -306,11 +306,11 @@ mod _js { #[pyimpl] impl JsClosure { - fn new(obj: PyObjectRef, once: bool, vm: &VirtualMachine) -> Self { + fn new(obj: PyObjectRef, once: bool, vm: &VirtualMachine) -> PyResult { let wasm_vm = WASMVirtualMachine { id: vm.wasm_id.clone().unwrap(), }; - let weak_py_obj = wasm_vm.push_held_rc(obj).unwrap(); + let weak_py_obj = wasm_vm.push_held_rc(obj).unwrap()?; let f = move |this: JsValue, args: Box<[JsValue]>| { let py_obj = match wasm_vm.assert_valid() { Ok(_) => weak_py_obj @@ -337,11 +337,11 @@ mod _js { Closure::once(Box::new(f)) }; let wrapped = PyJsValue::new(wrap_closure(closure.as_ref())).into_ref(vm); - JsClosure { + Ok(JsClosure { closure: Some((closure, wrapped)).into(), destroyed: false.into(), detached: false.into(), - } + }) } #[pyproperty] diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 0dc45fa9f..67b0d7582 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -8,7 +8,7 @@ use wasm_bindgen::prelude::*; use rustpython_vm::compile::{self, Mode}; use rustpython_vm::scope::Scope; use rustpython_vm::{InitParameter, Interpreter, PySettings, VirtualMachine}; -use rustpython_vm::{ItemProtocol, PyObjectRef, PyObjectWeak, PyValue}; +use rustpython_vm::{ItemProtocol, PyObjectRef, PyObjectWeak, PyResult, PyValue}; use crate::browser_module::setup_browser_module; use crate::convert::{self, PyResultExt}; @@ -195,11 +195,11 @@ impl WASMVirtualMachine { STORED_VMS.with(|cell| cell.borrow().contains_key(&self.id)) } - pub(crate) fn push_held_rc(&self, obj: PyObjectRef) -> Result { - self.with(|stored_vm| { - let weak = obj.downgrade(); + pub(crate) fn push_held_rc(&self, obj: PyObjectRef) -> Result, JsValue> { + self.with_vm(|vm, stored_vm| { + let weak = obj.downgrade(None, vm)?; stored_vm.held_objects.borrow_mut().push(obj); - weak + Ok(weak) }) }