diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index edea5bc05..17ef357c0 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -232,6 +232,33 @@ class ExceptionTests(unittest.TestCase): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) + def testSyntaxErrorRange(self): + # gh-156894: the position was reported in bytes, not in characters, + # for the errors which cover a range + for source, offset, end_offset in [ + ('abcd = 00010', 8, 11), + ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11), + ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15), + ("abcd = ub'a'", 8, 10), + ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 8, 10), + ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 12, 14), + ]: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual(cm.exception.offset, offset) + self.assertEqual(cm.exception.end_offset, end_offset) + + def testSyntaxErrorNonUTF8Offset(self): + # gh-157378: the position was reported one column short for each + # multi-byte character preceding the invalid byte on the same line + check = self.check + check(b'X\x80', 1, 2, 1, 2) + check(b'\xc3\xa9X\x80', 1, 3, 1, 3) + check(b'\t\xc3\xa9X\x80', 1, 4, 1, 4) + check(b'a\xc3\xa9b\x80c', 1, 4, 1, 4) + check(b'a\n\xc3\xa9X\x80', 2, 3, 2, 3) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) @@ -252,7 +279,16 @@ class ExceptionTests(unittest.TestCase): check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5) check('[file for\n str(file) in []]', 2, 2) check("ages = {'Alice'=22, 'Bob'=23}", 1, 9) - check('match ...:\n case {**rest, "key": value}:\n ...', 2, 19) + check(dedent("""\ + match ...: + case {**rest1, "after": after}: + ... + """), 2, 11) + check(dedent("""\ + match ...: + case {"before": before, **rest2, "after": after}: + ... + """), 2, 29) check("[a b c d e f]", 1, 2) check("for x yfff:", 1, 7) check("f(a for a in b, c)", 1, 3, 1, 15) @@ -328,7 +364,6 @@ class ExceptionTests(unittest.TestCase): check('x=1\nfrom __future__ import division', 2, 1) check('foo(1=2)', 1, 5) check('def f():\n x, y: int', 2, 3) - check('[*x for x in xs]', 1, 2) check('foo(x for x in range(10), 100)', 1, 5) check('for 1 in []: pass', 1, 5) check('(yield i) = 2', 1, 2) @@ -1510,6 +1545,7 @@ class ExceptionTests(unittest.TestCase): @cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") @force_not_colorized + @support.skip_if_huge_c_stack() def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1576,11 +1612,7 @@ class ExceptionTests(unittest.TestCase): sys.setrecursionlimit(recursionlimit) - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') - @unittest.skipIf(_testcapi is None, "requires _testcapi") + @support.nomemtest def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1707,6 +1739,20 @@ class ExceptionTests(unittest.TestCase): gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) + def test_oserror_reinit_leak(self): + # gh-150988: Check for memory leak when re-initializing OSError. + # Previously, setting OSError attributes in a subclass + # before calling super().__init__() leaked memory. + class LeakingOSError(OSError): + def __init__(self, code, message, filename, filename2): + self.strerror = message + self.filename = filename + self.filename2 = filename2 + super().__init__(code, message, filename, None, filename2) + + exc = LeakingOSError(1, "some message", "filename.py", "filename2.py") + exc.__init__(2, "another message", "filename3.py", "filename4.py") + def test_errno_ENOTDIR(self): # Issue #12802: "not a directory" errors are ENOTDIR even on Windows with self.assertRaises(OSError) as cm: @@ -1753,11 +1799,7 @@ class ExceptionTests(unittest.TestCase): self.assertIn("test message", report) self.assertEndsWith(report, "\n") - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') - @unittest.skipIf(_testcapi is None, "requires _testcapi") + @support.nomemtest def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1915,12 +1957,8 @@ class ExceptionTests(unittest.TestCase): exc2 = None - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @support.nomemtest def test_exec_set_nomemory_hang(self): - import_module("_testcapi") # gh-134163: A MemoryError inside code that was wrapped by a try/except # block would lead to an infinite loop. @@ -2117,6 +2155,50 @@ class ImportErrorTests(unittest.TestCase): self.assertEqual(exc.name, orig.name) self.assertEqual(exc.path, orig.path) + def test_repr(self): + exc = ImportError() + self.assertEqual(repr(exc), "ImportError()") + + exc = ImportError('test') + self.assertEqual(repr(exc), "ImportError('test')") + + exc = ImportError('test', 'case') + self.assertEqual(repr(exc), "ImportError('test', 'case')") + + exc = ImportError(name='somemodule') + self.assertEqual(repr(exc), "ImportError(name='somemodule')") + + exc = ImportError('test', name='somemodule') + self.assertEqual(repr(exc), "ImportError('test', name='somemodule')") + + exc = ImportError(path='somepath') + self.assertEqual(repr(exc), "ImportError(path='somepath')") + + exc = ImportError('test', path='somepath') + self.assertEqual(repr(exc), "ImportError('test', path='somepath')") + + exc = ImportError(name='somename', path='somepath') + self.assertEqual(repr(exc), + "ImportError(name='somename', path='somepath')") + + exc = ImportError('test', name='somename', path='somepath') + self.assertEqual(repr(exc), + "ImportError('test', name='somename', path='somepath')") + + exc = ModuleNotFoundError('test', name='somename', path='somepath') + self.assertEqual(repr(exc), + "ModuleNotFoundError('test', name='somename', path='somepath')") + + def test_ModuleNotFoundError_repr_with_failed_import(self): + with self.assertRaises(ModuleNotFoundError) as cm: + import does_not_exist # type: ignore[import] # noqa: F401 + + self.assertEqual(cm.exception.name, "does_not_exist") + self.assertIsNone(cm.exception.path) + + self.assertEqual(repr(cm.exception), + "ModuleNotFoundError(\"No module named 'does_not_exist'\", name='does_not_exist')") + def run_script(source): if isinstance(source, str): @@ -2373,7 +2455,6 @@ class SyntaxErrorTests(unittest.TestCase): ^^^^^ """, err.getvalue()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_encodings(self): self.addCleanup(unlink, TESTFN) source = ( @@ -2382,7 +2463,8 @@ class SyntaxErrorTests(unittest.TestCase): ) err = run_script(source.encode('cp437')) self.assertEqual(err[-3], ' "¢¢¢¢¢¢" + f(4, x for x in range(1))') - self.assertEqual(err[-2], ' ^^^') + self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') + self.assertEqual(err[-1], 'SyntaxError: Generator expression must be parenthesized') # Check backwards tokenizer errors source = '# -*- coding: ascii -*-\n\n(\n' diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 6f49e62e8..4fe19765c 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -134,6 +134,22 @@ Dict display element unpacking ... TypeError: 'list' object is not a mapping + >>> class OnlyKeys: + ... def keys(self): + ... return ['key'] + >>> {**OnlyKeys()} + Traceback (most recent call last): + ... + TypeError: 'OnlyKeys' object is not subscriptable + + >>> class BrokenKeys: + ... def keys(self): + ... return 1 + >>> {**BrokenKeys()} + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.BrokenKeys.keys() must return an iterable, not int + >>> len(eval("{" + ", ".join("**{{{}: {}}}".format(i, i) ... for i in range(1000)) + "}")) 1000 @@ -141,7 +157,7 @@ Dict display element unpacking >>> {0:1, **{0:2}, 0:3, 0:4} {0: 4} -List comprehension element unpacking +Comprehension element unpacking >>> a, b, c = [0, 1, 2], 3, 4 >>> [*a, b, c] @@ -149,46 +165,220 @@ List comprehension element unpacking >>> l = [a, (3, 4), {5}, {6: None}, (i for i in range(7, 10))] >>> [*item for item in l] - Traceback (most recent call last): - ... - SyntaxError: iterable unpacking cannot be used in comprehension + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> [*[0, 1] for i in range(10)] - Traceback (most recent call last): - ... - SyntaxError: iterable unpacking cannot be used in comprehension + >>> [*[0, 1] for i in range(5)] + [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] - >>> [*'a' for i in range(10)] - Traceback (most recent call last): - ... - SyntaxError: iterable unpacking cannot be used in comprehension + >>> [*'a' for i in range(5)] + ['a', 'a', 'a', 'a', 'a'] >>> [*[] for i in range(10)] - Traceback (most recent call last): - ... - SyntaxError: iterable unpacking cannot be used in comprehension + [] + + >>> [*(x*2) for x in [[1, 2, 3], [], 'cat']] + [1, 2, 3, 1, 2, 3, 'c', 'a', 't', 'c', 'a', 't'] >>> {**{} for a in [1]} + {} + + >>> {**{7: i} for i in range(10)} + {7: 9} + + >>> dicts = [{1: 2}, {3: 4}, {5: 6, 7: 8}, {}, {9: 10}, {1: 0}] + >>> {**d for d in dicts} + {1: 0, 3: 4, 5: 6, 7: 8, 9: 10} + + >>> gen = (*(0, 1) for i in range(5)) + >>> next(gen) + 0 + >>> list(gen) + [1, 0, 1, 0, 1, 0, 1, 0, 1] + +Comprehension unpacking with conditionals and double loops + + >>> [*[i, i+1] for i in range(5) if i % 2 == 0] + [0, 1, 2, 3, 4, 5] + + >>> [*y for x in [[[0], [1, 2, 3], [], [4, 5]], [[6, 7]]] for y in x] + [0, 1, 2, 3, 4, 5, 6, 7] + + >>> [*y for x in [[[0], [1, 2, 3], [], [4, 5]], [[6, 7]]] for y in x if y and y[0]>0] + [1, 2, 3, 4, 5, 6, 7] + + >>> dicts = [{1: 2}, {3: 4}, {5: 6, 7: 8}, {}, {9: 10}, {1: 0}] + >>> {**d for d in dicts if len(d) != 2} + {1: 0, 3: 4, 9: 10} + +Scoping of assignment expressions in comprehensions + + >>> [*((y := i**2), 2*y) for i in range(4)] + [0, 0, 1, 2, 4, 8, 9, 18] + >>> y + 9 + + >>> [*(y := [i, i+1, i+2]) for i in range(4)] + [0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, 5] + >>> y + [3, 4, 5] + + >>> g = (*(z := [i, i+1, i+2]) for i in range(4)) + >>> z Traceback (most recent call last): ... - SyntaxError: dict unpacking cannot be used in dict comprehension + NameError: name 'z' is not defined + >>> next(g) + 0 + >>> z + [0, 1, 2] + >>> next(g) + 1 + >>> z + [0, 1, 2] + >>> next(g) + 2 + >>> z + [0, 1, 2] + >>> next(g) + 1 + >>> z + [1, 2, 3] -# Pegen is better here. -# Generator expression in function arguments + >>> x = [1, 2, 3] + >>> y = [4, 5, 6] + >>> def f(*args): + ... print(args) -# >>> list(*x for x in (range(5) for i in range(3))) -# Traceback (most recent call last): -# ... -# list(*x for x in (range(5) for i in range(3))) -# ^ -# SyntaxError: invalid syntax + >>> f(*x if x else y) + (1, 2, 3) + + +Malformed comperehension element unpacking + + >>> [*x for x in [1, 2, 3]] + Traceback (most recent call last): + ... + [*x for x in [1, 2, 3]] + ^^ + TypeError: Value after * must be an iterable, not int + + +Error messages for specific failure modes of unpacking + + >>> [*x if x else y for x in z] + Traceback (most recent call last): + ... + [*x if x else y for x in z] + ^^^^^^^^^^^^^^ + SyntaxError: invalid starred expression. Did you forget to wrap the conditional expression in parentheses? + + >>> [*x if x else y] + Traceback (most recent call last): + ... + [*x if x else y] + ^^^^^^^^^^^^^^ + SyntaxError: invalid starred expression. Did you forget to wrap the conditional expression in parentheses? + + >>> [x if x else *y for x in z] + Traceback (most recent call last): + ... + [x if x else *y for x in z] + ^ + SyntaxError: cannot unpack only part of a conditional expression + + >>> [x if x else *y] + Traceback (most recent call last): + ... + [x if x else *y] + ^ + SyntaxError: cannot unpack only part of a conditional expression + + >>> {**x if x else y} + Traceback (most recent call last): + ... + {**x if x else y} + ^^^^^^^^^^^^^^^^ + SyntaxError: invalid double starred expression. Did you forget to wrap the conditional expression in parentheses? + >>> {x if x else **y} + Traceback (most recent call last): + ... + {x if x else **y} + ^^ + SyntaxError: cannot use dict unpacking on only part of a conditional expression + + >>> [**x for x in [{1: 2}]] + Traceback (most recent call last): + ... + [**x for x in [{1: 2}]] + ^^^ + SyntaxError: cannot use dict unpacking in list comprehension + + >>> (**x for x in [{1:2}]) + Traceback (most recent call last): + ... + (**x for x in [{1:2}]) + ^^^ + SyntaxError: cannot use dict unpacking in generator expression >>> dict(**x for x in [{1:2}]) Traceback (most recent call last): ... dict(**x for x in [{1:2}]) - ^ - SyntaxError: invalid syntax + ^^^ + SyntaxError: cannot use dict unpacking in generator expression + + >>> {*a: b for a, b in {1: 2}.items()} + Traceback (most recent call last): + ... + {*a: b for a, b in {1: 2}.items()} + ^^ + SyntaxError: cannot use a starred expression in a dictionary key + + >>> {**a: b for a, b in {1: 2}.items()} + Traceback (most recent call last): + ... + {**a: b for a, b in {1: 2}.items()} + ^^^ + SyntaxError: cannot use dict unpacking in a dictionary key + + >>> {a: *b for a, b in {1: 2}.items()} + Traceback (most recent call last): + ... + {a: *b for a, b in {1: 2}.items()} + ^^ + SyntaxError: cannot use a starred expression in a dictionary value + + >>> {a: **b for a, b in {1: 2}.items()} + Traceback (most recent call last): + ... + {a: **b for a, b in {1: 2}.items()} + ^^^ + SyntaxError: cannot use dict unpacking in a dictionary value + + >>> (**a) + Traceback (most recent call last): + ... + (**a) + ^^ + SyntaxError: cannot use dict unpacking here + + >>> {**a for a in {1: 2}.items(): b} + Traceback (most recent call last): + ... + {**a for a in {1: 2}.items(): b} + ^^ + SyntaxError: cannot use dict unpacking here + + +# Generator expression in function arguments + + >>> list(*x for x in (range(5) for i in range(3))) + [0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4] + + >>> def f(arg): + ... print(type(arg), list(arg), list(arg)) + >>> f(*x for x in [[1,2,3]]) + [1, 2, 3] [] Iterable argument unpacking @@ -356,7 +546,7 @@ Now some general starred expressions (all fail). ... SyntaxError: can't use starred expression here - >>> (*x),y = 1, 2 + >>> (*x),y = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here @@ -366,12 +556,12 @@ Now some general starred expressions (all fail). ... SyntaxError: cannot use starred expression here - >>> z,(*x),y = 1, 2, 4 + >>> z,(*x),y = 1, 2, 4 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here - >>> z,(*x) = 1, 2 + >>> z,(*x) = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here @@ -383,13 +573,13 @@ Now some general starred expressions (all fail). Some size constraints (all fail.) - >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range(1<<8 + 1)" + >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range((1<<8) + 1)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: too many expressions in star-unpacking assignment - >>> s = ", ".join("a%d" % i for i in range(1<<8 + 1)) + ", *rest = range(1<<8 + 2)" + >>> s = ", ".join("a%d" % i for i in range((1<<8) + 1)) + ", *rest = range((1<<8) + 2)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... @@ -400,12 +590,180 @@ Some size constraints (all fail.) """ +def test_errors_in_iter(): + """ + >>> class A: + ... def __iter__(self): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + AttributeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + TypeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class A: + ... def __iter__(self): + ... return I() + ... + + >>> exc = ZeroDivisionError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + AttributeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = TypeError('some error') + >>> [*A()] + Traceback (most recent call last): + ... + TypeError: some error + + >>> {*A()} + Traceback (most recent call last): + ... + TypeError: some error + """ + +def test_errors_in_keys(): + """ + >>> class D: + ... def keys(self): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_keys_next(): + """ + >>> class I: + ... def __iter__(self): + ... return self + ... def __next__(self): + ... raise exc + ... + >>> class D: + ... def keys(self): + ... return I() + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + +def test_errors_in_getitem(): + """ + >>> class D: + ... def keys(self): + ... return ['key'] + ... def __getitem__(self, key): + ... raise exc + ... + >>> exc = ZeroDivisionError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + ZeroDivisionError: some error + + >>> exc = AttributeError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + AttributeError: some error + + >>> exc = KeyError('some error') + >>> {**D()} + Traceback (most recent call last): + ... + KeyError: 'some error' + """ + __test__ = {'doctests' : doctests} - def load_tests(loader, tests, pattern): - from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON - tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # XXX: RUSTPYTHON + tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 35e4652a8..9bc04fc8e 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -216,6 +216,15 @@ class UnparseTestCase(ASTTestCase): self.check_ast_roundtrip('t""') self.check_ast_roundtrip("t'{(lambda x: x)}'") self.check_ast_roundtrip("t'{t'{x}'}'") + self.check_ast_roundtrip( + r"""t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}'''""" + ) + self.check_ast_roundtrip( + r'''t"""Value: {value =\ +}"""''' + ) def test_tstring_with_nonsensical_str_field(self): # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise @@ -403,6 +412,11 @@ class UnparseTestCase(ASTTestCase): def test_dict_comprehension(self): self.check_ast_roundtrip("{x: x*x for x in range(10)}") + def test_dict_comprehension_unpacking(self): + self.check_ast_roundtrip("{**x for x in ()}") + self.check_ast_roundtrip("{**x for x in range(10)}") + self.check_ast_roundtrip("[*x for x in ()]") + def test_class_decorators(self): self.check_ast_roundtrip(class_decorator) @@ -679,6 +693,24 @@ class UnparseTestCase(ASTTestCase): self.check_src_roundtrip(src, out, mode='single') + @test.support.subTests('case', [ + "case 'a string':", + "case True:", + "case _:", + "case [*_]:", + "case tuple():", + "case cls(arg, more=arg2):", + "case tuple() | list():", + "case [tuple() as obj, _, {'a': a, **more}]:", + "case -2:", + "case +2:", + "case 2 + 3j:", + "case +2 - 3j:", + ]) + def test_unparse_match(self, case): + src = 'match x:\n ' + case + '\n pass' + self.check_src_roundtrip(src) + class CosmeticTestCase(ASTTestCase): """Test if there are cosmetic issues caused by unnecessary additions""" diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index fe25baf41..53c87168a 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -9421,14 +9421,14 @@ impl<'warnings> Compiler<'warnings> { ), generators, &|compiler, collection_add_i| { - compiler.compile_comprehension_element(elt)?; + let unpack = compiler.compile_comprehension_element(elt)?; compiler.set_source_node(elt.range(), elt.node_index()); - emit!( - compiler, - Instruction::ListAppend { - i: collection_add_i.to_u32(), - } - ); + let i = collection_add_i.to_u32(); + if unpack { + emit!(compiler, Instruction::ListExtend { i }); + } else { + emit!(compiler, Instruction::ListAppend { i }); + } Ok(()) }, ComprehensionType::List, @@ -9451,14 +9451,14 @@ impl<'warnings> Compiler<'warnings> { ), generators, &|compiler, collection_add_i| { - compiler.compile_comprehension_element(elt)?; + let unpack = compiler.compile_comprehension_element(elt)?; compiler.set_source_node(elt.range(), elt.node_index()); - emit!( - compiler, - Instruction::SetAdd { - i: collection_add_i.to_u32(), - } - ); + let i = collection_add_i.to_u32(); + if unpack { + emit!(compiler, Instruction::SetUpdate { i }); + } else { + emit!(compiler, Instruction::SetAdd { i }); + } Ok(()) }, ComprehensionType::Set, @@ -9474,11 +9474,9 @@ impl<'warnings> Compiler<'warnings> { generators, .. }) => { - let Some(key) = key.as_deref() else { - self.restore_source_span(expression_span); - return Err(self.error(CodegenErrorType::SyntaxError( - "dict unpacking cannot be used in dict comprehension".to_owned(), - ))); + let element_span = match key.as_deref() { + Some(key) => self.dict_comprehension_element_span(key, value)?, + None => self.node_span(value.as_ref()), }; self.compile_comprehension( "", @@ -9490,29 +9488,30 @@ impl<'warnings> Compiler<'warnings> { ), generators, &|compiler, collection_add_i| { - // changed evaluation order for Py38 named expression PEP 572 - compiler.compile_expression(key)?; + // Keys precede values; a missing Ruff key denotes **value. + if let Some(key) = key { + // Changed evaluation order for Py38 named expression PEP 572. + compiler.compile_expression(key)?; + } compiler.compile_expression(value)?; - compiler.restore_source_span( - compiler.dict_comprehension_element_span(key, value)?, - ); - emit!( - compiler, - Instruction::MapAdd { - i: collection_add_i.to_u32(), - } - ); + compiler.restore_source_span(element_span); + let i = collection_add_i.to_u32(); + if key.is_some() { + emit!(compiler, Instruction::MapAdd { i }); + } else { + emit!(compiler, Instruction::DictUpdate { i }); + } Ok(()) }, ComprehensionType::Dict, - Self::contains_await(key) + key.as_deref().is_some_and(Self::contains_await) || Self::contains_await(value) || Self::generators_contain_await(generators), expression_span, - self.dict_comprehension_element_span(key, value)?, - self.node_span(key), + element_span, + self.node_span(key.as_deref().unwrap_or(value)), )?; } ast::Expr::Generator(ast::ExprGenerator { @@ -10255,16 +10254,13 @@ impl<'warnings> Compiler<'warnings> { Ok(()) } - fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> { - self.compile_expression(element).map_err(|e| { - if let CodegenErrorType::InvalidStarExpr = e.error { - self.error(CodegenErrorType::SyntaxError( - "iterable unpacking cannot be used in comprehension".to_owned(), - )) - } else { - e - } - }) + fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult { + let (value, unpack) = match element { + ast::Expr::Starred(starred) => (&*starred.value, true), + _ => (element, false), + }; + self.compile_expression(value)?; + Ok(unpack) } fn compile_generator_expression( @@ -10285,7 +10281,15 @@ impl<'warnings> Compiler<'warnings> { // Compile the element expression // Note: if element is an async comprehension, compile_expression // already handles awaiting it, so we don't need to await again here - compiler.compile_comprehension_element(elt)?; + let unpack = compiler.compile_comprehension_element(elt)?; + let unpack_loop = unpack.then(|| (compiler.new_block(), compiler.new_block())); + if let Some((start, end)) = unpack_loop { + // Unpacking forwards values, not send()/throw() as yield-from would. + compiler.set_source_node(elt.range(), elt.node_index()); + emit!(compiler, Instruction::GetIter { mode: 0 }); + compiler.use_cpython_label_block(start); + emit!(compiler, Instruction::ForIter { delta: end }); + } compiler.mark_generator(); if compiler.ctx.func == FunctionContext::AsyncFunction { @@ -10307,6 +10311,12 @@ impl<'warnings> Compiler<'warnings> { } ); emit!(compiler, Instruction::PopTop); + if let Some((start, end)) = unpack_loop { + emit!(compiler, PseudoInstruction::Jump { delta: start }); + compiler.set_no_location(); + compiler.use_cpython_label_block(end); + compiler.emit_sync_comprehension_end_for(); + } Ok(()) }, diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 292324263..da0a16acb 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -2419,16 +2419,14 @@ impl SymbolTableBuilder { self.in_iter_def_exp = true; } // Dict comprehension - is_generator = false (can be inlined) - let Some(key) = key.as_deref() else { - return Err(self.error_ranged( - "dict unpacking cannot be used in dict comprehension".to_owned(), - *range, - )); + let (element, value) = match key.as_deref() { + Some(key) => (key, Some(value.as_ref())), + None => (value.as_ref(), None), }; self.scan_comprehension( &"".into(), - key, - Some(value), + element, + value, generators, *range, false, diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 1a7df5e27..d78282589 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -1264,6 +1264,8 @@ fn first_parse_error( parsed: parser::Parsed, source_file: &SourceFile, mode: Mode, + prepared_source: &str, + parser_options: &parser::ParseOptions, ) -> Result, CompileError> { let Some((first, rest)) = parsed.errors().split_first() else { return Ok(parsed); @@ -1272,13 +1274,556 @@ fn first_parse_error( !matches!(error.error, ParseErrorType::Lexical(_)) && error.location.start() < first.location.start() }); - let mut error = CompileError::from_ruff_parse_error(first.clone(), source_file, mode); + let normalized = unpacking_parse_diagnostic( + first, + rest, + parsed.tokens(), + parsed.syntax(), + parsed.unsupported_syntax_errors().is_empty(), + ) + .or_else(|| call_argument_unpacking_diagnostic(&parsed, prepared_source, parser_options)) + .or_else(|| unfinished_dict_comprehension_diagnostic(&parsed, parser_options)) + .map_or_else( + || first.clone(), + |diagnostic| parser::ParseError { + error: ParseErrorType::OtherError(diagnostic.message), + location: diagnostic.range, + }, + ); + let mut error = CompileError::from_ruff_parse_error(normalized, source_file, mode); if after_syntax_error && let CompileError::Parse(error) = &mut error { error.is_unclosed_string = false; } Err(error) } +// A recovered call never parsed its comprehension clauses. For a narrowly owned +// whole-input argument list, ask the same parser to validate that exact slice once. +fn call_argument_unpacking_diagnostic( + parsed: &parser::Parsed, + prepared_source: &str, + options: &parser::ParseOptions, +) -> Option { + let first = parsed.errors().first()?; + if !matches!( + first.error, + ParseErrorType::ExpectedToken { + expected: TokenKind::Rpar, + found: TokenKind::For + } + ) || !parsed.unsupported_syntax_errors().is_empty() + || parsed + .errors() + .iter() + .any(|error| error.location.start() < first.location.start()) + { + return None; + } + let ast::Mod::Module(module) = parsed.syntax() else { + return None; + }; + let ast::Stmt::Expr(statement) = module.body.first()? else { + return None; + }; + let ast::Expr::Call(call) = statement.value.as_ref() else { + return None; + }; + let [keyword] = call.arguments.keywords.as_slice() else { + return None; + }; + if !call.arguments.args.is_empty() + || keyword.arg.is_some() + || keyword.end() != call.arguments.end() + || call.end() != keyword.end() + { + return None; + } + let tokens: Vec<_> = parsed + .tokens() + .iter() + .filter(|token| { + !matches!( + token.kind(), + TokenKind::Comment + | TokenKind::NonLogicalNewline + | TokenKind::Newline + | TokenKind::EndOfFile + ) + }) + .collect(); + if tokens.first()?.start() != call.start() { + return None; + } + let opening_index = tokens + .iter() + .position(|token| token.start() == call.arguments.start())?; + let opening = tokens.get(opening_index)?; + let operator = tokens.get(opening_index.checked_add(1)?)?; + let closing = tokens.last()?; + let clause = tokens.iter().find(|token| token.start() >= keyword.end())?; + if opening.kind() != TokenKind::Lpar + || operator.kind() != TokenKind::DoubleStar + || operator.start() != keyword.start() + || closing.kind() != TokenKind::Rpar + || clause.range() != first.location + || clause.kind() != TokenKind::For + { + return None; + } + let slice = prepared_source.get(usize::from(opening.start())..usize::from(closing.end()))?; + let reparsed = parser::parse_unchecked( + slice, + parser::ParseOptions::from(parser::Mode::Expression) + .with_target_version(options.target_version()), + ); + let [error] = reparsed.errors() else { + return None; + }; + if !matches!(error.error, ParseErrorType::ExpectedExpression) + || !reparsed.unsupported_syntax_errors().is_empty() + || opening.start().checked_add(error.location.start())? != operator.start() + || opening.start().checked_add(error.location.end())? != operator.end() + { + return None; + } + let ast::Mod::Expression(expression) = reparsed.syntax() else { + return None; + }; + let ast::Expr::Generator(generator) = expression.body.as_ref() else { + return None; + }; + let ast::Expr::BinOp(binary) = generator.elt.as_ref() else { + return None; + }; + if !generator.parenthesized + || generator.start() != TextSize::new(0) + || usize::from(generator.end()) != slice.len() + || binary.end() != binary.right.end() + { + return None; + } + // Both parses must retain the same unpacked operand, not just its operator. + if opening.start().checked_add(generator.elt.start())? != keyword.start() + || opening.start().checked_add(generator.elt.end())? != keyword.end() + || opening.start().checked_add(binary.right.start())? != keyword.value.start() + || opening.start().checked_add(binary.right.end())? != keyword.value.end() + { + return None; + } + let diagnostic = owned_unpacking_diagnostic(error, reparsed.tokens(), reparsed.syntax())?; + if usize::from(diagnostic.range.end()) > slice.len() { + return None; + } + // Module recovery forgets the call's opening delimiter before reaching its + // closing token. Only the complete, balanced generator reparse above can + // justify treating this exact unmatched-closing error as a recovery shadow. + if parsed.errors().iter().any(|error| match &error.error { + ParseErrorType::Lexical(parser::LexicalErrorType::UnmatchedBracket { closing: ')' }) => { + !error.location.is_empty() || error.location.start() != closing.start() + } + ParseErrorType::Lexical(_) => true, + _ => false, + }) { + return None; + } + Some(CpythonDiagnostic { + message: diagnostic.message, + range: ruff_text_size::TextRange::new( + opening.start().checked_add(diagnostic.range.start())?, + opening.start().checked_add(diagnostic.range.end())?, + ), + }) +} + +// An unfinished unpacking comprehension can recover as an annotated assignment. +// Recognize only the complete, owned recovery cascade, not arbitrary later errors. +fn unfinished_dict_comprehension_diagnostic( + parsed: &parser::Parsed, + options: &parser::ParseOptions, +) -> Option { + let first = parsed.errors().first()?; + if !matches!( + first.error, + ParseErrorType::ExpectedToken { + expected: TokenKind::Rbrace, + found: TokenKind::Colon + } + ) { + return None; + } + let ast::Mod::Module(module) = parsed.syntax() else { + return None; + }; + let [ast::Stmt::AnnAssign(statement)] = module.body.as_slice() else { + return None; + }; + let ast::Expr::DictComp(dict) = statement.target.as_ref() else { + return None; + }; + if dict.key.is_some() + || dict.end() != dict.generators.last()?.end() + || statement.start() != dict.start() + || statement.value.is_some() + || statement.annotation.range().is_empty() + || statement.annotation.start() < first.location.end() + || statement.annotation.end() != statement.end() + { + return None; + } + let tokens: Vec<_> = parsed + .tokens() + .iter() + .filter(|token| { + !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline | TokenKind::EndOfFile + ) + }) + .collect(); + let opening = tokens.first()?; + let operator = tokens.get(1)?; + if opening.kind() != TokenKind::Lbrace + || opening.start() != dict.start() + || operator.kind() != TokenKind::DoubleStar + || operator.end() > dict.value.start() + { + return None; + } + let colon = tokens.iter().find(|token| token.start() >= dict.end())?; + if colon.kind() != TokenKind::Colon || colon.range() != first.location { + return None; + } + let closing_index = tokens + .iter() + .position(|token| token.start() >= statement.end())?; + let closing = tokens.get(closing_index)?; + if closing.kind() != TokenKind::Rbrace { + return None; + } + let newline = match tokens.get(closing_index.checked_add(1)?..)? { + [] => None, + [newline] if newline.kind() == TokenKind::Newline => Some(*newline), + _ => return None, + }; + let [unsupported] = parsed.unsupported_syntax_errors() else { + return None; + }; + // The payload enum is not exported. This exact DictComp/operator range owns + // the parser's DictInDict producer, excluding other comprehension features. + if !matches!( + unsupported.kind, + parser::UnsupportedSyntaxErrorKind::UnpackingInComprehension(_) + ) || unsupported.range != ruff_text_size::TextRange::new(operator.start(), dict.value.end()) + || unsupported.target_version != options.target_version() + { + return None; + } + let errors = parsed.errors(); + if errors.len() != 3 + usize::from(newline.is_some()) + || !matches!( + errors.get(1)?.error, + ParseErrorType::InvalidAnnotatedAssignmentTarget + ) + || errors.get(1)?.location != dict.range() + { + return None; + } + let is_shadow = |error: &parser::ParseError, range| { + matches!(&error.error, ParseErrorType::OtherError(message) if message == "invalid syntax") + && error.location == range + }; + if !is_shadow(errors.get(2)?, closing.range()) + || newline.is_some_and(|newline| !is_shadow(&errors[3], newline.range())) + { + return None; + } + Some(CpythonDiagnostic { + message: "cannot use dict unpacking here".to_owned(), + range: operator.range(), + }) +} + +/// Refine unpacking errors whose grammar context and operand range the parser already +/// identified. Tokens only distinguish the unpacking operator; they do not reparse the input. +fn unpacking_parse_diagnostic( + error: &parser::ParseError, + following_errors: &[parser::ParseError], + tokens: &Tokens, + syntax: &ast::Mod, + no_unsupported_errors: bool, +) -> Option { + if matches!( + error.error, + ParseErrorType::InvalidStarredExpressionUsage | ParseErrorType::ExpectedExpression + ) && following_errors.is_empty() + && no_unsupported_errors + { + return owned_unpacking_diagnostic(error, tokens, syntax); + } + let ParseErrorType::OtherError(message) = &error.error else { + return None; + }; + let mut range = error.location; + let message = match message.as_str() { + "cannot use double starred expression here" if following_errors.is_empty() => { + let mut operand = tokens.iter().filter(|token| { + token.start() >= range.end() + && !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline + ) + }); + // The parser consumed a complete parenthesized operand. Lambda and + // `not` do not start the bitwise-or prefix required by CPython. + if matches!(operand.next()?.kind(), TokenKind::Lambda | TokenKind::Not) { + return None; + } + "cannot use dict unpacking here".to_owned() + } + "pattern cannot follow a double star pattern" => { + // A later recovery error can change which invalid production CPython + // selects. Leave those inputs to the existing diagnostic conversion. + if !following_errors.is_empty() { + return None; + } + let mut preceding = tokens.iter().rev().filter(|token| { + token.end() <= range.start() + && !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline + ) + }); + // The parser has already recognized a mapping entry after its rest + // capture. CPython points at that capture, not the following entry. + if preceding.next()?.kind() != TokenKind::Comma { + return None; + } + let capture = preceding.next()?; + if capture.kind() != TokenKind::Identifier + || preceding.next()?.kind() != TokenKind::DoubleStar + { + return None; + } + range = capture.range(); + "double star pattern must be the last (right-most) subpattern in the mapping pattern" + .to_owned() + } + "conditional expression cannot be used here" => { + let mut preceding = tokens.iter().rev().filter(|token| { + token.end() <= range.start() + && !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline + ) + }); + let operator = preceding.next()?; + let kind = match operator.kind() { + TokenKind::Star => "starred", + // Only the first dictionary spread uses this invalid production. + // Dictionary values and later spreads have different error priority. + TokenKind::DoubleStar + if preceding + .next() + .is_some_and(|token| token.kind() == TokenKind::Lbrace) => + { + "double starred" + } + _ => return None, + }; + range = ruff_text_size::TextRange::new(operator.start(), range.end()); + // Do not replace the operand error when the parser also rejected the + // enclosing unpacking operation, for example in a dictionary value. + if following_errors.iter().any(|error| { + error.location == range + && (matches!(error.error, ParseErrorType::InvalidStarredExpressionUsage) + || matches!(&error.error, ParseErrorType::OtherError(message) + if message == "cannot use a starred expression in a dictionary value")) + }) { + return None; + } + format!( + "invalid {kind} expression. Did you forget to wrap the conditional expression in parentheses?" + ) + } + "expected expression after 'else', but statement is given" => { + let operator = tokens.iter().find(|token| token.range() == range)?; + match operator.kind() { + TokenKind::Star => "cannot unpack only part of a conditional expression", + TokenKind::DoubleStar => { + "cannot use dict unpacking on only part of a conditional expression" + } + _ => return None, + } + .to_owned() + } + _ => return None, + }; + Some(CpythonDiagnostic { message, range }) +} + +fn owned_unpacking_diagnostic( + error: &parser::ParseError, + tokens: &Tokens, + syntax: &ast::Mod, +) -> Option { + use ast::visitor::Visitor; + + struct Checker<'a> { + range: ruff_text_size::TextRange, + tokens: &'a Tokens, + comprehension: bool, + message: Option<&'static str>, + } + + impl Checker<'_> { + fn is_double_starred_operand(&self, operand: &ast::Expr) -> bool { + self.tokens + .iter() + .rev() + .find(|token| { + token.end() <= operand.start() + && !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline + ) + }) + .is_some_and(|token| { + token.kind() == TokenKind::DoubleStar && token.start() == self.range.start() + }) + } + + fn check_comprehension( + &mut self, + element: &ast::Expr, + generators: &[ast::Comprehension], + delimiter: TokenKind, + container_start: TextSize, + message: &'static str, + ) { + let ast::Expr::BinOp(binary) = element else { + return; + }; + let ast::Expr::Name(missing) = binary.left.as_ref() else { + return; + }; + if binary.op != ast::Operator::Pow + || !missing.id.is_empty() + || missing.ctx != ast::ExprContext::Invalid + || !missing.range.is_empty() + || element.start() != self.range.start() + || generators.is_empty() + { + return; + } + let Some(operator) = self.tokens.iter().find(|token| token.range() == self.range) + else { + return; + }; + let preceding = self.tokens.iter().rev().find(|token| { + token.end() <= operator.start() + && !matches!( + token.kind(), + TokenKind::Comment | TokenKind::NonLogicalNewline + ) + }); + if operator.kind() == TokenKind::DoubleStar + && preceding.is_some_and(|token| { + token.kind() == delimiter && token.start() == container_start + }) + { + self.range = ruff_text_size::TextRange::new(operator.start(), element.end()); + self.message = Some(message); + } + } + + fn check_item(&mut self, key: Option<&ast::Expr>, value: &ast::Expr) { + let Some(key) = key else { return }; + // The parser reports InvalidStarredExpressionUsage at just the first + // byte of the operator. Recover the operand end from its retained role. + if self.range.len() != TextSize::new(1) { + return; + } + let (message, end) = if matches!(key, ast::Expr::Starred(_)) + && key.start() == self.range.start() + && self + .tokens + .iter() + .any(|token| token.kind() == TokenKind::Star && token.range() == self.range) + { + ( + "cannot use a starred expression in a dictionary key", + key.end(), + ) + } else if self.is_double_starred_operand(key) { + ("cannot use dict unpacking in a dictionary key", key.end()) + } else if self.is_double_starred_operand(value) { + ( + "cannot use dict unpacking in a dictionary value", + value.end(), + ) + } else { + return; + }; + self.range = ruff_text_size::TextRange::new(self.range.start(), end); + self.message = Some(message); + } + } + + impl<'a> Visitor<'a> for Checker<'_> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.message.is_some() || !expr.range().contains_range(self.range) { + return; + } + // Retained nodes establish the operand role; the parser reported + // no other errors. Dictionary errors here only have first-item producers. + match expr { + ast::Expr::ListComp(comp) if self.comprehension => self.check_comprehension( + &comp.elt, + &comp.generators, + TokenKind::Lsqb, + comp.start(), + "cannot use dict unpacking in list comprehension", + ), + ast::Expr::Generator(comp) if self.comprehension && comp.parenthesized => self + .check_comprehension( + &comp.elt, + &comp.generators, + TokenKind::Lpar, + comp.start(), + "cannot use dict unpacking in generator expression", + ), + ast::Expr::Dict(dict) if !self.comprehension => { + if let Some(item) = dict.items.first() { + self.check_item(item.key.as_ref(), &item.value); + } + } + ast::Expr::DictComp(dict) if !self.comprehension => { + self.check_item(dict.key.as_deref(), &dict.value) + } + _ => {} + } + if self.message.is_none() { + ast::visitor::walk_expr(self, expr); + } + } + } + + let mut checker = Checker { + range: error.location, + tokens, + comprehension: matches!(error.error, ParseErrorType::ExpectedExpression), + message: None, + }; + match syntax { + ast::Mod::Module(module) => checker.visit_body(&module.body), + ast::Mod::Expression(expression) => checker.visit_expr(&expression.body), + } + checker.message.map(|message| CpythonDiagnostic { + message: message.to_owned(), + range: checker.range, + }) +} + /// The first logical input accepted by the single-input grammar. #[derive(Clone, Copy)] pub enum SingleModeStatementBoundary { @@ -1586,7 +2131,7 @@ fn _compile_with_syntax_warning_handler<'a>( .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), ); pre_parse_source_error(&source_file)?; - let parsed = parser::parse_unchecked(barry_source.source(), parser_options); + let parsed = parser::parse_unchecked(barry_source.source(), parser_options.clone()); if matches!(mode, Mode::Single) && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { @@ -1595,7 +2140,13 @@ fn _compile_with_syntax_warning_handler<'a>( if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let parsed = first_parse_error(parsed, &source_file, mode)?; + let parsed = first_parse_error( + parsed, + &source_file, + mode, + barry_source.source(), + &parser_options, + )?; if matches!(mode, Mode::Single) && let Some(error) = single_mode_blank_source_error(&source_file) { @@ -1823,14 +2374,20 @@ pub fn interactive_input( options.clone(), future_features.contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), ); - let parsed = parser::parse_unchecked(prepared.source(), options); + let parsed = parser::parse_unchecked(prepared.source(), options.clone()); if let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { return Err(error); } if let Some(error) = prepared.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let parsed = first_parse_error(parsed, &source_file, Mode::Single)?; + let parsed = first_parse_error( + parsed, + &source_file, + Mode::Single, + prepared.source(), + &options, + )?; let ast::Mod::Module(module) = parsed.syntax() else { unreachable!() }; @@ -1926,7 +2483,8 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + let parsed = + ruff_python_parser::parse_unchecked(barry_source.source(), parser_options.clone()); if matches!(mode, Mode::Single) && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { @@ -1935,7 +2493,13 @@ pub fn _compile_symtable( if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let ast = first_parse_error(parsed, &source_file, mode)?; + let ast = first_parse_error( + parsed, + &source_file, + mode, + barry_source.source(), + &parser_options, + )?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -1958,11 +2522,18 @@ pub fn _compile_symtable( } Mode::Eval => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + let parsed = + ruff_python_parser::parse_unchecked(barry_source.source(), parser_options.clone()); if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let ast = first_parse_error(parsed, &source_file, mode)?; + let ast = first_parse_error( + parsed, + &source_file, + mode, + barry_source.source(), + &parser_options, + )?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { diff --git a/crates/doc/src/data.inc.rs b/crates/doc/src/data.inc.rs index 0d10b224d..c9a37dc9c 100644 --- a/crates/doc/src/data.inc.rs +++ b/crates/doc/src/data.inc.rs @@ -685,7 +685,7 @@ pub const DB: &[(&str, &str)] = &[ ("_ast.Dict.__str__", "Return str(self)."), ("_ast.Dict.__subclasshook__", "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).\n"), ("_ast.Dict.__weakref__", "list of weak references to the object"), - ("_ast.DictComp", "DictComp(expr key, expr value, comprehension* generators)"), + ("_ast.DictComp", "DictComp(expr key, expr? value, comprehension* generators)"), ("_ast.DictComp.__class__", "the object's class"), ("_ast.DictComp.__delattr__", "Implement delattr(self, name)."), ("_ast.DictComp.__dir__", "Default dir() implementation."), @@ -2886,7 +2886,7 @@ pub const DB: &[(&str, &str)] = &[ ("_ast.excepthandler.__str__", "Return str(self)."), ("_ast.excepthandler.__subclasshook__", "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).\n"), ("_ast.excepthandler.__weakref__", "list of weak references to the object"), - ("_ast.expr", "expr = BoolOp(boolop op, expr* values)\n | NamedExpr(expr target, expr value)\n | BinOp(expr left, operator op, expr right)\n | UnaryOp(unaryop op, expr operand)\n | Lambda(arguments args, expr body)\n | IfExp(expr test, expr body, expr orelse)\n | Dict(expr?* keys, expr* values)\n | Set(expr* elts)\n | ListComp(expr elt, comprehension* generators)\n | SetComp(expr elt, comprehension* generators)\n | DictComp(expr key, expr value, comprehension* generators)\n | GeneratorExp(expr elt, comprehension* generators)\n | Await(expr value)\n | Yield(expr? value)\n | YieldFrom(expr value)\n | Compare(expr left, cmpop* ops, expr* comparators)\n | Call(expr func, expr* args, keyword* keywords)\n | FormattedValue(expr value, int conversion, expr? format_spec)\n | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n | JoinedStr(expr* values)\n | TemplateStr(expr* values)\n | Constant(constant value, string? kind)\n | Attribute(expr value, identifier attr, expr_context ctx)\n | Subscript(expr value, expr slice, expr_context ctx)\n | Starred(expr value, expr_context ctx)\n | Name(identifier id, expr_context ctx)\n | List(expr* elts, expr_context ctx)\n | Tuple(expr* elts, expr_context ctx)\n | Slice(expr? lower, expr? upper, expr? step)"), + ("_ast.expr", "expr = BoolOp(boolop op, expr* values)\n | NamedExpr(expr target, expr value)\n | BinOp(expr left, operator op, expr right)\n | UnaryOp(unaryop op, expr operand)\n | Lambda(arguments args, expr body)\n | IfExp(expr test, expr body, expr orelse)\n | Dict(expr?* keys, expr* values)\n | Set(expr* elts)\n | ListComp(expr elt, comprehension* generators)\n | SetComp(expr elt, comprehension* generators)\n | DictComp(expr key, expr? value, comprehension* generators)\n | GeneratorExp(expr elt, comprehension* generators)\n | Await(expr value)\n | Yield(expr? value)\n | YieldFrom(expr value)\n | Compare(expr left, cmpop* ops, expr* comparators)\n | Call(expr func, expr* args, keyword* keywords)\n | FormattedValue(expr value, int conversion, expr? format_spec)\n | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n | JoinedStr(expr* values)\n | TemplateStr(expr* values)\n | Constant(constant value, string? kind)\n | Attribute(expr value, identifier attr, expr_context ctx)\n | Subscript(expr value, expr slice, expr_context ctx)\n | Starred(expr value, expr_context ctx)\n | Name(identifier id, expr_context ctx)\n | List(expr* elts, expr_context ctx)\n | Tuple(expr* elts, expr_context ctx)\n | Slice(expr? lower, expr? upper, expr? step)"), ("_ast.expr.__class__", "the object's class"), ("_ast.expr.__delattr__", "Implement delattr(self, name)."), ("_ast.expr.__dir__", "Default dir() implementation."), diff --git a/crates/vm/src/stdlib/_ast/expression.rs b/crates/vm/src/stdlib/_ast/expression.rs index 3aa2e6fda..a5625379f 100644 --- a/crates/vm/src/stdlib/_ast/expression.rs +++ b/crates/vm/src/stdlib/_ast/expression.rs @@ -858,16 +858,19 @@ fn expr_dict_comp_from_object_with_range( object: &PyObject, range: AstNodeRange, ) -> PyResult { + // Ruff uses a missing key for unpacking; Python's AST uses a missing value. + let key = get_required_node_field(vm, source_file, object, "key", "DictComp")?; + let value = get_node_field_opt(vm, object, "value")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?; + let (key, value) = match value { + Some(value) => (Some(key), value), + None => (None, key), + }; Ok(ast::ExprDictComp { node_index: range.node_index(), - key: Some(get_required_node_field( - vm, - source_file, - object, - "key", - "DictComp", - )?), - value: get_required_node_field(vm, source_file, object, "value", "DictComp")?, + key, + value, generators: get_node_list_field(vm, source_file, object, "generators", "DictComp")? .into_boxed_slice(), range: range.range, @@ -883,6 +886,10 @@ impl Node for ast::ExprDictComp { generators, range, } = self; + let (key, value) = match key { + Some(key) => (key, Some(value)), + None => (value, None), + }; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprDictComp::static_type().to_owned()) .unwrap(); diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs index dfcd6dc82..4632d6e03 100644 --- a/crates/vm/src/stdlib/_ast/pyast.rs +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -1291,7 +1291,7 @@ const FIELD_TYPES: &[(&str, &[(&str, FieldType)])] = &[ "DictComp", &[ ("key", FieldType::Node("expr")), - ("value", FieldType::Node("expr")), + ("value", FieldType::Optional("expr")), ("generators", FieldType::ListOf("comprehension")), ], ), diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs index 2449dcaf1..388322038 100644 --- a/crates/vm/src/stdlib/_ast/python.rs +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -601,7 +601,7 @@ This will become an error in Python 3.20", | Set(expr* elts)\n\ | ListComp(expr elt, comprehension* generators)\n\ | SetComp(expr elt, comprehension* generators)\n\ - | DictComp(expr key, expr value, comprehension* generators)\n\ + | DictComp(expr key, expr? value, comprehension* generators)\n\ | GeneratorExp(expr elt, comprehension* generators)\n\ | Await(expr value)\n\ | Yield(expr? value)\n\ diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs index abfbc636f..cda5ccb26 100644 --- a/crates/vm/src/stdlib/_ast/validate.rs +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -615,10 +615,9 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - } ast::Expr::DictComp(dict) => { validate_comprehension(vm, &dict.generators)?; - let key = dict.key.as_deref().ok_or_else(|| { - vm.new_value_error("field 'key' is required for DictComp".to_owned()) - })?; - validate_expr(vm, key, ast::ExprContext::Load)?; + if let Some(key) = &dict.key { + validate_expr(vm, key, ast::ExprContext::Load)?; + } validate_expr(vm, &dict.value, ast::ExprContext::Load) } ast::Expr::Generator(generator) => {