diff --git a/Cargo.lock b/Cargo.lock index aa1ec9cf2..4b15f9af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "char_str" -version = "0.0.2" +version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576ba56f6ca18ebb069d0d07260407171712aff4dfe15ee3f8982dc16a455bcf" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" dependencies = [ "castaway", "get-size2", @@ -1612,9 +1612,9 @@ dependencies = [ [[package]] name = "get-size-derive2" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" +checksum = "7efe46d2adb529015748742f1a38594039ce357dbb21e6d178de342cfce468d4" dependencies = [ "attribute-derive", "quote", @@ -1623,9 +1623,9 @@ dependencies = [ [[package]] name = "get-size2" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b411f34418305908ab15a82ff78958c2a9aee9a272b2a2e663836b20a4e4b9d3" +checksum = "d4d338de392104417fe6fdbe588c52383b9236956d8afe5654f0077176edc632" dependencies = [ "compact_str", "get-size-derive2", @@ -3935,9 +3935,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_ast" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc19eeb4cfa2c35428143693eadf85ee78816f22a829298593581a56700c0a21" +checksum = "5b7ffeb85f9a68c461d538839c79548584768df08f2d68cb1ee693be697e0355" dependencies = [ "aho-corasick", "arrayvec", @@ -3957,14 +3957,16 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_parser" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "480eec9ac4dda77b6724f0c9a54a81d3c57868d6a707c3d7c5133fd83c933f66" +checksum = "7c9729e35cac02839c7a67aaccfea3eba32032f7aeacca2e82ad79e8d7b7fd95" dependencies = [ "bitflags 2.13.2", "bstr", "drop_bomb", "get-size2", + "hashbrown 0.17.1", + "icu_properties", "memchr", "rustc-hash", "rustpython-ruff_python_ast", @@ -3980,22 +3982,24 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_trivia" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d69a5d9bb610a76bee05d23adce0de55f0fa06f251c0e1c7f7a0c625a366a58" +checksum = "764f43ea060f888164fbd360b0e1a36f2a35f5c03def85407bab7039fc62a645" dependencies = [ "itertools 0.15.0", + "memchr", "rustc-hash", "rustpython-ruff_source_file", "rustpython-ruff_text_size", "unicode-ident", + "unicode-normalization", ] [[package]] name = "rustpython-ruff_source_file" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3aa9c91c8a4e86bf0fa8a67aec284fa6c0c533cdefb900b08584be15748c606" +checksum = "cfe26430c3f2eadb40524b6df7ac9a3970dd2b58f921b01ed75df0121e332fad" dependencies = [ "memchr", "rustpython-ruff_text_size", @@ -4003,9 +4007,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_text_size" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fedbd1af4f91cd828185123d95119662db9c4a0b21c02b695783d74360d4f46" +checksum = "5e5377a04f2855030600b93a05754242950ea9b070507ef9aef67965a52eeb09" dependencies = [ "get-size2", ] diff --git a/Cargo.toml b/Cargo.toml index 74565c4b0..111ba9868 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,10 +188,10 @@ rustpython-wtf8 = { path = "crates/wtf8", version = "0.6.1" } rustpython-doc = { path = "crates/doc", version = "0.6.1" } # Use the RustPython Ruff fork for RustPython public `_ast` metadata. -ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.5" } -ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.5" } -ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.5" } -ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.5" } +ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.10" } +ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.10" } +ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.10" } +ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.10" } der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] } phf = { version = "0.14.0", default-features = false, features = ["macros"]} diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index fb519878c..3642b47c2 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -111,7 +111,6 @@ class TestInteractiveConsole(unittest.TestCase, MockSys): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - @unittest.expectedFailure # TODO: RUSTPYTHON; - 'IndentationError: unexpected indentation'] def test_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] self.console.interact() @@ -187,7 +186,6 @@ class TestInteractiveConsole(unittest.TestCase, MockSys): ' ^\n', 'SyntaxError: invalid syntax\n']) - @unittest.expectedFailure # TODO: RUSTPYTHON; + 'IndentationError: unexpected indent\n'] def test_sysexcepthook_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] hook = mock.Mock() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index fc7ebb0f5..26b56dac5 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -75,7 +75,6 @@ class DictComprehensionTest(unittest.TestCase): self.assertEqual(actual, expected) self.assertEqual(v, "Local variable") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_assignment(self): with self.assertRaisesRegex(SyntaxError, "cannot assign"): compile("{x: y for y, x in ((1, 2), (3, 4))} = 5", "", diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b68e356e0..baa638e07 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -865,7 +865,6 @@ x = ( x = X() self.assertEqual(f'{x} {x}', '1 2') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_expression(self): self.assertAllRaise(SyntaxError, "f-string: valid expression required before '}'", diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index a859e051d..cf4408067 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -98,7 +98,6 @@ class NamedExpressionInvalidTest(unittest.TestCase): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_expression_invalid_17(self): code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]" diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index d3cb80bbb..51a4a8155 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -326,7 +326,7 @@ SyntaxError: did you forget parentheses around the comprehension target? # Incorrectly closed strings ->>> "The interesting object "The important object" is very important" # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> "The interesting object "The important object" is very important" Traceback (most recent call last): SyntaxError: invalid syntax. Is this intended to be part of the string? @@ -423,7 +423,7 @@ SyntaxError: invalid syntax Traceback (most recent call last): SyntaxError: invalid syntax ->>> def foo(/,a,b=,c): # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> def foo(/,a,b=,c): ... pass Traceback (most recent call last): SyntaxError: at least one argument must precede / @@ -1045,7 +1045,7 @@ This tests assignment-context; there was a bug in Python 2.5 where compiling a complex 'if' (one with 'elif') would fail to notice an invalid suite, leading to spurious errors. - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... x() = 1 ... elif 1: ... pass @@ -1053,7 +1053,7 @@ leading to spurious errors. ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... x() = 1 @@ -1061,7 +1061,7 @@ leading to spurious errors. ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... x() = 1 ... elif 1: ... pass @@ -1071,7 +1071,7 @@ leading to spurious errors. ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... x() = 1 @@ -1081,7 +1081,7 @@ leading to spurious errors. ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... pass @@ -1185,7 +1185,7 @@ Missing ':' before suites: Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech as something) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech as something) ... pass Traceback (most recent call last): SyntaxError: expected ':' @@ -1195,12 +1195,12 @@ Missing ':' before suites: Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech, block as something) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech, block as something) ... pass Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech, block as something, bluch) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech, block as something, bluch) ... pass Traceback (most recent call last): SyntaxError: expected ':' @@ -1464,7 +1464,7 @@ Ensure that early = are not matched by the parser as invalid comparisons Traceback (most recent call last): SyntaxError: invalid syntax - >>> dict(x=34, (x for x in range 10), 1); x $ y # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> dict(x=34, (x for x in range 10), 1); x $ y Traceback (most recent call last): SyntaxError: invalid syntax @@ -2077,7 +2077,7 @@ SyntaxError: cannot use literal as import target Traceback (most recent call last): SyntaxError: cannot use literal as import target ->>> from a import (b as c.d) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import (b as c.d) Traceback (most recent call last): SyntaxError: cannot use attribute as import target @@ -2085,18 +2085,18 @@ SyntaxError: cannot use attribute as import target Traceback (most recent call last): SyntaxError: cannot use literal as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b as f()) Traceback (most recent call last): SyntaxError: cannot use function call as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b as [], ... ) Traceback (most recent call last): SyntaxError: cannot use list as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b, ... c as () ... ) @@ -2233,7 +2233,7 @@ Corner-cases that used to crash: Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - >>> import ä £ # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> import ä £ Traceback (most recent call last): SyntaxError: invalid character '£' (U+00A3) @@ -2245,7 +2245,7 @@ Corner-cases that used to crash: Traceback (most recent call last): SyntaxError: cannot use '_' as a target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case 42 as 1+2+4: ... ... Traceback (most recent call last): @@ -2263,13 +2263,13 @@ Corner-cases that used to crash: Traceback (most recent call last): SyntaxError: cannot use tuple as pattern target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case 42 as (a + 1): ... ... Traceback (most recent call last): SyntaxError: cannot use expression as pattern target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case (32 as x) | (42 as a()): ... ... Traceback (most recent call last): @@ -2375,7 +2375,7 @@ A[*(1:2)] A[*:] and A[:*] - >>> A[*:] # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> A[*:] Traceback (most recent call last): ... SyntaxError: Invalid star expression @@ -2386,7 +2386,7 @@ A[*:] and A[:*] A[*] - >>> A[*] # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> A[*] Traceback (most recent call last): ... SyntaxError: Invalid star expression diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index d7a77eb26..11b3f8ea0 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -217,7 +217,6 @@ class TestCheck(TestCase): with self.assertRaises(SystemExit): self.verify_tabnanny_check(file_path, err=err) - @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -231,7 +230,6 @@ class TestCheck(TestCase): tabnanny.verbose = 1 self.verify_tabnanny_check(file_path, out=out) - @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error(self): """A python source code file eligible for raising `tabnanny.NannyNag`.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: @@ -341,7 +339,6 @@ class TestCommandLine(TestCase): stdout = f"{file_path}\n" self.validate_cmd("-q", file_path, stdout=stdout) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_verbose_mode(self): """Should display more error information if verbose mode is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: @@ -350,7 +347,6 @@ class TestCommandLine(TestCase): ).strip() self.validate_cmd("-v", path, stdout=stdout, partial=True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_double_verbose_mode(self): """Should display detailed error information if double verbose is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 0e81c6f6d..6649de682 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3193,7 +3193,6 @@ async def f(): with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: IndentationError not raised by @support.skip_wasi_stack_overflow() def test_max_indent(self): MAXINDENT = 100 diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 27749a080..3680072d6 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -17,7 +17,6 @@ class PEP3131Test(unittest.TestCase): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 self.assertIn("Unicode", dir()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid(self): try: from test.tokenizedata import badsyntax_3131 # noqa: F401 diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index a5a200259..6f49e62e8 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -361,7 +361,7 @@ Now some general starred expressions (all fail). ... SyntaxError: cannot use starred expression here - >>> (((*x))),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + >>> (((*x))),y = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here @@ -376,7 +376,7 @@ Now some general starred expressions (all fail). ... SyntaxError: cannot use starred expression here - >>> ((*x),y) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + >>> ((*x),y) = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c2b6446ee..3aea9089d 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -8604,14 +8604,15 @@ impl<'warnings> Compiler<'warnings> { self.use_cpython_label_block(end); Ok(()) } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - .. - }) if ops.len() > 1 => { + ast::Expr::Compare(compare) if compare.ops.len() > 1 => { self.set_source_range(expression.range()); - self.compile_jump_if_compare(left, ops, comparators, condition, target_block) + self.compile_jump_if_compare( + compare.first_operand(), + &compare.ops, + compare.comparators(), + condition, + target_block, + ) } _ => { // Fall back case which always will work! @@ -8940,13 +8941,8 @@ impl<'warnings> Compiler<'warnings> { self.emit_load_attr(idx); } } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - .. - }) => { - self.compile_compare(left, ops, comparators)?; + ast::Expr::Compare(compare) => { + self.compile_compare(compare.first_operand(), &compare.ops, compare.comparators())?; } ast::Expr::Constant(ast::ExprConstant { value, .. }) => { self.emit_load_const(ast_constant_value_to_constant_data(value.clone())); @@ -12521,7 +12517,11 @@ impl<'warnings> Compiler<'warnings> { ast::Expr::Await(_) => self.found = true, ast::Expr::ListComp(ast::ExprListComp { generators, .. }) | ast::Expr::SetComp(ast::ExprSetComp { generators, .. }) - | ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) + if generators.iter().any(|generator| generator.is_async) => + { + self.found = true + } + ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) if generators.iter().any(|generator| generator.is_async) => { self.found = true @@ -12556,16 +12556,16 @@ impl<'warnings> Compiler<'warnings> { fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { let fstring_range = fstring.range; - let fstring = fstring.value.as_slice(); - if self.count_fstring_parts(fstring) > STACK_USE_GUIDELINE { - return self.compile_fstring_parts_joined(fstring, fstring_range); + let value = &fstring.value; + if self.count_fstring_parts(value) > STACK_USE_GUIDELINE { + return self.compile_fstring_parts_joined(value, fstring_range); } let mut element_count = 0; let mut pending_literal = None; let mut pending_literal_range = None; let mut pending_literal_no_location = false; - for part in fstring { + for part in value { self.compile_fstring_part_into( part, &mut pending_literal, @@ -12629,7 +12629,7 @@ impl<'warnings> Compiler<'warnings> { fn compile_fstring_parts_joined( &mut self, - fstring: &[ast::FStringPart], + fstring: &ast::FStringValue, fstring_range: TextRange, ) -> CompileResult<()> { self.set_source_range(fstring_range); @@ -12666,7 +12666,7 @@ impl<'warnings> Compiler<'warnings> { fn compile_fstring_part_into( &mut self, - part: &ast::FStringPart, + part: ast::FStringPartRef<'_>, pending_literal: &mut Option, pending_literal_range: &mut Option, pending_literal_no_location: &mut bool, @@ -12674,7 +12674,7 @@ impl<'warnings> Compiler<'warnings> { join_append_range: Option, ) -> CompileResult<()> { match part { - ast::FStringPart::Literal(string) => { + ast::FStringPartRef::Literal(string) => { let value = string_literal_part_value(&self.source_file, string); if pending_literal.is_none() { *pending_literal_range = Some(string.range); @@ -12687,7 +12687,7 @@ impl<'warnings> Compiler<'warnings> { } Ok(()) } - ast::FStringPart::FString(fstring) => self.compile_fstring_elements_into( + ast::FStringPartRef::FString(fstring) => self.compile_fstring_elements_into( fstring.flags, &fstring.elements, pending_literal, @@ -12801,7 +12801,7 @@ impl<'warnings> Compiler<'warnings> { } } - fn count_fstring_parts(&self, fstring: &[ast::FStringPart]) -> u32 { + fn count_fstring_parts(&self, fstring: &ast::FStringValue) -> u32 { let mut element_count = 0; let mut pending_literal = None; for part in fstring { @@ -12813,12 +12813,12 @@ impl<'warnings> Compiler<'warnings> { fn count_fstring_part_into( &self, - part: &ast::FStringPart, + part: ast::FStringPartRef<'_>, pending_literal: &mut Option, element_count: &mut u32, ) { match part { - ast::FStringPart::Literal(string) => { + ast::FStringPartRef::Literal(string) => { let value = string_literal_part_value(&self.source_file, string); if let Some(pending) = pending_literal.as_mut() { pending.push_wtf8(value.as_ref()); @@ -12826,7 +12826,7 @@ impl<'warnings> Compiler<'warnings> { *pending_literal = Some(value); } } - ast::FStringPart::FString(fstring) => self.count_fstring_elements_into( + ast::FStringPartRef::FString(fstring) => self.count_fstring_elements_into( fstring.flags, &fstring.elements, pending_literal, @@ -13972,7 +13972,7 @@ mod tests { arguments: ast::Arguments { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - args: Box::default(), + args: Default::default(), keywords: Default::default(), runtime_args: None, runtime_bases: None, @@ -14120,7 +14120,7 @@ mod tests { arguments: ast::Arguments { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - args: Box::default(), + args: Default::default(), keywords: Default::default(), runtime_args: None, runtime_bases: None, @@ -14196,12 +14196,14 @@ mod tests { let message = first_ast_constant_warning(ast::Expr::Compare(ast::ExprCompare { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - left: Box::new(left), ops: Box::new([ast::CmpOp::Is]), - comparators: Box::new([ast::Expr::NoneLiteral(ast::ExprNoneLiteral { - node_index: ast::AtomicNodeIndex::NONE, - range: TextRange::default(), - })]), + operands: Box::new([ + left, + ast::Expr::NoneLiteral(ast::ExprNoneLiteral { + node_index: ast::AtomicNodeIndex::NONE, + range: TextRange::default(), + }), + ]), runtime_comparators: None, })); assert!( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index bff326087..56512f4bb 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -2207,11 +2207,9 @@ impl SymbolTableBuilder { Expr::BoolOp(ExprBoolOp { values, .. }) => { self.scan_expressions(values, context)?; } - Expr::Compare(ExprCompare { - left, comparators, .. - }) => { - self.scan_expression(left, context)?; - self.scan_expressions(comparators, context)?; + Expr::Compare(compare) => { + self.scan_expression(compare.first_operand(), context)?; + self.scan_expressions(compare.comparators(), context)?; } Expr::Subscript(ExprSubscript { value, slice, .. }) => { self.scan_expression(value, ExpressionContext::Load)?; diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 86744a601..3ada6644c 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -380,18 +380,11 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { UnparseExpr::new(value, self.source) )?; } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - node_index: _, - range: _range, - .. - }) => { + ast::Expr::Compare(compare) => { group_if!(precedence::CMP, { let new_lvl = precedence::CMP + 1; - self.unparse_expr(left, new_lvl)?; - for (op, cmp) in ops.iter().zip(comparators) { + self.unparse_expr(compare.first_operand(), new_lvl)?; + for (_, op, cmp) in compare.iter() { self.p(" ")?; self.p(op.as_str())?; self.p(" ")?; @@ -700,8 +693,8 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p("f")?; let body = fmt::from_fn(|f| { value.iter().try_for_each(|part| match part { - ast::FStringPart::Literal(lit) => f.write_str(lit), - ast::FStringPart::FString(ast::FString { elements, .. }) => { + ast::FStringPartRef::Literal(lit) => f.write_str(lit), + ast::FStringPartRef::FString(ast::FString { elements, .. }) => { Unparser::new(f, self.source).unparse_fstring_body(elements) } }) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 534173d2c..71c2dfad9 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -63,7 +63,21 @@ impl CompileError { let raw_location = error.location; let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, - None => default_parse_diagnostic(error, source_file), + None => { + let is_unclosed_bracket = matches!( + error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }) + ); + let is_unclosed_string = + unclosed_string_is_continuable(&error, source_file.source_text(), mode); + let mut diagnostic = default_parse_diagnostic(error, source_file); + diagnostic.is_unclosed_bracket = is_unclosed_bracket; + diagnostic.is_unclosed_string = is_unclosed_string; + diagnostic + } }; Self::Parse(ParseError { @@ -89,8 +103,8 @@ impl CompileError { location, end_location, source_path: source_file.name().to_owned(), - is_unclosed_bracket: diagnostic.is_unclosed_bracket, - is_unclosed_string: diagnostic.is_unclosed_string, + is_unclosed_bracket: false, + is_unclosed_string: false, }) } @@ -182,22 +196,6 @@ impl NormalizedParseDiagnostic { } } - fn other(source_file: &SourceFile, diagnostic: CpythonDiagnostic) -> Self { - let (location, end_location) = source_locations( - source_file, - diagnostic.range.start(), - diagnostic.range.end(), - ); - let mut diagnostic_out = Self::new( - parser::ParseErrorType::OtherError(diagnostic.message), - location, - end_location, - ); - diagnostic_out.is_unclosed_string = diagnostic.is_unclosed_string; - diagnostic_out.is_unclosed_bracket = diagnostic.is_unclosed_bracket; - diagnostic_out - } - const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self { self.is_unclosed_bracket = is_unclosed_bracket; self @@ -212,8 +210,6 @@ impl NormalizedParseDiagnostic { struct CpythonDiagnostic { message: String, range: ruff_text_size::TextRange, - is_unclosed_string: bool, - is_unclosed_bracket: bool, } impl CpythonDiagnostic { @@ -228,66 +224,8 @@ impl CpythonDiagnostic { TextSize::new(start as u32), TextSize::new(end as u32), ), - is_unclosed_string: false, - is_unclosed_bracket: false, } } - - const fn with_unclosed_string(mut self) -> Self { - self.is_unclosed_string = true; - self - } - - const fn with_unclosed_bracket(mut self) -> Self { - self.is_unclosed_bracket = true; - self - } -} - -/// Lexer-class failures outrank print hints. Decode and f-string diagnostics -/// compete with lexer failures by offset, and an unterminated quote is only -/// a fallback when no decode/f-string diagnostic exists. Print is considered -/// against the final winner: a later 0x still beats print, but an f-string -/// that replaced that 0x must not hide an earlier print. A decode diagnostic -/// also suppresses an EOF unclosed opener. -#[derive(Clone, Copy, PartialEq, Eq)] -enum OverrideClass { - Lexer, - Decode, - Print, -} - -struct RankedOverride { - diagnostic: CpythonDiagnostic, - unclosed_bracket: bool, - class: OverrideClass, -} - -fn consider_override( - best: &mut Option, - diagnostic: CpythonDiagnostic, - class: OverrideClass, -) { - let unclosed_bracket = diagnostic.is_unclosed_bracket; - consider_ranked(best, diagnostic, unclosed_bracket, class); -} - -fn consider_ranked( - best: &mut Option, - diagnostic: CpythonDiagnostic, - unclosed_bracket: bool, - class: OverrideClass, -) { - if best - .as_ref() - .is_none_or(|current| diagnostic.range.start() < current.diagnostic.range.start()) - { - *best = Some(RankedOverride { - diagnostic, - unclosed_bracket, - class, - }); - } } fn cpython_parse_diagnostic_override( @@ -297,109 +235,42 @@ fn cpython_parse_diagnostic_override( ) -> Option { let source_text = source_file.source_text(); - macro_rules! source_error { - ($expr:expr) => { - if let Some(error) = $expr { - return Some(NormalizedParseDiagnostic::other(source_file, error)); - } - }; - } - - let mut earliest: Option = None; - if let Some(diagnostic) = invalid_number_literal_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - if let Some(diagnostic) = incompatible_string_prefix_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - if let Some(diagnostic) = non_printable_character_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - let bracket = bracket_syntax_error(source_text); - if let Some(bracket) = bracket.as_ref() { - // Unclosed openers are reported at the opener and only become errors - // at EOF. A later token-time diagnostic (invalid number, prefix, …) - // must keep winning. Mismatched closers stay in the lexer-class - // positional ranking. - if !bracket.unclosed { - consider_ranked( - &mut earliest, - bracket.diagnostic.clone(), - false, - OverrideClass::Lexer, - ); - } - } - let mut saw_decode = false; - if let Some(diagnostic) = malformed_unicode_n_escape_error(source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - if let Some(diagnostic) = invalid_interpolated_string_error(source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - if let Some(diagnostic) = mixed_tstring_literal_error(error, source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - // A later ordinary unterminated quote is only a fallback. Format-spec - // newlines and empty fields are decode diagnostics and must keep winning. + let tokenizer_error = matches!( + &error.error, + parser::ParseErrorType::Lexical(lexical) if lexical.is_tokenizer_error() + ); let line_continuation = matches!( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) ); - if !saw_decode - && !line_continuation - && let Some(diagnostic) = unterminated_string_error(source_text, mode) - { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - // Indentation errors outrank a print/exec missing-parentheses rewrite. - let indent_error = matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) - | parser::ParseErrorType::UnexpectedIndentation - ) || expected_indented_block_error(error, source_text).is_some(); - if !indent_error - && earliest - .as_ref() - .is_none_or(|current| current.class != OverrideClass::Lexer) - && let Some(diagnostic) = invalid_legacy_statement_error(source_text) - { - consider_override(&mut earliest, diagnostic, OverrideClass::Print); - } - if !saw_decode - && earliest - .as_ref() - .is_none_or(|current| current.class == OverrideClass::Print) - && let Some(bracket) = bracket.filter(|bracket| bracket.unclosed) - { - consider_ranked( - &mut earliest, - bracket.diagnostic, - true, - OverrideClass::Lexer, - ); - } - if let Some(override_diag) = earliest { - return Some( - NormalizedParseDiagnostic::other(source_file, override_diag.diagnostic) - .with_unclosed_bracket(override_diag.unclosed_bracket), - ); - } - - if matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) - ) { + if line_continuation { // exec input gets an implicit trailing newline, so a final `\` is a // continuation that then hits EOF (`E_EOF`). single/eval see `\` at // EOF as `E_LINECONT` instead. - let terminal_backslash = source_text.len().checked_sub(1); - if matches!(mode, Mode::Exec) - && terminal_backslash == Some(error.location.start().to_usize()) - { + if matches!(mode, Mode::Exec) && error.location.start().to_usize() == source_text.len() { + let source_with_newline = format!("{source_text}\n"); + let parsed = parser::parse_unchecked( + &source_with_newline, + parser::ParseOptions::from(parser::Mode::Module), + ); + if let Some(unclosed) = parsed.errors().first().filter(|first| { + matches!( + first.error, + parser::ParseErrorType::Lexical( + parser::LexicalErrorType::UnclosedBracket { .. } + ) + ) + }) { + let mut diagnostic = default_parse_diagnostic(unclosed.clone(), source_file); + diagnostic.is_unclosed_bracket = matches!( + unclosed.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }) + ); + return Some(diagnostic); + } let loc = source_line_end_location(source_file, error.location.start()); return Some(NormalizedParseDiagnostic::new( parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()), @@ -407,126 +278,22 @@ fn cpython_parse_diagnostic_override( loc, )); } - let loc = source_location(source_file, error.location.start() + TextSize::from(1)); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError( - "unexpected character after line continuation character".to_owned(), - ), - loc, - loc, - )); + return None; } - source_error!(unterminated_string_error(source_text, mode)); - source_error!(expected_indented_block_error(error, source_text)); - if matches!( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::Eof) ) { return Some(eof_parse_diagnostic(error, source_file)); } - - source_error!(invalid_type_param_error(source_text)); - source_error!(invalid_comprehension_error(source_text)); - source_error!(invalid_parameter_star_annotation_error(source_text)); - source_error!(invalid_parameter_list_error(source_text)); - source_error!(invalid_call_argument_error(source_text)); - - if is_missing_comma_between_literals(error) { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let msg = "invalid syntax. Perhaps you forgot a comma?".into(); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError(msg), - loc, - end_loc, - )); - } - - source_error!(invalid_dict_error(source_text)); - if !matches!(mode, Mode::Eval) { - source_error!(invalid_collection_assignment_error(source_text)); - } - source_error!(invalid_group_error(source_text)); - source_error!(invalid_def_type_params_error(source_text)); - source_error!(invalid_expression_error(source_text)); - source_error!(invalid_named_expression_error(source_text)); - // Assignment and other statements are not expressions. Eval keeps the - // generic parse error rather than a statement-target rewrite. - if !matches!(mode, Mode::Eval) { - source_error!(invalid_plain_assignment_error(source_text)); - source_error!(expression_assignment_error(source_text)); - source_error!(invalid_annotation_target_error(source_text)); - source_error!(invalid_assignment_target_error(source_text)); - source_error!(invalid_condition_assignment_error( - source_text, - error.location.start().to_usize() - )); - source_error!(invalid_augassign_target_error(source_text)); - source_error!(invalid_for_target_error(source_text)); - source_error!(invalid_with_target_error(source_text)); - source_error!(invalid_delete_target_error(source_text)); - source_error!(invalid_standalone_except_error(source_text)); - source_error!(invalid_import_statement_error(source_text)); - source_error!(invalid_import_target_error(source_text)); - source_error!(invalid_except_as_target_error(source_text)); - source_error!(invalid_match_mapping_rest_wildcard_error(source_text)); - source_error!(invalid_match_as_target_error(source_text)); - source_error!(invalid_for_if_clause_error(source_text)); - source_error!(invalid_if_expression_statement_error(source_text)); - source_error!(invalid_else_elif_error(source_text)); - source_error!(mixed_except_handlers_error(source_text)); - } - - if matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::InvalidByteLiteral) - ) && let Some((start, end)) = - bytes_literal_span(source_text, error.location.start().to_usize()) + if tokenizer_error + || matches!( + &error.error, + parser::ParseErrorType::ExpectedIndentedBlock { .. } + ) { - let (loc, end_loc) = source_locations( - source_file, - TextSize::new(start as u32), - TextSize::new(end as u32), - ); - return Some(NormalizedParseDiagnostic::new( - error.error.clone(), - loc, - end_loc, - )); - } - - if matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) - ) { - let end_loc = source_line_end_location(source_file, error.location.start()); - return Some(NormalizedParseDiagnostic::new( - error.error.clone(), - end_loc, - end_loc, - )); - } - - if matches!( - &error.error, - parser::ParseErrorType::InvalidAssignmentTarget - ) { - return Some(invalid_assignment_target_diagnostic(error, source_file)); - } - - if matches!( - &error.error, - parser::ParseErrorType::InvalidNamedAssignmentTarget - ) { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let target = source_file.source_text().slice(error.location); - let msg = format!("cannot use assignment expressions with {target}"); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError(msg), - loc, - end_loc, - )); + return None; } // CPython's PEG parser collapses a bare "expected an expression" failure @@ -569,41 +336,52 @@ fn eof_parse_diagnostic( } } -fn invalid_assignment_target_diagnostic( - error: &parser::ParseError, - source_file: &SourceFile, -) -> NormalizedParseDiagnostic { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let expr_str = source_file.source_text().slice(error.location); +/// Returns whether more input could still terminate the string that `error` reports as +/// unterminated. +/// +/// A string that reached the end of the source can continue, except a single-quoted f/t-string, +/// a single-quoted string in exec input whose implicit final newline ends it, and a string after +/// an assignment in eval input. +fn unclosed_string_is_continuable(error: &parser::ParseError, source: &str, mode: Mode) -> bool { + use parser::{InterpolatedStringErrorType, LexicalErrorType}; - let msg = parser::parse_expression(expr_str).map_or_else( - |_| match expr_str { - "yield" => "assignment to yield expression not possible".into(), - _ => format!("cannot assign to {expr_str}"), - }, - |parsed| match *parsed.syntax().body { - ast::Expr::Call(_) => "cannot assign to function call".into(), - ast::Expr::BinOp(_) => "cannot assign to expression".into(), - ast::Expr::If(_) => "cannot assign to conditional expression".into(), - ast::Expr::Generator(_) => "cannot assign to generator expression".into(), - ast::Expr::Yield(_) | ast::Expr::YieldFrom(_) => { - "cannot assign to yield expression here. Maybe you meant '==' instead of '='?" - .into() - } - ast::Expr::FString(_) => "invalid syntax".into(), - ast::Expr::StringLiteral(_) - | ast::Expr::BytesLiteral(_) - | ast::Expr::NumberLiteral(_) => { - "cannot assign to literal here. Maybe you meant '==' instead of '='?".into() - } - ast::Expr::EllipsisLiteral(_) => { - "cannot assign to ellipsis here. Maybe you meant '==' instead of '='?".into() - } - _ => format!("cannot assign to {expr_str}"), - }, - ); + let parser::ParseErrorType::Lexical(lexical) = &error.error else { + return false; + }; + let start = error.location.start().to_usize(); + let at_eof = match lexical { + LexicalErrorType::UnclosedStringError { + triple_quoted: true, + .. + } + | LexicalErrorType::FStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) + | LexicalErrorType::TStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) => true, + LexicalErrorType::UnclosedStringError { .. } => { + // Exec mode appends a newline, which only a final `\` can escape. + let rest = &source[start..]; + !has_unescaped_newline(rest) && (!matches!(mode, Mode::Exec) || rest.ends_with('\\')) + } + _ => false, + }; + at_eof && !(matches!(mode, Mode::Eval) && eval_has_assignment_before(source.as_bytes(), start)) +} - NormalizedParseDiagnostic::new(parser::ParseErrorType::OtherError(msg), loc, end_loc) +fn has_unescaped_newline(text: &str) -> bool { + let mut chars = text.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '\n' | '\r' => return true, + _ => {} + } + } + false } fn default_parse_diagnostic( @@ -646,14 +424,6 @@ fn source_line_end_location(source_file: &SourceFile, offset: TextSize) -> Sourc } } -fn is_missing_comma_between_literals(error: &parser::ParseError) -> bool { - matches!( - &error.error, - parser::ParseErrorType::ExpectedToken { expected, found } - if matches!((expected, found), (TokenKind::Comma, TokenKind::Int)) - ) -} - fn is_ascii_identifier_char(byte: u8) -> bool { byte == b'_' || byte.is_ascii_alphanumeric() } @@ -675,298 +445,6 @@ fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) } -fn numeric_keyword_suffix(rest: &[u8]) -> bool { - rest.starts_with(b"and") - || rest.starts_with(b"else") - || rest.starts_with(b"for") - || rest.starts_with(b"if") - || rest.starts_with(b"in") - || rest.starts_with(b"is") - || rest.starts_with(b"or") - || rest.starts_with(b"not") -} - -fn consume_decimal_digits(bytes: &[u8], mut index: usize) -> usize { - while index < bytes.len() { - match bytes[index] { - b'0'..=b'9' => index += 1, - b'_' if bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) => - { - index += 2; - } - _ => break, - } - } - index -} - -fn consume_radix_digits(bytes: &[u8], mut index: usize, is_digit: impl Fn(u8) -> bool) -> usize { - while index < bytes.len() { - if is_digit(bytes[index]) { - index += 1; - } else if bytes.get(index) == Some(&b'_') - && bytes.get(index + 1).is_some_and(|&byte| is_digit(byte)) - { - index += 2; - } else { - break; - } - } - index -} - -fn invalid_radix_literal_error( - bytes: &[u8], - start: usize, - kind: &'static str, - is_digit: impl Fn(u8) -> bool, -) -> Option<(String, usize)> { - let mut index = start + 2; - let mut has_digit = false; - loop { - let Some(&byte) = bytes.get(index) else { - return if has_digit { - None - } else { - Some((format!("invalid {kind} literal"), start + 1)) - }; - }; - if byte == b'_' { - let Some(&next) = bytes.get(index + 1) else { - return Some((format!("invalid {kind} literal"), index)); - }; - if is_digit(next) { - has_digit = true; - index += 2; - continue; - } - if next.is_ascii_digit() && matches!(kind, "binary" | "octal") { - return Some(( - format!("invalid digit '{}' in {kind} literal", next as char), - index + 1, - )); - } - return Some((format!("invalid {kind} literal"), index)); - } - if is_digit(byte) { - has_digit = true; - index += 1; - continue; - } - if byte.is_ascii_digit() && matches!(kind, "binary" | "octal") { - return Some(( - format!("invalid digit '{}' in {kind} literal", byte as char), - index, - )); - } - if has_digit { - return None; - } - return Some((format!("invalid {kind} literal"), start + 1)); - } -} - -fn decimal_tail_error(bytes: &[u8], mut index: usize) -> Option { - loop { - while bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - index += 1; - } - if bytes.get(index) != Some(&b'_') { - return None; - } - let underscore = index; - index += 1; - if !bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - return Some(underscore); - } - } -} - -fn decimal_tail_end(bytes: &[u8], mut index: usize) -> usize { - loop { - while bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - index += 1; - } - if bytes.get(index) == Some(&b'_') - && bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) - { - index += 2; - } else { - return index; - } - } -} - -fn invalid_decimal_literal_error(bytes: &[u8], start: usize) -> Option<(String, usize)> { - if bytes.get(start) == Some(&b'.') { - return None; - } - let message = "invalid decimal literal".to_owned(); - if let Some(offset) = decimal_tail_error(bytes, start) { - return Some((message, offset)); - } - - let mut index = decimal_tail_end(bytes, start); - if bytes.get(index) == Some(&b'.') { - if bytes.get(index + 1) == Some(&b'_') { - return Some((message, index)); - } - if let Some(offset) = decimal_tail_error(bytes, index + 1) { - return Some((message, offset)); - } - index = decimal_tail_end(bytes, index + 1); - } - if matches!(bytes.get(index), Some(b'e' | b'E')) { - let exponent = index; - index += 1; - let sign = if matches!(bytes.get(index), Some(b'+' | b'-')) { - let sign = index; - index += 1; - Some(sign) - } else { - None - }; - if !bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - // Without a sign the exponent letter is put back, so the position - // is the digit before it rather than the letter. - return Some((message, sign.unwrap_or_else(|| exponent.saturating_sub(1)))); - } - if let Some(offset) = decimal_tail_error(bytes, index) { - return Some((message, offset)); - } - } - None -} - -fn leading_zero_decimal_literal_error(bytes: &[u8], start: usize) -> Option { - if bytes.get(start) != Some(&b'0') { - return None; - } - let mut index = start; - loop { - match bytes.get(index) { - Some(b'0') => index += 1, - Some(b'_') - if bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) => - { - index += 1; - } - _ => break, - } - } - if bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - let after_digits = decimal_tail_end(bytes, index); - if !matches!( - bytes.get(after_digits), - Some(b'.' | b'e' | b'E' | b'j' | b'J') - ) { - return Some(CpythonDiagnostic::new( - "leading zeros in decimal integer literals are not permitted; use an 0o prefix for octal integers" - .to_owned(), - start, - index, - )); - } - } - None -} - -fn invalid_numeric_literal_error(bytes: &[u8], start: usize) -> Option { - if bytes.get(start) == Some(&b'0') { - let radix = match bytes.get(start + 1) { - Some(b'x' | b'X') => invalid_radix_literal_error(bytes, start, "hexadecimal", |byte| { - byte.is_ascii_hexdigit() - }), - Some(b'o' | b'O') => invalid_radix_literal_error(bytes, start, "octal", |byte| { - matches!(byte, b'0'..=b'7') - }), - Some(b'b' | b'B') => invalid_radix_literal_error(bytes, start, "binary", |byte| { - matches!(byte, b'0' | b'1') - }), - _ => None, - }; - if let Some(radix) = radix { - return Some(point_span(radix)); - } - if let Some(err) = leading_zero_decimal_literal_error(bytes, start) { - return Some(err); - } - } - invalid_decimal_literal_error(bytes, start).map(point_span) -} - -fn consume_exponent(bytes: &[u8], index: usize) -> usize { - if !matches!(bytes.get(index), Some(b'e' | b'E')) { - return index; - } - let mut cursor = index + 1; - if matches!(bytes.get(cursor), Some(b'+' | b'-')) { - cursor += 1; - } - if bytes.get(cursor).is_some_and(|byte| byte.is_ascii_digit()) { - consume_decimal_digits(bytes, cursor) - } else { - index - } -} - -fn number_literal_end(bytes: &[u8], start: usize) -> Option<(&'static str, usize)> { - if bytes.get(start) == Some(&b'.') { - if !bytes - .get(start + 1) - .is_some_and(|byte| byte.is_ascii_digit()) - { - return None; - } - let mut index = consume_decimal_digits(bytes, start + 1); - index = consume_exponent(bytes, index); - if matches!(bytes.get(index), Some(b'j' | b'J')) { - return Some(("imaginary", index + 1)); - } - return Some(("decimal", index)); - } - - if !bytes.get(start).is_some_and(|byte| byte.is_ascii_digit()) { - return None; - } - - if bytes.get(start) == Some(&b'0') { - match bytes.get(start + 1) { - Some(b'x' | b'X') => { - let end = consume_radix_digits(bytes, start + 2, |byte| byte.is_ascii_hexdigit()); - return Some(("hexadecimal", end)); - } - Some(b'o' | b'O') => { - let end = - consume_radix_digits(bytes, start + 2, |byte| matches!(byte, b'0'..=b'7')); - return Some(("octal", end)); - } - Some(b'b' | b'B') => { - let end = - consume_radix_digits(bytes, start + 2, |byte| matches!(byte, b'0' | b'1')); - return Some(("binary", end)); - } - _ => {} - } - } - - let mut index = consume_decimal_digits(bytes, start); - if bytes.get(index) == Some(&b'.') { - index = consume_decimal_digits(bytes, index + 1); - } - index = consume_exponent(bytes, index); - if matches!(bytes.get(index), Some(b'j' | b'J')) { - return Some(("imaginary", index + 1)); - } - Some(("decimal", index)) -} - fn quoted_string_is_closed(bytes: &[u8], start: usize) -> bool { let quote = bytes[start]; let triple = bytes.get(start + 1) == Some("e) && bytes.get(start + 2) == Some("e); @@ -981,32 +459,6 @@ fn quoted_string_is_closed(bytes: &[u8], start: usize) -> bool { } } -fn bytes_literal_span(source: &str, error_at: usize) -> Option<(usize, usize)> { - let bytes = source.as_bytes(); - if error_at > bytes.len() || bytes.is_empty() { - return None; - } - let mut quote_idx = error_at.min(bytes.len().saturating_sub(1)); - loop { - if matches!(bytes[quote_idx], b'\'' | b'"') { - break; - } - if quote_idx == 0 { - return None; - } - quote_idx -= 1; - } - let mut start = quote_idx; - while start > 0 && matches!(bytes[start - 1], b'b' | b'B' | b'r' | b'R') { - start -= 1; - } - if !matches!(bytes.get(start), Some(b'b' | b'B' | b'r' | b'R')) { - return None; - } - let end = skip_quoted_string(bytes, quote_idx); - Some((start, end)) -} - fn skip_quoted_string(bytes: &[u8], mut index: usize) -> usize { let quote = bytes[index]; let triple = bytes.get(index + 1) == Some("e) && bytes.get(index + 2) == Some("e); @@ -1030,178 +482,6 @@ fn skip_quoted_string(bytes: &[u8], mut index: usize) -> usize { index } -// An error the tokenizer reports at a single position spans nothing. -fn point_span((message, offset): (String, usize)) -> CpythonDiagnostic { - CpythonDiagnostic::new(message, offset, offset) -} - -fn invalid_number_literal_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - byte if byte >= 0x80 || byte == b'_' || byte.is_ascii_alphabetic() => { - index += 1; - while index < bytes.len() - && (bytes[index] >= 0x80 || is_ascii_identifier_char(bytes[index])) - { - index += 1; - } - } - b'.' | b'0'..=b'9' => { - if let Some(err) = invalid_numeric_literal_error(bytes, index) { - return Some(err); - } - let Some((kind, end)) = number_literal_end(bytes, index) else { - index += 1; - continue; - }; - if end > index { - if source[end..].starts_with('⁄') { - return Some(CpythonDiagnostic::new( - "invalid character '⁄' (U+2044)".to_owned(), - end, - end, - )); - } - if bytes - .get(end) - .is_some_and(|byte| *byte < 128 && is_ascii_identifier_char(*byte)) - && !numeric_keyword_suffix(&bytes[end..]) - { - let offset = end.saturating_sub(1); - return Some(CpythonDiagnostic::new( - format!("invalid {kind} literal"), - offset, - offset, - )); - } - } - index = end.max(index + 1); - } - _ => index += 1, - } - } - None -} - -fn cpython_indented_block_clause(message: &str) -> Option<&'static str> { - let clause = message.strip_prefix("Expected an indented block after ")?; - Some(match clause { - "`if` statement" => "'if' statement", - "`elif` clause" => "'elif' statement", - "`else` clause" => "'else' statement", - "`for` statement" => "'for' statement", - "`with` statement" => "'with' statement", - "`while` statement" => "'while' statement", - "`try` statement" => "'try' statement", - "`except` clause" => "'except' statement", - "`finally` clause" => "'finally' statement", - "`match` statement" => "'match' statement", - "`case` block" => "'case' statement", - "`class` definition" => "class definition", - "function definition" => "function definition", - _ => return None, - }) -} - -fn previous_non_empty_line_number(source: &str, offset: usize) -> Option { - let bytes = source.as_bytes(); - let mut index = offset.min(bytes.len()); - while index > 0 { - let line_end = index; - while index > 0 && bytes[index - 1] != b'\n' { - index -= 1; - } - let line_start = index; - let content_start = skip_horizontal_whitespace(bytes, line_start); - let mut content_end = line_end; - while content_end > content_start - && matches!( - bytes.get(content_end - 1), - Some(b' ' | b'\t' | b'\r' | b'\x0c') - ) - { - content_end -= 1; - } - if content_start < content_end { - return Some( - source[..line_start] - .bytes() - .filter(|byte| *byte == b'\n') - .count() - + 1, - ); - } - index = line_start.saturating_sub(1); - } - None -} - -fn expected_indented_block_error( - error: &parser::ParseError, - source: &str, -) -> Option { - let parser::ParseErrorType::OtherError(message) = &error.error else { - return None; - }; - let mut clause = cpython_indented_block_clause(message)?; - let start = error.location.start().to_usize(); - let end = error.location.end().to_usize(); - let line = previous_non_empty_line_number(source, start)?; - if clause == "'except' statement" - && let Some(previous_line) = previous_non_empty_line(source, start) - && matches!( - previous_line.trim_start(), - line if line.starts_with("except*") || line.starts_with("except *") - ) - { - clause = "'except*' statement"; - } - Some(CpythonDiagnostic::new( - format!("expected an indented block after {clause} on line {line}"), - start, - end, - )) -} - -fn previous_non_empty_line(source: &str, offset: usize) -> Option<&str> { - let bytes = source.as_bytes(); - let mut index = offset.min(bytes.len()); - while index > 0 { - let line_end = index; - while index > 0 && bytes[index - 1] != b'\n' { - index -= 1; - } - let line_start = index; - let mut content_start = line_start; - while content_start < line_end - && matches!(bytes[content_start], b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - { - content_start += 1; - } - let mut content_end = line_end; - while content_end > content_start - && matches!(bytes[content_end - 1], b' ' | b'\t' | b'\r' | b'\x0c') - { - content_end -= 1; - } - if content_start < content_end { - return source.get(line_start..line_end); - } - index = line_start.saturating_sub(1); - } - None -} - fn starts_identifier(bytes: &[u8], index: usize, word: &[u8]) -> bool { bytes.get(index..index + word.len()) == Some(word) && index @@ -1213,3246 +493,6 @@ fn starts_identifier(bytes: &[u8], index: usize, word: &[u8]) -> bool { .is_none_or(|byte| !is_ascii_identifier_char(*byte)) } -fn is_plain_assignment_operator(bytes: &[u8], index: usize) -> bool { - bytes.get(index) == Some(&b'=') - && bytes.get(index + 1) != Some(&b'=') - && !matches!( - index.checked_sub(1).and_then(|before| bytes.get(before)), - Some(b'=' | b'!' | b'<' | b'>' | b':') - ) -} - -fn is_simple_keyword_name(bytes: &[u8], mut start: usize, mut end: usize) -> bool { - while matches!( - bytes.get(start), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) { - start += 1; - } - while end > start - && matches!( - bytes.get(end - 1), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - end -= 1; - } - let Some(&first) = bytes.get(start) else { - return false; - }; - if !(first == b'_' || first.is_ascii_alphabetic() || first >= 0x80) { - return false; - } - let mut index = start + 1; - while index < end { - if bytes[index] < 0x80 && !is_ascii_identifier_char(bytes[index]) { - return false; - } - index += 1; - } - true -} - -fn is_function_parameter_list(bytes: &[u8], paren: usize) -> bool { - let mut cursor = paren; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - if cursor > 0 && bytes.get(cursor - 1) == Some(&b']') { - let mut bracket = cursor; - let mut level = 0usize; - while bracket > 0 { - bracket -= 1; - match bytes[bracket] { - b']' => level += 1, - b'[' => { - level = level.saturating_sub(1); - if level == 0 { - cursor = bracket; - break; - } - } - _ => {} - } - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - cursor >= 3 - && starts_identifier(bytes, cursor - 3, b"def") - && cursor - .checked_sub(4) - .and_then(|before| bytes.get(before)) - .is_none_or(|byte| !is_ascii_identifier_char(*byte)) -} - -#[derive(Clone, Copy)] -enum ParameterListKind { - Function, - Lambda, -} - -fn matching_delimiter(bytes: &[u8], open: usize, close: u8) -> Option { - let mut index = open; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - byte if byte == close => { - level = level.saturating_sub(1); - if level == 0 { - return Some(index); - } - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn find_lambda_parameter_end(bytes: &[u8], mut index: usize) -> Option { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn top_level_byte(bytes: &[u8], mut index: usize, end: usize, needle: u8) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - byte if level == 0 && byte == needle => return Some(index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn identifier_end(bytes: &[u8], mut index: usize, end: usize) -> usize { - if !bytes - .get(index) - .is_some_and(|byte| *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic()) - { - return index; - } - index += 1; - while index < end - && bytes - .get(index) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - index += 1; - } - index -} - -fn expression_slice_is_tuple(source: &str, start: usize, end: usize) -> bool { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - if start >= end { - return false; - } - let Ok(parsed) = parser::parse(&source[start..end], parser::Mode::Expression.into()) else { - return false; - }; - matches!(parsed.into_syntax(), ast::Mod::Expression(expression) if matches!(*expression.body, ast::Expr::Tuple(_))) -} - -fn type_param_list_open(bytes: &[u8], open: usize) -> bool { - let mut cursor = open; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - (cursor >= 3 && starts_identifier(bytes, cursor - 3, b"def")) - || (cursor >= 5 && starts_identifier(bytes, cursor - 5, b"class")) - || (cursor >= 4 && starts_identifier(bytes, cursor - 4, b"type")) -} - -fn invalid_type_param_item_error( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - if start >= end || bytes.get(start) != Some(&b'*') { - return None; - } - let is_param_spec = bytes.get(start + 1) == Some(&b'*'); - let name_start = start + if is_param_spec { 2 } else { 1 }; - let name_end = identifier_end(bytes, name_start, end); - if name_start == name_end { - return None; - } - let colon = next_non_horizontal_whitespace(bytes, name_end); - if colon >= end || bytes.get(colon) != Some(&b':') { - return None; - } - let has_constraints = expression_slice_is_tuple(source, colon + 1, end); - let message = match (is_param_spec, has_constraints) { - (false, false) => "cannot use bound with TypeVarTuple", - (false, true) => "cannot use constraints with TypeVarTuple", - (true, false) => "cannot use bound with ParamSpec", - (true, true) => "cannot use constraints with ParamSpec", - }; - Some(CpythonDiagnostic::new(message.to_owned(), colon, colon + 1)) -} - -fn invalid_type_param_list_error( - source: &str, - open: usize, - close: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut item_start = open + 1; - let mut index = item_start; - let mut level = 0usize; - while index <= close { - if index == close || (level == 0 && bytes.get(index) == Some(&b',')) { - if let Some(error) = invalid_type_param_item_error(source, item_start, index) { - return Some(error); - } - item_start = index + 1; - index += 1; - continue; - } - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_type_param_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'[' if type_param_list_open(bytes, index) => { - let Some(close) = matching_delimiter(bytes, index, b']') else { - index += 1; - continue; - }; - if let Some(error) = invalid_type_param_list_error(source, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_comprehension_in_slice( - bytes: &[u8], - open: usize, - close: usize, -) -> Option { - let for_index = find_keyword_at_level(bytes, open + 1, close, b"for")?; - let item_start = next_non_horizontal_whitespace(bytes, open + 1); - if item_start >= for_index { - return None; - } - if bytes.get(item_start..item_start + 2) == Some(b"**") && bytes.get(open) == Some(&b'{') { - return Some(CpythonDiagnostic::new( - "dict unpacking cannot be used in dict comprehension".to_owned(), - item_start, - item_start + 2, - )); - } - if bytes.get(item_start..item_start + 2) == Some(b"**") && bytes.get(open) == Some(&b'(') { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - for_index, - for_index + 3, - )); - } - if bytes.get(item_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "iterable unpacking cannot be used in comprehension".to_owned(), - item_start, - item_start + 1, - )); - } - if !matches!(bytes.get(open), Some(b'[' | b'{')) { - return None; - } - if top_level_colon(bytes, open + 1, for_index).is_none() - && let Some(comma) = top_level_byte(bytes, open + 1, for_index, b',') - { - let (start, _) = trim_target_range(bytes, open + 1, comma); - return Some(CpythonDiagnostic::new( - "did you forget parentheses around the comprehension target?".to_owned(), - start, - comma + 1, - )); - } - None -} - -fn invalid_comprehension_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - let close_byte = match bytes[index] { - b'(' => b')', - b'[' => b']', - _ => b'}', - }; - let Some(close) = matching_delimiter(bytes, index, close_byte) else { - index += 1; - continue; - }; - if let Some(error) = invalid_comprehension_in_slice(bytes, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_group_in_slice(bytes: &[u8], open: usize, close: usize) -> Option { - let (item_start, item_end) = trim_target_range(bytes, open + 1, close); - if item_start >= item_end - || top_level_byte(bytes, item_start, item_end, b',').is_some() - || top_level_colon(bytes, item_start, item_end).is_some() - || find_keyword_at_level(bytes, item_start, item_end, b"for").is_some() - { - return None; - } - if bytes.get(item_start..item_start + 2) == Some(b"**") { - return Some(CpythonDiagnostic::new( - "cannot use double starred expression here".to_owned(), - item_start, - item_start + 2, - )); - } - if bytes.get(item_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot use starred expression here".to_owned(), - item_start, - item_start + 1, - )); - } - None -} - -fn invalid_group_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - let Some(close) = matching_delimiter(bytes, index, b')') else { - index += 1; - continue; - }; - if let Some(error) = invalid_group_in_slice(bytes, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_parameter_star_annotation_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - let Some(close) = matching_delimiter(bytes, index, b')') else { - index += 1; - continue; - }; - let mut param_start = index + 1; - while param_start < close { - let param_end = - find_byte_at_level(bytes, param_start, close, b',').unwrap_or(close); - if let Some(colon) = top_level_colon(bytes, param_start, param_end) { - let value_start = next_non_horizontal_whitespace(bytes, colon + 1); - if bytes.get(value_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - value_start, - value_start + 1, - )); - } - } - param_start = param_end.saturating_add(1); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_def_type_params_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"def") => { - let name_start = skip_horizontal_whitespace(bytes, index + 3); - let name_end = identifier_end(bytes, name_start, bytes.len()); - let bracket = skip_horizontal_whitespace(bytes, name_end); - if bytes.get(bracket) == Some(&b'[') { - let Some(close) = matching_delimiter(bytes, bracket, b']') else { - index = bracket + 1; - continue; - }; - let after_close = skip_horizontal_whitespace(bytes, close + 1); - if bytes.get(after_close) == Some(&b'(') - && type_param_list_is_malformed(bytes, bracket + 1, close) - { - return Some(CpythonDiagnostic::new( - "expected '('".to_owned(), - bracket, - bracket + 1, - )); - } - } - index = name_end.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn type_param_list_is_malformed(bytes: &[u8], start: usize, end: usize) -> bool { - let mut index = start; - let mut expect_item = true; - while index < end { - index = skip_horizontal_whitespace(bytes, index); - if index >= end { - break; - } - if bytes[index] == b',' { - if expect_item { - return true; - } - expect_item = true; - index += 1; - continue; - } - if !expect_item { - return true; - } - if bytes.get(index..index + 2) == Some(b"**") { - index += 2; - } else if bytes.get(index) == Some(&b'*') { - index += 1; - } - let item_start = skip_horizontal_whitespace(bytes, index); - let item_end = identifier_end(bytes, item_start, end); - if item_end == item_start { - return true; - } - index = item_end; - if bytes.get(skip_horizontal_whitespace(bytes, index)) == Some(&b':') { - index = skip_horizontal_whitespace(bytes, index) + 1; - while index < end && bytes[index] != b',' { - index = match bytes[index] { - b'\'' | b'"' => skip_quoted_string(bytes, index), - _ => index + 1, - }; - } - } - expect_item = false; - } - false -} - -fn invalid_parameter_list_slice_error( - source: &str, - start: usize, - end: usize, - kind: ParameterListKind, -) -> Option { - let bytes = source.as_bytes(); - let mut index = start; - let mut level = 0usize; - let mut default_seen = false; - let mut keyword_only = false; - let mut slash_seen = false; - let mut var_keyword_seen = false; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if level == 0 - && var_keyword_seen - && bytes.get(index).is_some_and(|byte| { - *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic() - }) => - { - let name_end = identifier_end(bytes, index, end); - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - name_end, - )); - } - _ if level == 0 - && !keyword_only - && bytes.get(index).is_some_and(|byte| { - *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic() - }) => - { - let param_end = find_byte_at_level(bytes, index, end, b',') - .or_else(|| top_level_byte(bytes, index, end, b')')) - .or_else(|| { - matches!(kind, ParameterListKind::Lambda) - .then(|| top_level_byte(bytes, index, end, b':')) - .flatten() - }) - .unwrap_or(end); - let name_end = identifier_end(bytes, index, param_end); - if let Some(eq) = top_level_byte(bytes, index, param_end, b'=') { - let value_start = next_non_horizontal_whitespace(bytes, eq + 1); - if value_start >= param_end - || matches!(bytes.get(value_start), Some(b',' | b')' | b':')) - { - let message = if matches!(kind, ParameterListKind::Lambda) - && matches!(bytes.get(value_start), Some(b':')) - { - "invalid syntax" - } else { - "expected default value expression" - }; - return Some(CpythonDiagnostic::new(message.to_owned(), eq, eq + 1)); - } - default_seen = true; - } else if default_seen { - return Some(CpythonDiagnostic::new( - "parameter without a default follows parameter with a default".to_owned(), - index, - name_end, - )); - } - index = param_end; - } - b'(' if level == 0 => { - let close = matching_delimiter(bytes, index, b')') - .filter(|close| *close <= end) - .unwrap_or(index + 1); - let message = match kind { - ParameterListKind::Function => "Function parameters cannot be parenthesized", - ParameterListKind::Lambda => { - "Lambda expression parameters cannot be parenthesized" - } - }; - return Some(CpythonDiagnostic::new(message.to_owned(), index, close + 1)); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'/' if level == 0 => { - if var_keyword_seen { - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - index + 1, - )); - } - if slash_seen { - return Some(CpythonDiagnostic::new( - "/ may appear only once".to_owned(), - index, - index + 1, - )); - } - slash_seen = true; - let next = next_non_horizontal_whitespace(bytes, index + 1); - if bytes.get(next) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "expected comma between / and *".to_owned(), - next, - next + 1, - )); - } - index += 1; - } - b'*' if level == 0 => { - if var_keyword_seen { - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - index + 1, - )); - } - keyword_only = true; - let stars = usize::from(bytes.get(index + 1) == Some(&b'*')) + 1; - let name_start = next_non_horizontal_whitespace(bytes, index + stars); - for keyword in [b"True".as_slice(), b"False".as_slice(), b"None".as_slice()] { - if starts_identifier(bytes, name_start, keyword) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - name_start, - name_start + keyword.len(), - )); - } - } - let param_end = find_byte_at_level(bytes, name_start, end, b',') - .or_else(|| top_level_byte(bytes, name_start, end, b')')) - .or_else(|| { - matches!(kind, ParameterListKind::Lambda) - .then(|| top_level_byte(bytes, name_start, end, b':')) - .flatten() - }) - .unwrap_or(end); - if stars == 1 && matches!(bytes.get(name_start), Some(b')' | b',' | b':')) { - return Some(CpythonDiagnostic::new( - "named arguments must follow bare *".to_owned(), - index, - index + 1, - )); - } - if stars == 1 && top_level_byte(bytes, name_start, param_end, b'=').is_some() { - return Some(CpythonDiagnostic::new( - "var-positional argument cannot have default value".to_owned(), - index, - index + 1, - )); - } - if stars == 2 && top_level_byte(bytes, name_start, param_end, b'=').is_some() { - return Some(CpythonDiagnostic::new( - "var-keyword argument cannot have default value".to_owned(), - index, - index + 2, - )); - } - if stars == 2 { - var_keyword_seen = true; - index = param_end; - continue; - } - index += stars; - } - b'=' if level == 0 => { - let value_start = next_non_horizontal_whitespace(bytes, index + 1); - if value_start >= end || matches!(bytes.get(value_start), Some(b',' | b')' | b':')) - { - if matches!(kind, ParameterListKind::Lambda) - && matches!(bytes.get(value_start), Some(b':')) - { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - index, - index + 1, - )); - } - return Some(CpythonDiagnostic::new( - "expected default value expression".to_owned(), - index, - index + 1, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_parameter_list_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"def") => { - let Some(paren) = top_level_byte(bytes, index + 3, bytes.len(), b'(') else { - index += 3; - continue; - }; - let Some(close) = matching_delimiter(bytes, paren, b')') else { - index = paren + 1; - continue; - }; - if let Some(error) = invalid_parameter_list_slice_error( - source, - paren + 1, - close, - ParameterListKind::Function, - ) { - return Some(error); - } - index = close + 1; - } - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - let Some(params_end) = find_lambda_parameter_end(bytes, params_start) else { - index = params_start; - continue; - }; - if let Some(error) = invalid_parameter_list_slice_error( - source, - params_start, - params_end, - ParameterListKind::Lambda, - ) { - return Some(error); - } - index = params_end + 1; - } - _ => index += 1, - } - } - None -} - -#[derive(Clone, Copy)] -struct CallArgFrame { - level: usize, - arg_start: Option, - in_call: bool, -} - -fn next_non_horizontal_whitespace(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t' | b'\x0c')) { - index += 1; - } - index -} - -fn invalid_call_argument_assignment_error( - source: &str, - arg_start: usize, - equal: usize, -) -> Option { - let bytes = source.as_bytes(); - let start = bytes[arg_start..equal] - .iter() - .rposition(|byte| *byte == b'\n') - .map_or(arg_start, |newline| arg_start + newline + 1); - let (target_start, target_end) = trim_target_range(bytes, start, equal); - if target_start >= target_end { - return None; - } - let value_start = next_non_horizontal_whitespace(bytes, equal + 1); - if matches!(bytes.get(value_start), None | Some(b',' | b')')) { - return Some(CpythonDiagnostic::new( - "expected argument value expression".to_owned(), - target_start, - equal + 1, - )); - } - if bytes.get(target_start..target_start + 2) == Some(b"**") { - return Some(CpythonDiagnostic::new( - "cannot assign to keyword argument unpacking".to_owned(), - target_start, - value_start, - )); - } - if bytes.get(target_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot assign to iterable argument unpacking".to_owned(), - target_start, - value_start, - )); - } - for keyword in [b"True".as_slice(), b"False".as_slice(), b"None".as_slice()] { - if bytes.get(target_start..target_end) == Some(keyword) { - let keyword = ::core::str::from_utf8(keyword).ok()?; - return Some(CpythonDiagnostic::new( - format!("cannot assign to {keyword}"), - target_start, - target_end, - )); - } - } - if is_simple_keyword_name(bytes, target_start, target_end) { - // NAME '=' expression for_if_clauses - if unparenthesized_comprehension(bytes, equal + 1) { - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - return None; - } - Some(CpythonDiagnostic::new( - "expression cannot contain assignment, perhaps you meant \"==\"?".to_owned(), - target_start, - equal, - )) -} - -fn unparenthesized_comprehension(bytes: &[u8], mut index: usize) -> bool { - index = next_non_horizontal_whitespace(bytes, index); - if index >= bytes.len() { - return false; - } - let start = index; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - if level == 0 { - return false; - } - level -= 1; - index += 1; - } - b',' | b':' if level == 0 => return false, - _ if level == 0 && index > start && starts_identifier(bytes, index, b"for") => { - return true; - } - _ => index += 1, - } - } - false -} - -fn invalid_call_star_expression_error( - bytes: &[u8], - arg_start: usize, - index: usize, -) -> Option { - let start = next_non_horizontal_whitespace(bytes, arg_start); - if start != index || bytes.get(index) != Some(&b'*') { - return None; - } - let after_star = next_non_horizontal_whitespace(bytes, index + 1); - if matches!(bytes.get(after_star), None | Some(b',' | b')' | b':')) { - return Some(CpythonDiagnostic::new( - "Invalid star expression".to_owned(), - index, - (index + 1).min(bytes.len()), - )); - } - None -} - -fn invalid_call_argument_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut level = 0usize; - let mut frames: Vec = Vec::new(); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - if let Some(params_end) = find_lambda_parameter_end(bytes, params_start) { - index = params_end + 1; - } else { - index = params_start; - } - } - b'(' => { - level += 1; - let in_call = opening_paren_is_call(bytes, index) - || frames.last().is_some_and(|frame| frame.in_call); - frames.push(CallArgFrame { - level, - arg_start: (in_call && !is_function_parameter_list(bytes, index)) - .then_some(index + 1), - in_call, - }); - index += 1; - } - b')' => { - if matches!(frames.last(), Some(frame) if frame.level == level) { - frames.pop(); - } - level = level.saturating_sub(1); - index += 1; - } - b'[' | b'{' => { - level += 1; - index += 1; - } - b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' => { - if let Some(frame) = frames.last_mut() - && frame.level == level - && frame.arg_start.is_some() - { - frame.arg_start = Some(index + 1); - } - index += 1; - } - b'*' => { - if let Some(CallArgFrame { - level: frame_level, - arg_start: Some(arg_start), - in_call: true, - }) = frames.last().copied() - && frame_level == level - && let Some(error) = invalid_call_star_expression_error(bytes, arg_start, index) - { - return Some(error); - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some(CallArgFrame { - level: frame_level, - arg_start: Some(arg_start), - in_call: true, - }) = frames.last().copied() - && frame_level == level - && let Some(error) = - invalid_call_argument_assignment_error(source, arg_start, index) - { - return Some(error); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn top_level_colon(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn expression_slice_is_valid(source: &str, start: usize, end: usize) -> bool { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - start < end - && parser::parse(&source[start..end], parser::Mode::Expression.into()) - .is_ok_and(|parsed| matches!(parsed.into_syntax(), ast::Mod::Expression(_))) -} - -fn invalid_dict_entry_error( - source: &str, - item_start: usize, - item_end: usize, - colon: Option, - saw_dict_item: bool, -) -> Option { - let bytes = source.as_bytes(); - let (item_start, item_end) = trim_target_range(bytes, item_start, item_end); - if item_start >= item_end { - return None; - } - if let Some(colon) = colon { - let value_start = next_non_horizontal_whitespace(bytes, colon + 1); - if value_start >= item_end { - return Some(CpythonDiagnostic::new( - "expression expected after dictionary key and ':'".to_owned(), - colon, - colon + 1, - )); - } - if bytes.get(value_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot use a starred expression in a dictionary value".to_owned(), - value_start, - value_start + 1, - )); - } - if !expression_slice_is_valid(source, value_start, item_end) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - value_start, - value_start, - )); - } - } else if saw_dict_item { - return Some(CpythonDiagnostic::new( - "':' expected after dictionary key".to_owned(), - item_end.saturating_sub(1), - item_end, - )); - } - None -} - -fn invalid_dict_literal_error( - source: &str, - open: usize, - close: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut item_start = open + 1; - let mut index = item_start; - let mut level = 0usize; - let mut saw_dict_item = false; - let mut item_colon = None; - while index <= close { - if index == close || (level == 0 && bytes.get(index) == Some(&b',')) { - if let Some(error) = - invalid_dict_entry_error(source, item_start, index, item_colon, saw_dict_item) - { - return Some(error); - } - saw_dict_item |= item_colon.is_some(); - item_start = index + 1; - item_colon = None; - index += 1; - continue; - } - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 && item_colon.is_none() => { - item_colon = Some(index); - saw_dict_item = true; - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_dict_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - let Some(close) = matching_delimiter(bytes, index, b'}') else { - index += 1; - continue; - }; - if top_level_colon(bytes, index + 1, close).is_some() - && let Some(error) = invalid_dict_literal_error(source, index, close) - { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn collection_open_is_call(bytes: &[u8], open: usize) -> bool { - if bytes.get(open) != Some(&b'(') { - return false; - } - let mut cursor = open; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - matches!( - cursor.checked_sub(1).and_then(|before| bytes.get(before)), - Some(b')' | b']' | b'_' | b'a'..=b'z' | b'A'..=b'Z' | 0x80..=0xff) - ) -} - -fn invalid_collection_assignment_in_slice( - source: &str, - bytes: &[u8], - start: usize, - end: usize, -) -> Option { - let mut item_start = start; - let mut index = start; - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' if level == 0 => { - item_start = index + 1; - index += 1; - } - b'=' if level == 0 && is_plain_assignment_operator(bytes, index) => { - if top_level_colon(bytes, item_start, index).is_none() { - let start = next_non_horizontal_whitespace(bytes, item_start); - let target_end = trim_end_horizontal_whitespace(bytes, start, index); - if start < target_end - && let Some((expr_name, expr_start, expr_end, _)) = - expression_name_and_range(&source[start..target_end]) - { - if matches!(expr_name, "list" | "tuple") { - return None; - } - if matches!(expr_name, "expression" | "attribute" | "subscript") { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - start + expr_start, - start + expr_end, - )); - } - } - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - start, - index + 1, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_collection_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - let close_byte = match bytes[index] { - b'(' => b')', - b'[' => b']', - _ => b'}', - }; - let Some(close) = matching_delimiter(bytes, index, close_byte) else { - index += 1; - continue; - }; - if !collection_open_is_call(bytes, index) - && let Some(error) = - invalid_collection_assignment_in_slice(source, bytes, index + 1, close) - { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn expression_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - let mut paren_arg_starts: Vec<(Option, bool)> = Vec::new(); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - if let Some(params_end) = find_lambda_parameter_end(bytes, params_start) { - index = params_end + 1; - } else { - index = params_start; - } - } - b'(' => { - let in_call_context = opening_paren_is_call(bytes, index) - || paren_arg_starts.last().is_some_and(|(_, in_call)| *in_call); - paren_arg_starts.push(( - (!is_function_parameter_list(bytes, index)).then_some(index + 1), - in_call_context, - )); - index += 1; - } - b')' => { - paren_arg_starts.pop(); - index += 1; - } - b',' => { - if let Some((start, _)) = paren_arg_starts.last_mut() - && start.is_some() - { - *start = Some(index + 1); - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some((Some(start), true)) = paren_arg_starts.last().copied() - && !is_simple_keyword_name(bytes, start, index) - { - let mut expr_start = start; - while matches!(bytes.get(expr_start), Some(b' ' | b'\t' | b'\x0c')) { - expr_start += 1; - } - return Some(CpythonDiagnostic::new( - "expression cannot contain assignment, perhaps you meant \"==\"?" - .to_owned(), - expr_start, - index, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_named_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index + 1 < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b':' if bytes.get(index + 1) == Some(&b'=') => { - let target_start = named_expression_target_start(bytes, index); - let target_end = trim_end_horizontal_whitespace(bytes, target_start, index); - if target_start < target_end - && let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - && !is_name - { - return Some(CpythonDiagnostic::new( - format!("cannot use assignment expressions with {expr_name}"), - target_start + start, - target_start + end, - )); - } - index += 2; - } - _ => index += 1, - } - } - None -} - -#[derive(Clone, Copy)] -struct AssignmentContext { - start: usize, - call: bool, -} - -fn invalid_plain_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut stack: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - stack.push(AssignmentContext { - start: index + 1, - call: bytes[index] == b'(' && opening_paren_is_call(bytes, index), - }); - index += 1; - } - b')' | b']' | b'}' => { - stack.pop(); - index += 1; - } - b',' => { - if let Some(context) = stack.last_mut() - && !context.call - { - context.start = index + 1; - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some(context) = stack.last().copied() - && !context.call - { - let target_start = skip_horizontal_whitespace(bytes, context.start); - let target_end = trim_end_horizontal_whitespace(bytes, target_start, index); - if target_start < target_end - && let Some((expr_name, start, end, _)) = - expression_name_and_range(&source[target_start..target_end]) - && matches!(expr_name, "expression" | "attribute" | "subscript") - { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - target_start + start, - target_start + end, - )); - } - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn opening_paren_is_call(bytes: &[u8], paren: usize) -> bool { - let mut cursor = paren; - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - cursor > 0 - && (bytes[cursor - 1] >= 0x80 - || is_ascii_identifier_char(bytes[cursor - 1]) - || matches!(bytes[cursor - 1], b')' | b']')) -} - -fn named_expression_target_start(bytes: &[u8], walrus: usize) -> usize { - let mut index = walrus; - let mut level = 0usize; - while index > 0 { - index -= 1; - match bytes[index] { - b')' | b']' | b'}' => level += 1, - b'(' | b'[' | b'{' if level > 0 => level -= 1, - b'(' | b'[' | b'{' if level == 0 => return index + 1, - b',' | b'\n' | b';' if level == 0 => return index + 1, - _ => {} - } - } - 0 -} - -fn trim_end_horizontal_whitespace(bytes: &[u8], start: usize, mut end: usize) -> usize { - while end > start && matches!(bytes[end - 1], b' ' | b'\t' | b'\x0c') { - end -= 1; - } - end -} - -fn annotation_target_error_for_slice( - source: &str, - start: usize, - colon: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, colon); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - match expression.body.as_ref() { - ast::Expr::Name(_) | ast::Expr::Attribute(_) | ast::Expr::Subscript(_) => None, - ast::Expr::List(_) => Some(CpythonDiagnostic::new( - "only single target (not list) can be annotated".to_owned(), - target_start, - target_end, - )), - ast::Expr::Tuple(_) => Some(CpythonDiagnostic::new( - "only single target (not tuple) can be annotated".to_owned(), - target_start, - target_end, - )), - _ => Some(CpythonDiagnostic::new( - "illegal target for annotation".to_owned(), - target_start, - target_end, - )), - } -} - -fn invalid_annotation_line_start(bytes: &[u8], line_start: usize) -> bool { - let column = skip_horizontal_whitespace(bytes, line_start); - for keyword in [ - b"async".as_slice(), - b"case", - b"class", - b"def", - b"elif", - b"else", - b"except", - b"finally", - b"for", - b"if", - b"match", - b"try", - b"while", - b"with", - ] { - if starts_identifier(bytes, column, keyword) { - return false; - } - } - true -} - -fn invalid_annotation_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - if invalid_annotation_line_start(bytes, line_start) - && let Some(colon) = find_byte_at_level(bytes, line_start, line_end, b':') - && bytes.get(colon + 1) != Some(&b'=') - && colon.checked_sub(1).and_then(|before| bytes.get(before)) != Some(&b':') - && let Some(error) = annotation_target_error_for_slice(source, line_start, colon) - { - return Some(error); - } - line_start = line_end; - } - None -} - -fn statement_target_end(bytes: &[u8], mut index: usize) -> usize { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return index, - b'\n' | b';' if level == 0 => return index, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - index -} - -fn invalid_assignment_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { - elts.iter().find_map(invalid_assignment_target) - } - ast::Expr::Starred(ast::ExprStarred { value, .. }) => invalid_assignment_target(value), - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - _ => Some(expression), - } -} - -fn invalid_for_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => elts.iter().find_map(invalid_for_target), - ast::Expr::Starred(ast::ExprStarred { value, .. }) => invalid_for_target(value), - ast::Expr::Compare(ast::ExprCompare { left, ops, .. }) => { - if matches!(ops.first(), Some(ast::CmpOp::In)) { - invalid_for_target(left) - } else { - None - } - } - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - _ => Some(expression), - } -} - -fn invalid_delete_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { - elts.iter().find_map(invalid_delete_target) - } - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - ast::Expr::Starred(_) => Some(expression), - ast::Expr::Compare(_) => Some(expression), - _ => Some(expression), - } -} - -fn delete_target_expr_name(expression: &ast::Expr) -> &'static str { - match expression { - ast::Expr::Attribute(_) => "attribute", - ast::Expr::Subscript(_) => "subscript", - ast::Expr::Starred(_) => "starred", - ast::Expr::Name(_) => "name", - ast::Expr::List(_) => "list", - ast::Expr::Tuple(_) => "tuple", - ast::Expr::Lambda(_) => "lambda", - ast::Expr::Call(_) => "function call", - ast::Expr::BoolOp(_) | ast::Expr::BinOp(_) | ast::Expr::UnaryOp(_) => "expression", - ast::Expr::Generator(_) => "generator expression", - ast::Expr::Yield(_) | ast::Expr::YieldFrom(_) => "yield expression", - ast::Expr::Await(_) => "await expression", - ast::Expr::ListComp(_) => "list comprehension", - ast::Expr::SetComp(_) => "set comprehension", - ast::Expr::DictComp(_) => "dict comprehension", - ast::Expr::Dict(_) => "dict literal", - ast::Expr::Set(_) => "set display", - ast::Expr::FString(_) => "f-string expression", - ast::Expr::TString(_) => "t-string expression", - ast::Expr::NumberLiteral(_) | ast::Expr::StringLiteral(_) | ast::Expr::BytesLiteral(_) => { - "literal" - } - ast::Expr::Constant(expr) => match &expr.value { - ast::ConstantValue::None => "None", - ast::ConstantValue::Boolean(true) => "True", - ast::ConstantValue::Boolean(false) => "False", - ast::ConstantValue::Ellipsis => "ellipsis", - ast::ConstantValue::Tuple(_) => "tuple", - ast::ConstantValue::Frozenset(_) => "literal", - ast::ConstantValue::Str(_) - | ast::ConstantValue::Bytes(_) - | ast::ConstantValue::Integer(_) - | ast::ConstantValue::Float(_) - | ast::ConstantValue::Complex { .. } => "literal", - }, - ast::Expr::BooleanLiteral(boolean) => { - if boolean.value { - "True" - } else { - "False" - } - } - ast::Expr::NoneLiteral(_) => "None", - ast::Expr::EllipsisLiteral(_) => "ellipsis", - ast::Expr::Compare(_) => "comparison", - ast::Expr::If(_) => "conditional expression", - ast::Expr::Named(_) => "named expression", - ast::Expr::Slice(_) | ast::Expr::IpyEscapeCommand(_) => "expression", - } -} - -fn parenthesized_single_starred_delete_target(bytes: &[u8], start: usize, end: usize) -> bool { - let mut cursor = start; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if bytes.get(cursor) != Some(&b'(') { - return false; - } - cursor += 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if bytes.get(cursor) != Some(&b'*') { - return false; - } - let mut level = 1usize; - cursor += 1; - while cursor < end { - match bytes[cursor] { - b'\'' | b'"' => { - cursor = skip_quoted_string(bytes, cursor); - } - b'(' | b'[' | b'{' => { - level += 1; - cursor += 1; - } - b')' => { - level = level.saturating_sub(1); - if level == 0 { - cursor += 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - return cursor == end; - } - cursor += 1; - } - b',' if level == 1 => return false, - b']' | b'}' => { - level = level.saturating_sub(1); - cursor += 1; - } - _ => cursor += 1, - } - } - false -} - -fn assignment_target_expr_range(source: &str, start: usize, end: usize) -> Option<(usize, usize)> { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - if parser::parse( - &source[target_start..target_end], - parser::Mode::Expression.into(), - ) - .is_ok() - { - return Some((target_start, target_end)); - } - // `def f(): (yield bar)` — skip the suite header so the remaining - // text is the assignment target expression. Other colons (`x: int += 1`) - // are not suite headers for this diagnostic. Grouping parentheses may - // wrap the yield expression. - let colon = top_level_colon(bytes, target_start, target_end)?; - let after = skip_horizontal_whitespace(bytes, colon + 1); - let yield_at = skip_opening_parentheses(bytes, after, target_end); - if yield_at >= target_end || !starts_identifier(bytes, yield_at, b"yield") { - return None; - } - Some((after, target_end)) -} - -fn skip_opening_parentheses(bytes: &[u8], mut index: usize, end: usize) -> usize { - loop { - index = skip_horizontal_whitespace(bytes, index); - if index >= end || bytes[index] != b'(' { - return index; - } - index += 1; - } -} - -fn trim_target_range(bytes: &[u8], mut start: usize, mut end: usize) -> (usize, usize) { - while start < end - && matches!( - bytes.get(start), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - start += 1; - } - while end > start - && matches!( - bytes.get(end - 1), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - end -= 1; - } - (start, end) -} - -fn invalid_assignment_message(name: &'static str, top_level_bitwise: bool) -> String { - if top_level_bitwise { - format!("cannot assign to {name} here. Maybe you meant '==' instead of '='?") - } else { - format!("cannot assign to {name}") - } -} - -fn assignment_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = assignment_target_expr_range(source, start, end)?; - if starts_identifier(bytes, target_start, b"yield") { - return Some(CpythonDiagnostic::new( - "assignment to yield expression not possible".to_owned(), - target_start, - target_start + 5, - )); - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let invalid_target = invalid_assignment_target(&expression.body)?; - let invalid_start = target_start + invalid_target.range().start().to_usize(); - let invalid_end = target_start + invalid_target.range().end().to_usize(); - let name = delete_target_expr_name(invalid_target); - let top_level = invalid_target.range() == expression.body.range(); - let bitwise_like = matches!( - invalid_target, - ast::Expr::Call(_) - | ast::Expr::BoolOp(_) - | ast::Expr::BinOp(_) - | ast::Expr::UnaryOp(_) - | ast::Expr::NumberLiteral(_) - | ast::Expr::StringLiteral(_) - | ast::Expr::BytesLiteral(_) - | ast::Expr::EllipsisLiteral(_) - | ast::Expr::Yield(_) - | ast::Expr::YieldFrom(_) - | ast::Expr::Set(_) - | ast::Expr::Dict(_) - | ast::Expr::FString(_) - | ast::Expr::TString(_) - ); - Some(CpythonDiagnostic::new( - invalid_assignment_message(name, top_level && bitwise_like), - invalid_start, - invalid_end, - )) -} - -fn star_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - invalid_target_error_for_slice(source, start, end, invalid_assignment_target) -} - -fn for_target_error_for_slice(source: &str, start: usize, end: usize) -> Option { - invalid_target_error_for_slice(source, start, end, invalid_for_target) -} - -fn invalid_target_error_for_slice( - source: &str, - start: usize, - end: usize, - invalid_target: for<'a> fn(&'a ast::Expr) -> Option<&'a ast::Expr>, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let invalid_target = invalid_target(&expression.body)?; - let name = delete_target_expr_name(invalid_target); - let invalid_start = target_start + invalid_target.range().start().to_usize(); - let invalid_end = target_start + invalid_target.range().end().to_usize(); - Some(CpythonDiagnostic::new( - format!("cannot assign to {name}"), - invalid_start, - invalid_end, - )) -} - -fn first_compare_operator_at_level(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'<' | b'>' if level == 0 => return Some(index), - b'=' if level == 0 && bytes.get(index + 1) == Some(&b'=') => return Some(index), - b'!' if level == 0 && bytes.get(index + 1) == Some(&b'=') => return Some(index), - _ if level == 0 && starts_identifier(bytes, index, b"is") => return Some(index), - _ if level == 0 && starts_identifier(bytes, index, b"not") => return Some(index), - _ => index += 1, - } - } - None -} - -fn non_in_compare_for_target_error( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let ast::Expr::Compare(ast::ExprCompare { ops, .. }) = expression.body.as_ref() else { - return None; - }; - if matches!(ops.first(), Some(ast::CmpOp::In)) { - return None; - } - let operator = first_compare_operator_at_level(bytes, target_start, target_end)?; - Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - operator, - (operator + 1).min(target_end), - )) -} - -fn top_level_plain_assignment_offsets(bytes: &[u8]) -> Vec { - let mut offsets = Vec::new(); - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'=' if level == 0 && is_plain_assignment_operator(bytes, index) => { - offsets.push(index); - index += 1; - } - _ => index += 1, - } - } - offsets -} - -fn invalid_condition_assignment_error( - source: &str, - parse_error_offset: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut line_start = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' => { - line_start = index + 1; - index += 1; - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 - && index == skip_horizontal_whitespace(bytes, line_start) - && (starts_identifier(bytes, index, b"if") - || starts_identifier(bytes, index, b"elif") - || starts_identifier(bytes, index, b"while")) => - { - let keyword_len = if starts_identifier(bytes, index, b"while") { - 5 - } else if starts_identifier(bytes, index, b"elif") { - 4 - } else { - 2 - }; - let cond_start = skip_horizontal_whitespace(bytes, index + keyword_len); - let Some(colon) = top_level_colon(bytes, cond_start, bytes.len()) else { - index += keyword_len; - continue; - }; - if parse_error_offset < index || parse_error_offset > colon { - index += keyword_len; - continue; - } - let Some(equal) = condition_plain_assignment(bytes, cond_start, colon) else { - index += keyword_len; - continue; - }; - let (target_start, target_end) = trim_target_range(bytes, cond_start, equal); - if target_start >= target_end { - index += keyword_len; - continue; - } - if is_simple_keyword_name(bytes, target_start, target_end) { - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - if let Some((expr_name, start, end, _)) = - expression_name_and_range(&source[target_start..target_end]) - { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - target_start + start, - target_start + end, - )); - } - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - _ => index += 1, - } - } - None -} - -fn condition_plain_assignment(bytes: &[u8], start: usize, end: usize) -> Option { - let mut index = start; - let mut nest = Vec::new(); - while index < end { - match bytes[index] { - b'#' => { - while index < end && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - nest.push(if is_call_open(bytes, start, index) { - b'c' - } else { - b'g' - }); - index += 1; - } - b'[' => { - nest.push(b'['); - index += 1; - } - b'{' => { - nest.push(b'{'); - index += 1; - } - b')' | b']' | b'}' => { - nest.pop(); - index += 1; - } - b':' if nest.last() == Some(&b'l') => { - nest.pop(); - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) - && !nest.contains(&b'c') - && !nest.contains(&b'l') => - { - return Some(index); - } - _ if starts_identifier(bytes, index, b"lambda") => { - nest.push(b'l'); - index += 6; - } - _ => index += 1, - } - } - None -} - -fn is_call_open(bytes: &[u8], start: usize, open: usize) -> bool { - let mut index = open; - while index > start { - index -= 1; - match bytes[index] { - b' ' | b'\t' | b'\n' | b'\r' | b'\x0c' => {} - b')' | b']' => return true, - byte if byte >= 0x80 || is_ascii_identifier_char(byte) => { - let mut ident_start = index; - while ident_start > start - && bytes - .get(ident_start - 1) - .is_some_and(|b| *b >= 0x80 || is_ascii_identifier_char(*b)) - { - ident_start -= 1; - } - return !is_condition_keyword(&bytes[ident_start..=index]); - } - _ => return false, - } - } - false -} - -fn is_condition_keyword(word: &[u8]) -> bool { - matches!( - word, - b"not" | b"and" | b"or" | b"in" | b"is" | b"if" | b"elif" | b"while" | b"await" | b"lambda" - ) -} - -fn invalid_assignment_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let offsets = top_level_plain_assignment_offsets(bytes); - if offsets.is_empty() { - return None; - } - let mut start = 0usize; - for offset in offsets { - if let Some(error) = assignment_target_error_for_slice(source, start, offset) { - return Some(error); - } - start = offset + 1; - } - None -} - -fn top_level_augassign_offset(bytes: &[u8]) -> Option<(usize, usize)> { - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'+' | b'-' | b'*' | b'@' | b'/' | b'%' | b'&' | b'|' | b'^' - if level == 0 && bytes.get(index + 1) == Some(&b'=') => - { - return Some((index, 2)); - } - b'<' | b'>' - if level == 0 - && bytes.get(index + 1) == Some(&bytes[index]) - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - b'*' if level == 0 - && bytes.get(index + 1) == Some(&b'*') - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - b'/' if level == 0 - && bytes.get(index + 1) == Some(&b'/') - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - _ => index += 1, - } - } - None -} - -fn invalid_augassign_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let (operator, _) = top_level_augassign_offset(bytes)?; - let (target_start, target_end) = assignment_target_expr_range(source, 0, operator)?; - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let name = delete_target_expr_name(&expression.body); - Some(CpythonDiagnostic::new( - format!("'{name}' is an illegal expression for augmented assignment"), - target_start, - target_end, - )) -} - -fn find_for_target_delimiter(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"in") => return Some(index), - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_for_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"for") => { - let target_start = skip_horizontal_whitespace(bytes, index + 3); - let line_end = source[index..] - .find('\n') - .map_or(bytes.len(), |newline| index + newline); - if let Some(target_end) = find_for_target_delimiter(bytes, target_start, line_end) { - if let Some(error) = - for_target_error_for_slice(source, target_start, target_end) - { - return Some(error); - } - if let Some(error) = - non_in_compare_for_target_error(source, target_start, target_end) - { - return Some(error); - } - } - index = target_start.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn find_with_target_delimiter(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' if level == 0 => return Some(index), - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' | b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_with_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if starts_identifier(bytes, column, b"async") { - column = skip_horizontal_whitespace(bytes, column + 5); - } - if !starts_identifier(bytes, column, b"with") { - line_start = line_end; - continue; - } - let mut index = column + 4; - while let Some(as_index) = find_keyword_at_level(bytes, index, line_end, b"as") { - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - if let Some(target_end) = find_with_target_delimiter(bytes, target_start, line_end) { - if let Some(error) = star_target_error_for_slice(source, target_start, target_end) { - return Some(error); - } - index = target_end.saturating_add(1); - } else { - break; - } - } - line_start = line_end; - } - None -} - -fn find_missing_in_if_keyword(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level == 0 => return None, - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"in") => return None, - _ if level == 0 && starts_identifier(bytes, index, b"if") => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_for_if_clause_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level > 0 && starts_identifier(bytes, index, b"for") => { - let target_start = skip_horizontal_whitespace(bytes, index + 3); - let line_end = source[index..] - .find('\n') - .map_or(bytes.len(), |newline| index + newline); - if let Some(if_index) = find_missing_in_if_keyword(bytes, target_start, line_end) { - return Some(CpythonDiagnostic::new( - "'in' expected after for-loop variables".to_owned(), - if_index, - (if_index + 2).min(line_end), - )); - } - index = target_start.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn invalid_delete_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'd' if starts_identifier(bytes, index, b"del") => { - let mut target_start = index + 3; - if !matches!(bytes.get(target_start), Some(b' ' | b'\t' | b'\x0c')) { - index += 3; - continue; - } - while matches!(bytes.get(target_start), Some(b' ' | b'\t' | b'\x0c')) { - target_start += 1; - } - let mut target_end = statement_target_end(bytes, target_start); - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - if target_start >= target_end { - index = target_end.max(index + 3); - continue; - } - if parenthesized_single_starred_delete_target(bytes, target_start, target_end) { - return Some(CpythonDiagnostic::new( - "cannot use starred expression here".to_owned(), - target_start, - target_end, - )); - } - if bytes.get(target_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot delete starred".to_owned(), - target_start, - (target_start + 1).min(target_end), - )); - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - index = target_end; - continue; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - index = target_end; - continue; - }; - let Some(invalid_target) = invalid_delete_target(&expression.body) else { - index = target_end; - continue; - }; - let start = target_start + invalid_target.range().start().to_usize(); - let end = target_start + invalid_target.range().end().to_usize(); - if matches!(invalid_target, ast::Expr::FString(_)) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - start, - end, - )); - } - let name = delete_target_expr_name(invalid_target); - return Some(CpythonDiagnostic::new( - format!("cannot delete {name}"), - start, - end, - )); - } - _ => index += 1, - } - } - None -} - -fn skip_horizontal_whitespace(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t' | b'\x0c')) { - index += 1; - } - index -} - -fn find_keyword_at_level( - bytes: &[u8], - mut index: usize, - end: usize, - keyword: &[u8], -) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, keyword) => return Some(index), - _ => index += 1, - } - } - None -} - -fn find_byte_at_level(bytes: &[u8], mut index: usize, end: usize, needle: u8) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - byte if level == 0 && byte == needle => return Some(index), - _ => index += 1, - } - } - None -} - -fn expression_name_and_range(source: &str) -> Option<(&'static str, usize, usize, bool)> { - let parsed = parser::parse(source, parser::Mode::Expression.into()).ok()?; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let is_name = matches!(expression.body.as_ref(), ast::Expr::Name(_)); - Some(( - delete_target_expr_name(&expression.body), - expression.body.range().start().to_usize(), - expression.body.range().end().to_usize(), - is_name, - )) -} - -fn invalid_standalone_except_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut seen_try = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end { - line_start = line_end; - continue; - } - if starts_identifier(bytes, column, b"try") { - seen_try = true; - } else if (bytes.get(column..column + 7) == Some(b"except*") - || starts_identifier(bytes, column, b"except")) - && !seen_try - { - let end = if bytes.get(column..column + 7) == Some(b"except*") { - column + 7 - } else { - column + 6 - }; - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - column, - end, - )); - } - line_start = line_end; - } - None -} - -fn invalid_import_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - if column < line_end - && starts_identifier(bytes, column, b"import") - && find_keyword_at_level(bytes, column + 6, line_end, b"from").is_some() - { - return Some(CpythonDiagnostic::new( - "Did you mean to use 'from ... import ...' instead?".to_owned(), - column, - column + 6, - )); - } - line_start = line_end; - } - None -} - -fn import_as_target_end(bytes: &[u8], mut index: usize) -> usize { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return index, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' if level == 0 => return index, - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' | b';' | b'\n' if level == 0 => return index, - _ => index += 1, - } - } - index -} - -fn valid_import_alias_name(bytes: &[u8], mut start: usize, end: usize) -> bool { - start = skip_horizontal_whitespace(bytes, start); - let Some(&first) = bytes.get(start) else { - return false; - }; - if !(first == b'_' || first.is_ascii_alphabetic() || first >= 0x80) { - return false; - } - let mut index = start + 1; - while index < end { - match bytes[index] { - b' ' | b'\t' | b'\x0c' => break, - byte if byte >= 0x80 || is_ascii_identifier_char(byte) => index += 1, - _ => return false, - } - } - let index = skip_horizontal_whitespace(bytes, index); - matches!( - bytes.get(index), - None | Some(b',' | b')' | b';' | b'\n' | b'\r') - ) -} - -fn import_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end || valid_import_alias_name(bytes, target_start, target_end) { - return None; - } - let parsed = parser::parse( - &source[target_start..target_end], - parser::Mode::Expression.into(), - ) - .ok()?; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let name = delete_target_expr_name(&expression.body); - let start = target_start + expression.body.range().start().to_usize(); - let end = target_start + expression.body.range().end().to_usize(); - Some(CpythonDiagnostic::new( - format!("cannot use {name} as import target"), - start, - end, - )) -} - -fn statement_starts_import(bytes: &[u8], line_start: usize, line_end: usize) -> bool { - let column = skip_horizontal_whitespace(bytes, line_start); - if starts_identifier(bytes, column, b"import") { - return true; - } - starts_identifier(bytes, column, b"from") - && find_keyword_at_level(bytes, column + 4, line_end, b"import").is_some() -} - -fn invalid_import_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut in_parenthesized_from_import = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let starts_import = statement_starts_import(bytes, line_start, line_end); - if starts_import && bytes[line_start..line_end].contains(&b'(') { - in_parenthesized_from_import = true; - } - if starts_import || in_parenthesized_from_import { - let mut index = line_start; - while index < line_end { - if starts_identifier(bytes, index, b"as") { - let target_start = skip_horizontal_whitespace(bytes, index + 2); - let target_end = import_as_target_end(bytes, target_start); - if let Some(error) = - import_target_error_for_slice(source, target_start, target_end) - { - return Some(error); - } - index = target_end.max(index + 2); - } else { - index += 1; - } - } - } - if in_parenthesized_from_import && bytes[line_start..line_end].contains(&b')') { - in_parenthesized_from_import = false; - } - line_start = line_end; - } - None -} - -fn invalid_except_as_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut seen_try = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end { - line_start = line_end; - continue; - } - if starts_identifier(bytes, column, b"try") { - seen_try = true; - line_start = line_end; - continue; - } - let (keyword_len, starred) = if bytes.get(column..column + 7) == Some(b"except*") { - (7, true) - } else if starts_identifier(bytes, column, b"except") { - (6, false) - } else { - line_start = line_end; - continue; - }; - if !seen_try { - line_start = line_end; - continue; - } - column += keyword_len; - let Some(as_index) = find_keyword_at_level(bytes, column, line_end, b"as") else { - line_start = line_end; - continue; - }; - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - let Some(delimiter) = find_byte_at_level(bytes, target_start, line_end, b':') - .into_iter() - .chain(find_byte_at_level(bytes, target_start, line_end, b',')) - .min() - else { - line_start = line_end; - continue; - }; - let mut target_end = delimiter; - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - else { - line_start = line_end; - continue; - }; - if !is_name { - let statement = if starred { "except*" } else { "except" }; - return Some(CpythonDiagnostic::new( - format!("cannot use {statement} statement with {expr_name}"), - target_start + start, - target_start + end, - )); - } - line_start = line_end; - } - None -} - -fn invalid_match_as_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let quoted_ranges = quoted_string_ranges(bytes); - let mut quoted_range = 0usize; - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end - || offset_in_ranges("ed_ranges, &mut quoted_range, column) - || !starts_identifier(bytes, column, b"case") - { - line_start = line_end; - continue; - } - column += 4; - let Some(as_index) = find_keyword_at_level(bytes, column, line_end, b"as") else { - line_start = line_end; - continue; - }; - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - let Some(delimiter) = find_byte_at_level(bytes, target_start, line_end, b':') - .into_iter() - .chain(find_byte_at_level(bytes, target_start, line_end, b',')) - .min() - else { - line_start = line_end; - continue; - }; - let mut target_end = delimiter; - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - if source[target_start..target_end].trim() == "_" { - return Some(CpythonDiagnostic::new( - "cannot use '_' as a target".to_owned(), - target_start, - target_end, - )); - } - let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - else { - line_start = line_end; - continue; - }; - if !is_name { - if matches!(expr_name, "expression" | "subscript") { - line_start = line_end; - continue; - } - return Some(CpythonDiagnostic::new( - format!("cannot use {expr_name} as pattern target"), - target_start + start, - target_start + end, - )); - } - line_start = line_end; - } - None -} - -fn quoted_string_ranges(bytes: &[u8]) -> Vec<(usize, usize)> { - let mut ranges = Vec::new(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - let end = skip_quoted_string(bytes, index); - ranges.push((index, end)); - index = end; - } - _ => index += 1, - } - } - ranges -} - -fn offset_in_ranges(ranges: &[(usize, usize)], range_index: &mut usize, offset: usize) -> bool { - while ranges - .get(*range_index) - .is_some_and(|(_, end)| *end <= offset) - { - *range_index += 1; - } - ranges - .get(*range_index) - .is_some_and(|(start, end)| *start <= offset && offset < *end) -} - -fn invalid_match_mapping_rest_wildcard_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let next_line_end = |line_start: usize| { - line_start - + bytes[line_start..] - .iter() - .position(|byte| *byte == b'\n') - .unwrap_or(bytes.len() - line_start) - }; - let mut index = 0usize; - let mut line_start = 0usize; - let mut line_end = next_line_end(line_start); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'\n' => { - index += 1; - line_start = index; - line_end = next_line_end(line_start); - } - _ => { - let column = skip_horizontal_whitespace(bytes, line_start); - if index != column - || column >= line_end - || !starts_identifier(bytes, column, b"case") - { - index += 1; - continue; - } - let mut cursor = column + 4; - while cursor < line_end { - match bytes[cursor] { - b'#' => break, - b'\'' | b'"' => cursor = skip_quoted_string(bytes, cursor), - b'{' => { - let rest = next_non_horizontal_whitespace(bytes, cursor + 1); - if bytes.get(rest..rest + 2) == Some(b"**") { - let name_start = next_non_horizontal_whitespace(bytes, rest + 2); - let name_end = identifier_end(bytes, name_start, line_end); - if source.get(name_start..name_end) == Some("_") { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - name_start, - name_end, - )); - } - } - cursor += 1; - } - _ => cursor += 1, - } - } - index = line_end; - } - } - } - None -} - -fn invalid_if_expression_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - if let Some(if_index) = find_keyword_at_level(bytes, line_start, line_end, b"if") - && let Some((start, end)) = statement_before_if_expression(bytes, line_start, if_index) - && find_keyword_at_level(bytes, if_index + 2, line_end, b"else").is_some() - { - return Some(CpythonDiagnostic::new( - "expected expression before 'if', but statement is given".to_owned(), - start, - end, - )); - } - if let Some(else_index) = find_keyword_at_level(bytes, line_start, line_end, b"else") - && find_keyword_at_level(bytes, line_start, else_index, b"if").is_some() - && let Some((start, end)) = - statement_after_else_expression(bytes, else_index + 4, line_end) - { - return Some(CpythonDiagnostic::new( - "expected expression after 'else', but statement is given".to_owned(), - start, - end, - )); - } - line_start = line_end; - } - None -} - -fn statement_before_if_expression( - bytes: &[u8], - line_start: usize, - if_index: usize, -) -> Option<(usize, usize)> { - let mut start = if_index; - while start > line_start && matches!(bytes.get(start - 1), Some(b' ' | b'\t' | b'\x0c')) { - start -= 1; - } - while start > line_start - && !matches!( - bytes.get(start - 1), - Some(b'=' | b':' | b',' | b'(' | b'[' | b'{') - ) - { - start -= 1; - } - start = skip_horizontal_whitespace(bytes, start); - for keyword in [b"pass".as_slice(), b"break", b"continue"] { - if starts_identifier(bytes, start, keyword) { - return Some((start, start + keyword.len())); - } - } - None -} - -fn statement_after_else_expression( - bytes: &[u8], - else_end: usize, - line_end: usize, -) -> Option<(usize, usize)> { - let start = skip_horizontal_whitespace(bytes, else_end); - for keyword in [ - b"pass".as_slice(), - b"return", - b"raise", - b"del", - b"yield", - b"assert", - b"break", - b"continue", - b"import", - b"from", - ] { - if starts_identifier(bytes, start, keyword) { - let end = statement_target_end(bytes, start).min(line_end); - return Some((start, end.max(start + keyword.len()))); - } - } - None -} - -fn invalid_else_elif_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut else_indents: Vec = Vec::new(); - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - let line_column = column.saturating_sub(line_start); - if column >= line_end { - line_start = line_end; - continue; - } - while else_indents - .last() - .is_some_and(|indent| line_column < *indent) - { - else_indents.pop(); - } - if starts_identifier(bytes, column, b"else") - && find_byte_at_level(bytes, column + 4, line_end, b':').is_some() - { - else_indents.push(line_column); - } else if starts_identifier(bytes, column, b"elif") && else_indents.contains(&line_column) { - return Some(CpythonDiagnostic::new( - "'elif' block follows an 'else' block".to_owned(), - column, - column + 4, - )); - } - line_start = line_end; - } - None -} - -fn mixed_except_handlers_error(source: &str) -> Option { - let message = "cannot have both 'except' and 'except*' on the same 'try'".to_owned(); - let mut seen_except = false; - let mut seen_except_star = false; - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let bytes = line.as_bytes(); - let mut column = 0usize; - while matches!(bytes.get(column), Some(b' ' | b'\t' | b'\x0c')) { - column += 1; - } - let token_start = line_start + column; - if bytes.get(column..column + 7) == Some(b"except*") { - if seen_except { - return Some(CpythonDiagnostic::new( - message, - token_start, - token_start + 7, - )); - } - seen_except_star = true; - } else if starts_identifier(bytes, column, b"except") { - if seen_except_star { - return Some(CpythonDiagnostic::new( - message, - token_start, - token_start + 6, - )); - } - seen_except = true; - } - line_start += line.len(); - } - None -} - -fn non_printable_character_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - byte if byte.is_ascii_control() && !matches!(byte, b'\t' | b'\n' | b'\r' | b'\x0c') => { - return Some(CpythonDiagnostic::new( - format!("invalid non-printable character U+{byte:04X}"), - index, - index + 1, - )); - } - byte if byte >= 0x80 => { - let ch = source[index..].chars().next()?; - if ch.is_control() { - return Some(CpythonDiagnostic::new( - format!("invalid non-printable character U+{:04X}", ch as u32), - index, - index + ch.len_utf8(), - )); - } - index += ch.len_utf8(); - } - _ => index += 1, - } - } - None -} - -fn unterminated_string_error(source: &str, mode: Mode) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - let mut line = 1usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' => { - line += 1; - index += 1; - } - quote @ (b'\'' | b'"') => { - let start = index; - let start_line = line; - let quote_size = if bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - 3 - } else { - 1 - }; - index += quote_size; - let mut has_escaped_quote = false; - let mut ended_with_escape = false; - let mut closed = false; - while index < bytes.len() { - let c = bytes[index]; - if c == b'\n' { - if quote_size == 1 { - // A single-quoted literal cannot span a line, so this is the same - // "the literal never ended" case as running out of source; CPython - // spells the two as one condition in lexer.c too. - break; - } - line += 1; - index += 1; - } else if c == quote { - if quote_size == 3 { - if bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - index += 3; - closed = true; - break; - } - index += 1; - } else { - index += 1; - closed = true; - break; - } - } else if c == b'\\' { - if bytes.get(index + 1) == Some("e) { - has_escaped_quote = true; - } - ended_with_escape = index + 1 >= bytes.len(); - index = (index + 2).min(bytes.len()); - } else { - index += 1; - } - } - if !closed { - if let Some(error) = - unclosed_replacement_field_error(bytes, start, start + quote_size, index) - { - return Some(error); - } - let detected_line = if quote_size == 3 { line } else { start_line }; - let interpolated = interpolated_string_prefix(bytes, start); - let diagnostic = CpythonDiagnostic::new( - unterminated_string_message( - detected_line, - quote_size == 3, - has_escaped_quote, - interpolated, - ), - start, - start, - ); - // E_EOLS is only for a plain single-quoted string at EOF. - // Single-quoted f/t-strings never set it. exec input appends a - // newline, so a single-quoted literal becomes the newline case - // unless a final `\` consumes that newline. - let exec_implicit_newline = - matches!(mode, Mode::Exec) && quote_size == 1 && !ended_with_escape; - let eval_assignment = - matches!(mode, Mode::Eval) && eval_has_assignment_before(bytes, start); - let continuable = index >= bytes.len() - && !(interpolated.is_some() && quote_size == 1) - && !exec_implicit_newline - && !eval_assignment; - return Some(if continuable { - diagnostic.with_unclosed_string() - } else { - diagnostic - }); - } - } - _ => index += 1, - } - } - None -} - fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool { let mut index = 0; let mut level = 0usize; @@ -4497,162 +537,6 @@ fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool { false } -fn invalid_interpolated_string_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - quote @ (b'\'' | b'"') => { - let Some(prefix) = interpolated_string_prefix(bytes, index) else { - index = skip_quoted_string(bytes, index); - continue; - }; - if let Some(error) = - single_quoted_format_spec_newline_error(bytes, index, quote, prefix) - { - return Some(error); - } - let Some((content_start, content_end)) = - quoted_string_content_range(bytes, index, quote) - else { - index = skip_quoted_string(bytes, index); - continue; - }; - if let Some(error) = - invalid_replacement_field_error(bytes, content_start, content_end, prefix) - { - return Some(error); - } - index = skip_quoted_string(bytes, index); - } - _ => index += 1, - } - } - None -} - -/// Whether the literal opened at `quote` carries a `b` in its prefix. -fn is_bytes_literal(bytes: &[u8], quote: usize) -> bool { - let mut start = quote; - while quote - start < 2 - && start > 0 - && matches!( - bytes[start - 1].to_ascii_lowercase(), - b'r' | b'b' | b'u' | b'f' | b't' - ) - { - start -= 1; - } - if start > 0 && is_ascii_identifier_char(bytes[start - 1]) { - return false; - } - bytes[start..quote] - .iter() - .any(|byte| byte.eq_ignore_ascii_case(&b'b')) -} - -/// Which of CPython's two messages a mixed literal concatenation gets. Ruff reports only the -/// bytes/non-bytes mix, so `t"x" b"y"` arrives here as a bytes error where CPython names the -/// t-string, and this scanner owns the choice between them. -/// -/// CPython's `invalid_string_tstring_concat` is an `invalid_` rule, reached only on the error -/// pass, and the `strings` rule tries `(fstring|string)+` first. That alternative consumes the -/// concatenation's leading run of non-t-string literals, so a bytes/non-bytes mix among *those* -/// raises from `_PyPegen_concatenate_strings` on the first pass and the t-string rule never runs. -/// The bytes message therefore keeps precedence for `"a" b"b" t"c"` but not for `t"x" b"y"`. -fn mixed_tstring_literal_error( - error: &parser::ParseError, - source: &str, -) -> Option { - let parser::ParseErrorType::OtherError(message) = &error.error else { - return None; - }; - if !message.eq_ignore_ascii_case("bytes literal cannot be mixed with non-bytes literals") { - return None; - } - - let start = error.location.start().to_usize(); - let end = error.location.end().to_usize(); - let bytes = source.as_bytes(); - if end > bytes.len() { - return None; - } - - let mut index = start; - let (mut bytes_seen, mut nonbytes_seen) = (false, false); - while index < end { - match bytes[index] { - b'\'' | b'"' => { - if interpolated_string_prefix(bytes, index) == Some("t-string") { - let message = if bytes_seen && nonbytes_seen { - // Reported here rather than left to ruff's error so that the scanners - // further down the chain cannot claim the concatenation instead. - "cannot mix bytes and nonbytes literals" - } else { - "cannot mix t-string literals with string or bytes literals" - }; - return Some(CpythonDiagnostic::new(message.to_owned(), start, end)); - } - if is_bytes_literal(bytes, index) { - bytes_seen = true; - } else { - nonbytes_seen = true; - } - index = skip_quoted_string(bytes, index); - } - _ => index += 1, - } - } - None -} - -fn single_quoted_format_spec_newline_error( - bytes: &[u8], - quote_index: usize, - quote: u8, - prefix: &str, -) -> Option { - if bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e) { - return None; - } - - let (content_start, content_end) = quoted_string_content_range(bytes, quote_index, quote)?; - let mut index = content_start; - while index < content_end { - match bytes[index] { - b'{' if bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - let expr_start = skip_ascii_whitespace(bytes, index + 1, content_end); - if let Some(separator) = replacement_field_separator(bytes, expr_start, content_end) - && bytes[separator] == b':' - { - let format_end = - replacement_field_closing_brace(bytes, separator + 1, content_end) - .unwrap_or(content_end); - if bytes[separator + 1..format_end].contains(&b'\n') { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: newlines are not allowed in format specifiers for single quoted {prefix}s" - ), - quote_index, - quote_index + 1, - )); - } - } - index += 1; - } - _ => index += 1, - } - } - None -} - fn interpolated_string_prefix(bytes: &[u8], quote: usize) -> Option<&'static str> { let prev = quote.checked_sub(1).and_then(|index| bytes.get(index))?; let lower_prev = prev.to_ascii_lowercase(); @@ -4774,1559 +658,6 @@ fn interpolated_string_content_range(bytes: &[u8], quote_index: usize) -> Option Some((quote_index + quote_len, end - quote_len)) } -fn quoted_string_content_range( - bytes: &[u8], - quote_index: usize, - quote: u8, -) -> Option<(usize, usize)> { - let triple = - bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); - let quote_len = if triple { 3 } else { 1 }; - let content_start = quote_index + quote_len; - let mut index = content_start; - while index < bytes.len() { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - } else if (triple - && bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e)) - || (!triple && bytes[index] == quote) - { - return Some((content_start, index)); - } else { - index += 1; - } - } - None -} - -fn invalid_replacement_field_error( - bytes: &[u8], - start: usize, - end: usize, - prefix: &str, -) -> Option { - let mut index = start; - while index < end { - match bytes[index] { - b'{' if bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - if let Some(error) = replacement_field_error(bytes, index, end, prefix, 0) { - return Some(error); - } - // Resume in the literal text. Inside the field a `#` starts a comment and a `{` - // opens no new field, so walking on byte by byte would misread both. - index = replacement_field_end(bytes, index, end).unwrap_or(end); - } - _ => index += 1, - } - } - None -} - -/// Position just past a replacement field's closing brace. The expression part is code, where a -/// comment runs to the end of the line; past the separator only nested fields nest. -fn replacement_field_end(bytes: &[u8], open: usize, end: usize) -> Option { - let expr_start = skip_replacement_field_trivia(bytes, open + 1, end); - let separator = replacement_field_separator(bytes, expr_start, end)?; - if bytes[separator] == b'}' { - return Some(separator + 1); - } - replacement_field_closing_brace(bytes, separator + 1, end).map(|brace| brace + 1) -} - -/// The one-based line holding `offset`. Counted on demand: these scanners only run once the -/// source has failed to parse, and tracking a line through the spans they skip would have to -/// re-scan them anyway. -fn line_of_offset(bytes: &[u8], offset: usize) -> usize { - 1 + bytes[..offset] - .iter() - .filter(|byte| **byte == b'\n') - .count() -} - -/// Brackets opened in a replacement field's expression must close before the field does. CPython -/// words these exactly as it does for brackets anywhere else, except that a closer with nothing -/// open is attributed to the literal. `start..end` must cover only the expression: a format spec -/// is plain text, so a bracket there opens nothing. -fn replacement_field_bracket_error( - bytes: &[u8], - start: usize, - end: usize, - prefix: &str, -) -> Option { - // Each entry carries where its bracket opened, so a mismatch can name that line. - let mut stack: Vec<(u8, usize)> = Vec::new(); - let mut index = start; - while index < end { - let byte = bytes[index]; - match byte { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - continue; - } - b'#' => { - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - b'(' | b'[' | b'{' => stack.push((byte, index)), - b')' | b']' | b'}' => { - let expected = expected_opening_bracket(byte as char) as u8; - match stack.last() { - // The field's own closing brace: everything inside it balanced. - None if byte == b'}' => return None, - None => { - return Some(CpythonDiagnostic::new( - format!("{prefix}: unmatched '{}'", byte as char), - index, - index + 1, - )); - } - Some(&(opening, _)) if opening == expected => { - stack.pop(); - } - Some(&(opening, opened_at)) => { - // CPython names the opening line only when it is not the closing one, - // comparing `parenlinenostack[level]` against the current `lineno`. - let opening_line = line_of_offset(bytes, opened_at); - let suffix = if opening_line == line_of_offset(bytes, index) { - String::new() - } else { - format!(" on line {opening_line}") - }; - return Some(CpythonDiagnostic::new( - format!( - "closing parenthesis '{}' does not match opening parenthesis '{}'{suffix}", - byte as char, opening as char - ), - index, - index + 1, - )); - } - } - } - _ => {} - } - index += 1; - } - None -} - -/// A `#` in a replacement field's expression starts a comment. When no newline follows it before -/// the end of the literal, the comment swallows the closing brace and the field can never close. -/// `start..end` must cover only the expression: `#` is an ordinary character in a format spec, -/// where it selects the alternate form. -fn replacement_field_comment_error( - bytes: &[u8], - open: usize, - start: usize, - end: usize, - literal_end: usize, -) -> Option { - let mut index = start; - while index < end { - match bytes[index] { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - continue; - } - b'#' => { - if !bytes[index..literal_end].contains(&b'\n') { - return Some( - CpythonDiagnostic::new("'{' was never closed".to_owned(), open, open + 1) - .with_unclosed_bracket(), - ); - } - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - _ => index += 1, - } - } - None -} - -/// Advances past a comment, stopping on the newline that ends it. -fn skip_replacement_field_comment(bytes: &[u8], mut index: usize, end: usize) -> usize { - while index < end && bytes[index] != b'\n' { - index += 1; - } - index -} - -/// Whitespace and comments may both precede a replacement field's expression. -fn skip_replacement_field_trivia(bytes: &[u8], mut index: usize, end: usize) -> usize { - loop { - index = skip_ascii_whitespace(bytes, index, end); - if index < end && bytes[index] == b'#' { - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - return index; - } -} - -/// Bounds the mutual recursion between a replacement field and the format spec it contains, -/// so that pathologically nested input cannot exhaust the stack. -const MAX_REPLACEMENT_FIELD_DEPTH: usize = 32; - -fn replacement_field_error( - bytes: &[u8], - open: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let expr_start = skip_replacement_field_trivia(bytes, open + 1, end); - - // The expression runs up to the field's first top-level `=`, `!`, `:` or `}`. Everything past - // it is a conversion or a format spec, which are text rather than code, so the checks that - // treat the field as code have to stop here. - let separator = replacement_field_separator(bytes, expr_start, end); - let expr_end = separator.unwrap_or(end); - - if let Some(backslash) = replacement_field_line_continuation(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - "unexpected character after line continuation character".to_owned(), - backslash + 1, - (backslash + 2).min(end), - )); - } - if let Some(quote) = unterminated_string_in_replacement_field(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - unterminated_string_message(1, false, false, None), - quote, - quote + 1, - )); - } - // Brackets are reported ahead of everything else in the field, and a comment that swallows - // the closing brace ahead of the expression itself. The comment scan starts at the brace - // because a comment may also sit between it and the expression. - if let Some(error) = replacement_field_bracket_error(bytes, expr_start, expr_end, prefix) { - return Some(error); - } - if let Some(error) = replacement_field_comment_error(bytes, open, open + 1, expr_end, end) { - return Some(error); - } - - // Nothing follows `{` at all, so the field simply never closed. CPython asks for the - // brace here; an expression that merely fails to start is reported further down. - if expr_start >= end { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - open, - open + 1, - )); - } - - if let marker @ (b'=' | b'!' | b':' | b'}') = bytes[expr_start] - && is_replacement_field_marker(bytes, expr_start) - { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: valid expression required before '{}'", - marker as char - ), - expr_start, - expr_start + 1, - )); - } - - // A `{` display is a valid expression, but only if something can follow it. CPython - // reports the inner brace when it cannot, as in `f'{3:{{>10}'`. - if bytes[expr_start] == b'{' { - let inner = skip_ascii_whitespace(bytes, expr_start + 1, end); - if inner < end - && bytes[inner] != b'}' - && invalid_replacement_expression_start(bytes, inner, end) - { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting a valid expression after '{{'"), - expr_start, - expr_start + 1, - )); - } - } - - if starts_identifier(bytes, expr_start, b"lambda") { - return Some(CpythonDiagnostic::new( - format!("{prefix}: lambda expressions are not allowed without parentheses"), - expr_start, - expr_start + b"lambda".len(), - )); - } - - if invalid_replacement_expression_start(bytes, expr_start, end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting a valid expression after '{{'"), - open, - open + 1, - )); - } - - // Adjacent atoms in a replacement field are a missing comma, not an - // f-string "expecting '}'" at the opening brace. Prefix `yield`/`await`/ - // `not` and continuation keywords (`and`, `for`, ...) are not a second atom. - if !starts_identifier(bytes, expr_start, b"yield") - && !starts_identifier(bytes, expr_start, b"await") - && !starts_identifier(bytes, expr_start, b"not") - && let Some(atom_end) = adjacent_atom_end(bytes, expr_start) - { - let next = skip_ascii_whitespace(bytes, atom_end, expr_end); - if next > atom_end - && expression_atom_start(bytes, next) - && !expression_continuation_keyword(bytes, next) - { - let second_end = adjacent_atom_end(bytes, next).unwrap_or(next + 1); - return Some(CpythonDiagnostic::new( - "invalid syntax. Perhaps you forgot a comma?".to_owned(), - expr_start, - second_end, - )); - } - } - - // The expression started fine but ran into a character that cannot continue it. CPython - // points at that character and lists the separators it wanted instead. - if let Some(stray) = replacement_expression_stray_character(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '=', or '!', or ':', or '}}'"), - stray, - stray + 1, - )); - } - - let Some(separator) = separator else { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - open, - open + 1, - )); - }; - - // Only now that the field is known to close. A stray character above is a finished token - // the parser rejects on lookahead, but a dangling operator makes it ask for one more, and - // producing that token runs into the literal's closing quote: lexer.c then answers from - // `INSIDE_FSTRING(tok)` with "%c-string: expecting '}'" before any `invalid_` rule is tried. - if let Some(operator) = dangling_operator(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '=', or '!', or ':', or '}}'"), - operator, - operator + 1, - )); - } - - if let Some(lambda_at) = top_level_lambda(bytes, expr_start, expr_end) - && lambda_allowed_at_expression_position(bytes, expr_start, lambda_at) - { - return Some(CpythonDiagnostic::new( - format!("{prefix}: lambda expressions are not allowed without parentheses"), - lambda_at, - lambda_at + b"lambda".len(), - )); - } - - if bytes[separator] == b':' - && replacement_expression_has_parse_error(bytes, expr_start, separator) - { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - expr_start, - separator, - )); - } - - match bytes[separator] { - b'=' => invalid_debug_expression_error(bytes, separator, end, prefix, depth), - b'!' => invalid_conversion_error(bytes, separator, end, prefix, depth), - b':' => invalid_format_spec_error(bytes, separator, end, prefix, depth), - b'}' => None, - _ => unreachable!(), - } -} - -fn replacement_field_line_continuation( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'\\' => return Some(index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'=' | b'!' | b':' | b'}' - if level == 0 && is_replacement_field_marker(bytes, index) => - { - return None; - } - _ => index += 1, - } - } - None -} - -fn unterminated_string_in_replacement_field( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - while index < end { - match bytes[index] { - quote @ (b'\'' | b'"') => { - let string_end = skip_quoted_string(bytes, index); - if string_end >= end && !bytes[index + 1..end].contains("e) { - return Some(index); - } - index = string_end; - } - b'#' => index = skip_replacement_field_comment(bytes, index, end), - _ => index += 1, - } - } - None -} - -fn top_level_lambda(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"lambda") => { - return Some(index); - } - _ => index += 1, - } - } - None -} - -/// Unparenthesized `lambda` is allowed in the same places a `lambdef` can appear: -/// at the start of the field or after a top-level comma. -fn lambda_allowed_at_expression_position(bytes: &[u8], mut index: usize, lambda_at: usize) -> bool { - let mut after_comma = true; - let mut level = 0usize; - while index < lambda_at { - match bytes[index] { - b'\'' | b'"' => { - after_comma = false; - index = skip_quoted_string(bytes, index); - } - b'#' => index = skip_replacement_field_comment(bytes, index, lambda_at), - b'(' | b'[' | b'{' => { - level += 1; - after_comma = false; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - after_comma = false; - index += 1; - } - b',' if level == 0 => { - after_comma = true; - index += 1; - } - b' ' | b'\t' | b'\n' | b'\r' | b'\x0c' => index += 1, - _ => { - after_comma = false; - index += 1; - } - } - } - after_comma -} - -fn replacement_expression_has_parse_error(bytes: &[u8], start: usize, end: usize) -> bool { - let Ok(expression) = ::core::str::from_utf8(&bytes[start..end]) else { - return false; - }; - parser::parse_expression(expression).is_err() -} - -fn invalid_replacement_expression_start(bytes: &[u8], index: usize, end: usize) -> bool { - if index >= end { - return true; - } - - // A leading `.` is Ellipsis or a float without an integer part, both of which start a - // perfectly good expression. - if bytes[index] == b'.' - && (bytes[index..end].starts_with(b"...") - || bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit())) - { - return false; - } - - if matches!( - bytes[index], - b'.' | b',' | b'*' | b'/' | b'%' | b'&' | b'|' | b'^' | b'<' | b'>' | b'@' | b'=' | b'!' - ) || is_stray_in_replacement_expression(bytes[index]) - { - return true; - } - - if matches!(bytes[index], b'+' | b'-' | b'~') { - let operand = skip_ascii_whitespace(bytes, index + 1, end); - if starts_identifier(bytes, operand, b"lambda") { - return true; - } - return !bytes.get(operand).is_some_and(|byte| { - *byte >= 0x80 - || *byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(*byte, b'\'' | b'"' | b'(' | b'[' | b'{') - // `-.5` is a signed float. - || (*byte == b'.' - && bytes - .get(operand + 1) - .is_some_and(|next| next.is_ascii_digit())) - }); - } - - [ - b"and".as_slice(), - b"as".as_slice(), - b"else".as_slice(), - b"for".as_slice(), - b"if".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"or".as_slice(), - ] - .iter() - .any(|keyword| starts_identifier(bytes, index, keyword)) -} - -/// A top-level `=` or `!` marks a debug specifier or a conversion only when it is not part of a -/// longer operator. CPython's tokenizer consumes `==`, `!=`, `<=`, `+=` and the rest as single -/// tokens before it ever considers the debug marker, so those must not end the expression here. -fn is_replacement_field_marker(bytes: &[u8], index: usize) -> bool { - match bytes[index] { - b'=' => { - if bytes.get(index + 1) == Some(&b'=') { - return false; - } - !index - .checked_sub(1) - .and_then(|previous| bytes.get(previous).copied()) - .is_some_and(precedes_equals_in_one_operator) - } - b'!' => bytes.get(index + 1) != Some(&b'='), - _ => true, - } -} - -fn replacement_field_separator(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'=' | b'!' | b':' | b'}' - if level == 0 && is_replacement_field_marker(bytes, index) => - { - return Some(index); - } - _ => index += 1, - } - } - None -} - -/// Characters that can neither start nor continue a replacement field expression. CPython's -/// interpolated-string tokenizer stops at them and asks for a separator instead. -const fn is_stray_in_replacement_expression(byte: u8) -> bool { - matches!(byte, b';' | b'$' | b'?' | b'`') -} - -fn replacement_expression_stray_character( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - byte if is_stray_in_replacement_expression(byte) => return Some(index), - _ => index += 1, - } - } - None -} - -/// A byte that pairs with a following `=` to make one operator token, so that the `=` belongs -/// to it rather than opening a debug specifier: `==`, `!=`, `<=`, `>=`, the augmented -/// assignments, and the `:=` walrus. Against `is_dangling_operator_byte` below, this has `:` -/// and lacks `.` and `~`, because neither `.=` nor `~=` is a Python operator. -const fn precedes_equals_in_one_operator(byte: u8) -> bool { - matches!( - byte, - b'!' | b'%' - | b'&' - | b'*' - | b'+' - | b'-' - | b'/' - | b':' - | b'<' - | b'=' - | b'>' - | b'@' - | b'^' - | b'|' - ) -} - -/// Characters that make up an operator which cannot end an expression. Against -/// `precedes_equals_in_one_operator` above, this has `.` and `~` for `a.b.` and `~a`, and lacks -/// `:`, which separates a replacement field's parts rather than operating on anything. -const fn is_dangling_operator_byte(byte: u8) -> bool { - matches!( - byte, - b'!' | b'%' - | b'&' - | b'*' - | b'+' - | b'-' - | b'.' - | b'/' - | b'<' - | b'=' - | b'>' - | b'@' - | b'^' - | b'|' - | b'~' - ) -} - -/// The identifier that ends just before `word`, skipping the whitespace between them. -fn preceding_keyword(bytes: &[u8], start: usize, word: usize) -> Option<(usize, &[u8])> { - let mut cursor = word; - while cursor > start && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\r' | b'\n' | 0x0c) { - cursor -= 1; - } - let stop = cursor; - while cursor > start && is_ascii_identifier_char(bytes[cursor - 1]) { - cursor -= 1; - } - (cursor < stop).then(|| (cursor, &bytes[cursor..stop])) -} - -/// A replacement field expression cannot end on an operator that still wants an operand. -/// CPython's tokenizer then reaches the field brace with the expression unfinished and asks for -/// a separator, pointing at the operator that left it that way. Returns that operator's start. -fn dangling_operator(bytes: &[u8], start: usize, end: usize) -> Option { - // The region cannot be read backwards: the newline that ends a comment is whitespace, so - // trimming from `end` would step into the comment body. Walk forward instead and keep the - // position just past the last byte that is really part of the expression. - let mut index = start; - let mut tail = start; - while index < end { - match bytes[index] { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index).min(end); - tail = index; - } - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b' ' | b'\t' | b'\r' | b'\n' | 0x0c => index += 1, - _ => { - index += 1; - tail = index; - } - } - } - if tail == start { - return None; - } - - // A trailing keyword operator reads back as an identifier. - if is_ascii_identifier_char(bytes[tail - 1]) { - let mut word = tail; - while word > start && is_ascii_identifier_char(bytes[word - 1]) { - word -= 1; - } - if ![ - b"and".as_slice(), - b"or".as_slice(), - b"not".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"if".as_slice(), - b"for".as_slice(), - ] - .iter() - .any(|keyword| &bytes[word..tail] == *keyword) - { - return None; - } - // `is not` and `not in` are single operators, and CPython points at their first word. - let previous = preceding_keyword(bytes, start, word); - return Some(match (previous, &bytes[word..tail]) { - (Some((at, b"is")), b"not") | (Some((at, b"not")), b"in") => at, - _ => word, - }); - } - - if !is_dangling_operator_byte(bytes[tail - 1]) { - return None; - } - let mut token = tail; - while token > start && is_dangling_operator_byte(bytes[token - 1]) { - token -= 1; - } - - // `1.` is a float: that dot finishes the literal rather than dangling. - if bytes[token..tail] == *b"." && token > start && bytes[token - 1].is_ascii_digit() { - return None; - } - // `...` is Ellipsis, so consume whole triples: a leftover dot is what dangles. - if bytes[token..tail].iter().all(|byte| *byte == b'.') { - let leftover = (tail - token) % 3; - return (leftover != 0).then(|| tail - leftover); - } - Some(token) -} - -fn invalid_debug_expression_error( - bytes: &[u8], - equals: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let next = equals + 1; - if next >= end { - return None; - } - // A conversion or format spec may follow `=`, and is validated exactly as it would be - // when following the expression directly. - match bytes[next] { - b'!' => return invalid_conversion_error(bytes, next, end, prefix, depth), - b':' => return invalid_format_spec_error(bytes, next, end, prefix, depth), - b'}' => return None, - _ => {} - } - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '!', or ':', or '}}'"), - next, - next.saturating_add(1).min(end), - )) -} - -fn invalid_conversion_error( - bytes: &[u8], - bang: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let next = bang + 1; - if next >= end { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - bang, - bang + 1, - )); - } - - if bytes[next].is_ascii_whitespace() { - let following = skip_ascii_whitespace(bytes, next, end); - let message = if bytes - .get(following) - .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') - { - "conversion type must come right after the exclamation mark" - } else { - "missing conversion character" - }; - return Some(CpythonDiagnostic::new( - format!("{prefix}: {message}"), - next, - next + 1, - )); - } - - if matches!(bytes[next], b':' | b'}') { - return Some(CpythonDiagnostic::new( - format!("{prefix}: missing conversion character"), - next, - next + 1, - )); - } - - // A non-ASCII character is still a conversion character as far as CPython is concerned, - // so it gets named in the message like any other invalid one. - if bytes[next] >= 0x80 - && let Some(character) = ::core::str::from_utf8(&bytes[next..end]) - .ok() - .and_then(|text| text.chars().next()) - { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: invalid conversion character '{character}': expected 's', 'r', or 'a'" - ), - next, - next + character.len_utf8(), - )); - } - - if !bytes[next].is_ascii_alphabetic() && bytes[next] != b'_' { - return Some(CpythonDiagnostic::new( - format!("{prefix}: invalid conversion character"), - next, - next + 1, - )); - } - - let conversion_end = identifier_end(bytes, next, end); - let conversion = &bytes[next..conversion_end]; - if !matches!(conversion, b"s" | b"r" | b"a") { - let conversion = ::core::str::from_utf8(conversion).unwrap_or(""); - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: invalid conversion character '{conversion}': expected 's', 'r', or 'a'" - ), - next, - conversion_end, - )); - } - - if conversion_end >= end { - return None; - } - - match bytes[conversion_end] { - b':' => return invalid_format_spec_error(bytes, conversion_end, end, prefix, depth), - b'}' => return None, - _ => {} - } - - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting ':' or '}}'"), - conversion_end, - conversion_end + 1, - )) -} - -fn invalid_format_spec_error( - bytes: &[u8], - colon: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - // Unlike literal text, a format spec has no `{{` escape: every `{` opens a nested - // replacement field, so the outer scanner cannot validate these for us. - let mut index = colon + 1; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - if depth < MAX_REPLACEMENT_FIELD_DEPTH - && let Some(error) = - replacement_field_error(bytes, index, end, prefix, depth + 1) - { - return Some(error); - } - // Resume after the nested field. Walking into it again from every enclosing - // spec would re-scan the same bytes once per level, which costs O(2^depth). - let Some(after) = replacement_field_end(bytes, index, end) else { - break; - }; - index = after; - } - b'}' => return None, - _ => index += 1, - } - } - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}', or format specs"), - colon, - colon + 1, - )) -} - -fn replacement_field_closing_brace(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - level += 1; - index += 1; - } - b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'}' => return Some(index), - _ => index += 1, - } - } - None -} - -fn skip_ascii_whitespace(bytes: &[u8], mut index: usize, end: usize) -> usize { - while index < end && matches!(bytes[index], b' ' | b'\t' | b'\r' | b'\n' | 0x0c) { - index += 1; - } - index -} - -fn string_literal_end_at(bytes: &[u8], index: usize) -> Option { - match bytes.get(index).copied()? { - b'\'' | b'"' => Some(skip_quoted_string(bytes, index)), - first if first.is_ascii_alphabetic() => { - if matches!(bytes.get(index + 1), Some(b'\'' | b'"')) { - return string_literal_prefix(bytes, index, index + 1) - .then(|| skip_quoted_string(bytes, index + 1)); - } - if matches!(bytes.get(index + 2), Some(b'\'' | b'"')) { - return string_literal_prefix(bytes, index, index + 2) - .then(|| skip_quoted_string(bytes, index + 2)); - } - None - } - _ => None, - } -} - -fn string_literal_prefix(bytes: &[u8], start: usize, quote: usize) -> bool { - let prefix = &bytes[start..quote]; - let valid = matches!( - prefix, - b"b" | b"B" - | b"r" - | b"R" - | b"u" - | b"U" - | b"f" - | b"F" - | b"t" - | b"T" - | b"br" - | b"bR" - | b"Br" - | b"BR" - | b"rb" - | b"rB" - | b"Rb" - | b"RB" - | b"fr" - | b"fR" - | b"Fr" - | b"FR" - | b"rf" - | b"rF" - | b"Rf" - | b"RF" - | b"tr" - | b"tR" - | b"Tr" - | b"TR" - | b"rt" - | b"rT" - | b"Rt" - | b"RT" - ); - valid && (start == 0 || !is_ascii_identifier_char(bytes[start - 1])) -} - -fn invalid_expression_error(source: &str) -> Option { - invalid_string_expression_error(source).or_else(|| missing_comma_expression_error(source)) -} - -fn invalid_string_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - if let Some(first_string_end) = string_literal_end_at(bytes, index) { - let expr_start = skip_ascii_whitespace(bytes, first_string_end, bytes.len()); - if expression_atom_start(bytes, expr_start) - && let Some(expr_end) = adjacent_atom_end(bytes, expr_start) - { - let next = skip_ascii_whitespace(bytes, expr_end, bytes.len()); - if string_literal_end_at(bytes, next).is_some() { - return Some(CpythonDiagnostic::new( - "invalid syntax. Is this intended to be part of the string?".to_owned(), - expr_start, - expr_end, - )); - } - } - index = first_string_end; - } else { - index += 1; - } - } - None -} - -fn missing_comma_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut stack: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - if bytes[index] == b'#' { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } else if let Some(string_end) = string_literal_end_at(bytes, index) { - index = string_end; - } else { - match bytes[index] { - b'(' | b'[' | b'{' => { - if bytes[index] == b'[' && opening_bracket_is_class_type_params(bytes, index) { - let Some(close) = matching_delimiter(bytes, index, b']') else { - index += 1; - continue; - }; - index = close + 1; - continue; - } - stack.push(bytes[index]); - index += 1; - } - b')' | b']' | b'}' => { - stack.pop(); - index += 1; - } - _ if !stack.is_empty() && expression_continuation_keyword(bytes, index) => { - index = identifier_end(bytes, index, bytes.len()); - } - byte if !stack.is_empty() && expression_atom_start_byte(byte) => { - // `yield x` / `await x` are prefix expressions, not two - // adjacent atoms missing a comma. - if starts_identifier(bytes, index, b"yield") { - index = identifier_end(bytes, index, bytes.len()); - let after_yield = skip_ascii_whitespace(bytes, index, bytes.len()); - if starts_identifier(bytes, after_yield, b"from") { - index = identifier_end(bytes, after_yield, bytes.len()); - } - continue; - } - if starts_identifier(bytes, index, b"await") { - index = identifier_end(bytes, index, bytes.len()); - continue; - } - let atom_end = adjacent_atom_end(bytes, index).unwrap_or(index + 1); - let next = skip_ascii_whitespace(bytes, atom_end, bytes.len()); - if next > atom_end - && expression_atom_start(bytes, next) - && !expression_continuation_keyword(bytes, next) - { - // The diagnostic covers both atoms, so it ends past - // the whole second one. `next + 1` ended past its - // first byte instead, which is only the same thing - // when that atom is one ASCII character. - let second_end = adjacent_atom_end(bytes, next).unwrap_or(next + 1); - return Some(CpythonDiagnostic::new( - "invalid syntax. Perhaps you forgot a comma?".to_owned(), - index, - second_end, - )); - } - index = atom_end; - } - _ => index += 1, - } - } - } - None -} - -fn opening_bracket_is_class_type_params(bytes: &[u8], bracket: usize) -> bool { - let mut cursor = bracket; - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - cursor >= 5 && starts_identifier(bytes, cursor - 5, b"class") -} - -fn expression_continuation_keyword(bytes: &[u8], index: usize) -> bool { - [ - b"and".as_slice(), - b"else".as_slice(), - b"for".as_slice(), - b"if".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"not".as_slice(), - b"or".as_slice(), - ] - .iter() - .any(|keyword| starts_identifier(bytes, index, keyword)) -} - -fn expression_atom_start(bytes: &[u8], index: usize) -> bool { - bytes - .get(index) - .is_some_and(|byte| expression_atom_start_byte(*byte)) - || string_literal_end_at(bytes, index).is_some() -} - -fn expression_atom_start_byte(byte: u8) -> bool { - byte >= 0x80 - || byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(byte, b'\'' | b'"' | b'(' | b'[' | b'{') -} - -fn adjacent_atom_end(bytes: &[u8], index: usize) -> Option { - if let Some(string_end) = string_literal_end_at(bytes, index) { - return Some(string_end); - } - match bytes.get(index).copied()? { - byte if byte >= 0x80 || byte == b'_' || byte.is_ascii_alphabetic() => { - Some(identifier_end(bytes, index, bytes.len())) - } - byte if byte.is_ascii_digit() => { - let mut end = index + 1; - while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') { - end += 1; - } - Some(end) - } - b'(' | b'[' | b'{' => Some(index + 1), - _ => None, - } -} - -struct OpenDelimiter { - position: usize, - opener: u8, - in_format_spec: bool, -} - -/// An f-string or t-string that runs off the end with a replacement field still open reports -/// the brace rather than the missing quote, matching CPython — but only while the tokenizer is -/// still reading the field's *expression*. Past the field's own `:` it is emitting FSTRING_MIDDLE -/// again, so running out of input there is an unterminated literal like any other, and -/// `f'{a:>5` gets the quote message where `f'{a` gets the brace. -fn unclosed_replacement_field_error( - bytes: &[u8], - quote_start: usize, - content_start: usize, - content_end: usize, -) -> Option { - interpolated_string_prefix(bytes, quote_start)?; - - // Brackets and quotes are delimiters only once a field is open; in the literal's text they - // are ordinary characters. And past a field's own format spec the tokenizer emits literal - // text again, so a field still open at end of input is an unclosed brace only before that. - let mut open: Vec = Vec::new(); - let mut index = content_start; - while index < content_end { - let inside_expression = matches!( - open.last(), - Some(OpenDelimiter { - opener: b'{', - in_format_spec: false, - .. - }) - ); - match bytes[index] { - // `{{` and `}}` escape only in literal text, which is where a brace at depth zero is. - b'{' if open.is_empty() && bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if open.is_empty() && bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - open.push(OpenDelimiter { - position: index, - opener: b'{', - in_format_spec: false, - }); - index += 1; - } - b'}' => { - open.pop(); - index += 1; - } - byte @ (b'(' | b'[') if !open.is_empty() => { - open.push(OpenDelimiter { - position: index, - opener: byte, - in_format_spec: false, - }); - index += 1; - } - b')' | b']' if !open.is_empty() => { - open.pop(); - index += 1; - } - // Only the field's own `:` opens a format spec — one in a slice, a display or a - // lambda belongs to whatever bracket encloses it. - b':' if inside_expression => { - open.last_mut().expect("a brace is open").in_format_spec = true; - index += 1; - } - b'\'' | b'"' if !open.is_empty() => { - index = skip_quoted_string(bytes, index).min(content_end); - } - b'#' if inside_expression => { - index = skip_replacement_field_comment(bytes, index, content_end); - } - _ => index += 1, - } - } - - // CPython names the innermost unclosed delimiter, so a bracket opened inside the expression - // takes the message from the field that encloses it. - let &OpenDelimiter { - position, - opener, - in_format_spec, - } = open.last()?; - (!in_format_spec).then(|| { - CpythonDiagnostic::new( - format!("'{}' was never closed", opener as char), - position, - position + 1, - ) - .with_unclosed_bracket() - }) -} - -fn unterminated_string_message( - detected_line: usize, - triple: bool, - has_escaped_quote: bool, - prefix: Option<&str>, -) -> String { - // CPython names the literal kind, e.g. "unterminated f-string literal". - let kind = prefix.unwrap_or("string"); - if triple { - format!("unterminated triple-quoted {kind} literal (detected at line {detected_line})") - // The escaped-quote hint belongs to the plain-string branch of CPython's tokenizer alone. - // Its interpolated branch has only the two forms above and below, so pairing the hint with - // a prefix would invent a message CPython never emits. - } else if has_escaped_quote && prefix.is_none() { - format!( - "unterminated {kind} literal (detected at line {detected_line}); perhaps you escaped the end quote?" - ) - } else { - format!("unterminated {kind} literal (detected at line {detected_line})") - } -} - -fn expected_opening_bracket(closing: char) -> char { - match closing { - ')' => '(', - ']' => '[', - '}' => '{', - _ => unreachable!(), - } -} - -/// A bracket diagnostic, and whether it is an opener that was never closed. The caller needs -/// that apart from the message because ruff reports the unclosed case as an EOF error. -#[derive(Clone)] -struct BracketError { - diagnostic: CpythonDiagnostic, - unclosed: bool, -} - -fn bracket_syntax_error(source: &str) -> Option { - let mut stack: Vec<(char, usize, usize)> = Vec::new(); - let mut in_string = false; - let mut string_quote = '\0'; - let mut triple_quote = false; - let mut escape_next = false; - let mut is_raw_string = false; - let mut line = 1usize; - - let chars: Vec<(usize, char)> = source.char_indices().collect(); - let mut index = 0; - while index < chars.len() { - let (byte_offset, ch) = chars[index]; - - if ch == '\n' { - line += 1; - } - - if escape_next { - escape_next = false; - index += 1; - continue; - } - - if in_string { - if ch == '\\' && !is_raw_string { - escape_next = true; - } else if triple_quote { - if ch == string_quote - && index + 2 < chars.len() - && chars[index + 1].1 == string_quote - && chars[index + 2].1 == string_quote - { - in_string = false; - index += 3; - continue; - } - } else if ch == string_quote { - in_string = false; - } - index += 1; - continue; - } - - if ch == '#' { - while index < chars.len() && chars[index].1 != '\n' { - index += 1; - } - continue; - } - - if ch == '\\' { - match chars.get(index + 1).map(|(_, next)| *next) { - Some('\n' | '\r') => { - escape_next = true; - index += 1; - continue; - } - Some(_) => { - index += 2; - continue; - } - None => { - index += 1; - continue; - } - } - } - - if ch == '\'' || ch == '"' { - is_raw_string = false; - for look_back in 1..=2.min(index) { - let prev = chars[index - look_back].1; - if matches!(prev, 'r' | 'R') { - is_raw_string = true; - break; - } - if !matches!(prev, 'b' | 'B' | 'f' | 'F' | 'u' | 'U') { - break; - } - } - string_quote = ch; - if index + 2 < chars.len() && chars[index + 1].1 == ch && chars[index + 2].1 == ch { - triple_quote = true; - in_string = true; - index += 3; - continue; - } - triple_quote = false; - in_string = true; - index += 1; - continue; - } - - match ch { - '(' | '[' | '{' => stack.push((ch, byte_offset, line)), - ')' | ']' | '}' => { - let expected = expected_opening_bracket(ch); - let Some(&(opening, _, opening_line)) = stack.last() else { - return Some(BracketError { - diagnostic: CpythonDiagnostic::new( - format!("unmatched '{ch}'"), - byte_offset, - byte_offset, - ), - unclosed: false, - }); - }; - if opening == expected { - stack.pop(); - } else { - let suffix = if opening_line != line { - format!(" on line {opening_line}") - } else { - String::new() - }; - return Some(BracketError { - diagnostic: CpythonDiagnostic::new( - format!( - "closing parenthesis '{ch}' does not match opening parenthesis '{opening}'{suffix}" - ), - byte_offset, - byte_offset, - ), - unclosed: false, - }); - } - } - _ => {} - } - - index += 1; - } - - stack.last().map(|(opening, byte_offset, _)| BracketError { - diagnostic: CpythonDiagnostic::new( - format!("'{opening}' was never closed"), - *byte_offset, - *byte_offset, - ), - unclosed: true, - }) -} - -fn is_legacy_statement_expression_start(byte: u8) -> bool { - byte >= 0x80 - || byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(byte, b'\'' | b'"' | b'{' | b'[') -} - -fn legacy_statement_container_has_invalid_attribute(bytes: &[u8], start: usize) -> bool { - let Some(&opening) = bytes.get(start) else { - return false; - }; - if !matches!(opening, b'{' | b'[') { - return false; - } - - let mut index = start; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' | b';' if level == 0 => return false, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - if level == 0 { - return false; - } - } - b'.' => { - let mut cursor = index + 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if matches!(bytes.get(cursor), Some(b')' | b']' | b'}')) { - return true; - } - index += 1; - } - _ => index += 1, - } - } - false -} - -fn invalid_legacy_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'p' | b'e' => { - let keyword = if starts_identifier(bytes, index, b"print") { - Some("print") - } else if starts_identifier(bytes, index, b"exec") { - Some("exec") - } else { - None - }; - let Some(keyword) = keyword else { - index += 1; - continue; - }; - let after_keyword = index + keyword.len(); - if !matches!(bytes.get(after_keyword), Some(b' ' | b'\t' | b'\x0c')) { - index = after_keyword; - continue; - } - let mut cursor = after_keyword; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if legacy_statement_container_has_invalid_attribute(bytes, cursor) { - index = after_keyword; - continue; - } - if bytes.get(cursor).is_some_and(|byte| { - *byte != b'(' && is_legacy_statement_expression_start(*byte) - }) { - return Some(CpythonDiagnostic::new( - format!( - "Missing parentheses in call to '{keyword}'. Did you mean {keyword}(...)?" - ), - index, - after_keyword, - )); - } - index = after_keyword; - } - _ => index += 1, - } - } - None -} - /// Return the syntax error for a decimal integer literal exceeding the configured limit. /// /// The parser has already distinguished integer literals from strings, comments, floats, and @@ -6363,265 +694,6 @@ pub fn long_decimal_integer_literal_error( }) } -fn invalid_parenthesized_import_star_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'f' if starts_identifier(bytes, index, b"from") => { - let mut cursor = index + 4; - while cursor < bytes.len() && !matches!(bytes[cursor], b'\n' | b';') { - if starts_identifier(bytes, cursor, b"import") { - cursor += 6; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\r')) { - cursor += 1; - } - if bytes.get(cursor) == Some(&b'(') { - cursor += 1; - while cursor < bytes.len() - && !matches!(bytes[cursor], b')' | b'\n' | b';') - { - if bytes[cursor] == b'*' { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - cursor, - cursor + 1, - )); - } - cursor += 1; - } - } - break; - } - cursor += 1; - } - index = cursor; - } - _ => index += 1, - } - } - None -} - -fn prefix_letters_before_quote(bytes: &[u8], quote_index: usize) -> Option<(usize, Vec)> { - let mut letters = Vec::new(); - let mut index = quote_index; - while index > 0 { - let byte = bytes[index - 1]; - if !matches!( - byte, - b'r' | b'R' | b'b' | b'B' | b'u' | b'U' | b'f' | b'F' | b't' | b'T' - ) { - break; - } - letters.push(byte.to_ascii_lowercase()); - index -= 1; - } - if letters.is_empty() { - return None; - } - if index > 0 && (bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphabetic()) { - return None; - } - letters.reverse(); - Some((index, letters)) -} - -fn incompatible_string_prefix_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - if let Some((start, letters)) = prefix_letters_before_quote(bytes, index) - && let Some(message) = incompatible_prefix_message(&letters) - { - return Some(CpythonDiagnostic::new(message, start, start + 1)); - } - index = skip_string_token(bytes, index); - } - _ => index += 1, - } - } - None -} - -fn incompatible_prefix_message(letters: &[u8]) -> Option { - if letters.len() < 2 { - return None; - } - let mut seen_r = false; - let mut seen_b = false; - let mut seen_u = false; - let mut seen_f = false; - let mut seen_t = false; - for &letter in letters { - match letter { - b'r' if seen_r => return None, - b'b' if seen_b => return None, - b'u' if seen_u => return None, - b'f' if seen_f => return None, - b't' if seen_t => return None, - b'r' => seen_r = true, - b'b' => seen_b = true, - b'u' => seen_u = true, - b'f' => seen_f = true, - b't' => seen_t = true, - _ => {} - } - } - let pair = if seen_u && seen_b { - ("u", "b") - } else if seen_u && seen_r { - ("u", "r") - } else if seen_u && seen_f { - ("u", "f") - } else if seen_u && seen_t { - ("u", "t") - } else if seen_b && seen_f { - ("b", "f") - } else if seen_b && seen_t { - ("b", "t") - } else if seen_f && seen_t { - ("f", "t") - } else { - return None; - }; - Some(format!( - "'{}' and '{}' prefixes are incompatible", - pair.0, pair.1 - )) -} - -fn malformed_unicode_n_escape_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - let quote_index = index; - let interpolated = interpolated_string_prefix(bytes, quote_index).is_some(); - let Some((content_start, content_end)) = - quoted_string_content_range(bytes, quote_index, bytes[quote_index]) - else { - index = skip_quoted_string(bytes, quote_index); - continue; - }; - if let Some(error) = malformed_unicode_n_in_content( - bytes, - content_start, - content_end, - quote_index, - interpolated, - ) { - return Some(error); - } - index = skip_string_token(bytes, quote_index); - } - _ => index += 1, - } - } - None -} - -fn malformed_unicode_n_in_content( - bytes: &[u8], - start: usize, - end: usize, - quote_index: usize, - interpolated: bool, -) -> Option { - let mut index = start; - let mut brace_depth = 0usize; - while index + 1 < end { - if interpolated && brace_depth == 0 && bytes[index] == b'{' { - if bytes.get(index + 1) == Some(&b'{') { - index += 2; - continue; - } - brace_depth = 1; - index += 1; - continue; - } - if interpolated && brace_depth > 0 { - match bytes[index] { - b'\'' | b'"' => index = skip_string_token(bytes, index).max(index + 1), - b'{' => { - brace_depth += 1; - index += 1; - } - b'}' => { - brace_depth -= 1; - index += 1; - } - _ => index += 1, - } - continue; - } - if bytes[index] == b'\\' && bytes[index + 1] == b'N' { - let escape_start = index; - if bytes.get(index + 2) == Some(&b'{') { - let mut look = index + 3; - while look < end && bytes[look] != b'}' { - look += 1; - } - if look >= end { - return Some(unicode_n_diagnostic(escape_start, end, start, quote_index)); - } - index = look + 1; - continue; - } - return Some(unicode_n_diagnostic( - escape_start, - escape_start + 2, - start, - quote_index, - )); - } - if interpolated && bytes[index] == b'}' && bytes.get(index + 1) == Some(&b'}') { - index += 2; - continue; - } - index += 1; - } - None -} - -fn unicode_n_diagnostic( - escape_start: usize, - escape_end: usize, - content_start: usize, - quote_index: usize, -) -> CpythonDiagnostic { - let start = escape_start - content_start; - let end = escape_end.saturating_sub(content_start).saturating_sub(1); - CpythonDiagnostic::new( - format!( - "(unicode error) 'unicodeescape' codec can't decode bytes in position {start}-{end}: malformed \\N character escape" - ), - quote_index, - quote_index + 1, - ) -} - fn too_many_nested_interpolated_strings(source: &str) -> Option { too_many_nested_interpolated_strings_in(source.as_bytes(), 0, source.len(), 0) } @@ -6668,76 +740,6 @@ fn too_many_nested_interpolated_strings_in( None } -fn too_many_nested_parentheses_error(source: &str) -> Option { - const MAXLEVEL: usize = 200; - - let bytes = source.as_bytes(); - let mut index = 0; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - if level >= MAXLEVEL { - return Some(CpythonDiagnostic::new( - "too many nested parentheses".to_owned(), - index, - index + 1, - )); - } - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_unparenthesized_yield_after_comma_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b',' => { - let mut cursor = index + 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if starts_identifier(bytes, cursor, b"yield") { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - cursor, - cursor + 5, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - /// Horizontal and vertical space the tokenizer skips, not `str::trim()`. const fn is_ascii_tokenizer_whitespace(c: char) -> bool { matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') @@ -6796,6 +798,43 @@ pub fn leading_byte_order_mark_error(source_file: &SourceFile) -> Option Option { + const MAXLEVEL: usize = 200; + + let bytes = source.as_bytes(); + let mut index = 0; + let mut level = 0usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\'' | b'"' => { + index = skip_quoted_string(bytes, index); + } + b'(' | b'[' | b'{' => { + if level >= MAXLEVEL { + return Some(CpythonDiagnostic::new( + "too many nested parentheses".to_owned(), + index, + index, + )); + } + level += 1; + index += 1; + } + b')' | b']' | b'}' => { + level = level.saturating_sub(1); + index += 1; + } + _ => index += 1, + } + } + None +} + /// Source-level errors raised before parsing, as CPython's tokenizer does. /// /// Checking after the parse would build the tree first, and one nested that @@ -6818,17 +857,7 @@ fn post_parse_source_error( if let Some(error) = too_many_nested_interpolated_strings(source_file.source_text()) { return Some(CompileError::from_source_error(source_file, error)); } - if let Some(error) = - long_decimal_integer_literal_error(source_file, tokens, opts.int_max_str_digits) - { - return Some(error); - } - invalid_call_argument_error(source_file.source_text()) - .or_else(|| invalid_match_mapping_rest_wildcard_error(source_file.source_text())) - .or_else(|| invalid_match_as_target_error(source_file.source_text())) - .or_else(|| invalid_unparenthesized_yield_after_comma_error(source_file.source_text())) - .or_else(|| invalid_parenthesized_import_star_error(source_file.source_text())) - .map(|error| CompileError::from_source_error(source_file, error)) + long_decimal_integer_literal_error(source_file, tokens, opts.int_max_str_digits) } fn is_compound_stmt(stmt: &ast::Stmt) -> bool { @@ -7006,7 +1035,7 @@ pub fn unsupported_grammar_error(ast: &ast::Mod, source_file: &SourceFile) -> Op match expr { ast::Expr::FString(fstring) => { for part in &fstring.value { - if let ast::FStringPart::FString(part) = part { + if let ast::FStringPartRef::FString(part) = part { self.check_format_specs("f-string", &part.elements, 0); } } @@ -7042,22 +1071,6 @@ fn single_mode_body_error(body: &[ast::Stmt], source_file: &SourceFile) -> Optio let first_start = source_code.source_location(first.range().start(), PositionEncoding::Utf8); let first_end = source_code.source_location(first.range().end(), PositionEncoding::Utf8); - if body.iter().skip(1).any(|stmt| { - source_code - .source_location(stmt.range().start(), PositionEncoding::Utf8) - .line - > first_start.line - }) { - return Some(CompileError::from_source_error( - source_file, - CpythonDiagnostic::new( - "multiple statements found while compiling a single statement".to_owned(), - first.range().end().to_usize(), - first.range().end().to_usize(), - ), - )); - } - if is_compound_stmt(first) && first_start.line == first_end.line && !ends_with_line_break(source_file.source_text()) @@ -7074,6 +1087,77 @@ fn single_mode_body_error(body: &[ast::Stmt], source_file: &SourceFile) -> Optio None } +/// Converts the first parse error. A tokenizer error that follows an earlier syntax error is +/// only found by the pass over the rest of the source, so it never asks for more input. +fn first_parse_error( + parsed: parser::Parsed, + source_file: &SourceFile, + mode: Mode, +) -> Result, CompileError> { + let Some((first, rest)) = parsed.errors().split_first() else { + return Ok(parsed); + }; + let after_syntax_error = rest.iter().any(|error| { + !matches!(error.error, ParseErrorType::Lexical(_)) + && error.location.start() < first.location.start() + }); + let mut error = CompileError::from_ruff_parse_error(first.clone(), source_file, mode); + if after_syntax_error && let CompileError::Parse(error) = &mut error { + error.is_unclosed_string = false; + } + Err(error) +} + +/// Single mode reads only the first statement when it is a simple statement, so any code after +/// its line is reported as multiple statements, ahead of errors the rest of the source has. +#[must_use] +pub fn single_mode_multiple_statements_error( + source_file: &SourceFile, + parsed: &parser::Parsed, +) -> Option { + let ast::Mod::Module(module) = parsed.syntax() else { + return None; + }; + if is_compound_stmt(module.body.first()?) { + return None; + } + let tokens = parsed.tokens(); + let newline_index = tokens + .iter() + .position(|token| token.kind() == TokenKind::Newline)?; + let newline = &tokens[newline_index]; + if parsed + .errors() + .iter() + .any(|error| error.location.start() < newline.end()) + { + return None; + } + let mut rest = &source_file.source_text()[newline.end().to_usize()..]; + loop { + rest = rest.trim_start_matches([' ', '\t', '\n', '\r', '\x0c']); + match rest.strip_prefix('#') { + Some(comment) => rest = comment.find('\n').map_or("", |end| &comment[end..]), + None if rest.is_empty() => return None, + None => break, + } + } + // The error is at the newline token, which starts at a trailing comment. + let position = match newline_index.checked_sub(1).map(|index| &tokens[index]) { + Some(comment) if comment.kind() == TokenKind::Comment => comment.start(), + _ => newline.start(), + } + .to_usize(); + Some(CompileError::from_source_error( + source_file, + CpythonDiagnostic::new( + "multiple statements found while compiling a single statement".to_owned(), + position, + position, + ), + )) +} + fn single_mode_source_error(ast: &ast::Mod, source_file: &SourceFile) -> Option { let ast::Mod::Module(module) = ast else { return None; @@ -7277,12 +1361,16 @@ fn _compile_with_syntax_warning_handler<'a>( .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), ); pre_parse_source_error(&source_file)?; - let parsed = parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = parser::parse_unchecked(barry_source.source(), parser_options); + if matches!(mode, Mode::Single) + && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) + { return Err(error); } - let parsed = - parsed.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let parsed = first_parse_error(parsed, &source_file, mode)?; if matches!(mode, Mode::Single) && let Some(error) = single_mode_blank_source_error(&source_file) { @@ -7580,12 +1668,16 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + if matches!(mode, Mode::Single) + && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) + { return Err(error); } - let ast = - parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let ast = first_parse_error(parsed, &source_file, mode)?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -7608,12 +1700,11 @@ pub fn _compile_symtable( } Mode::Eval => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let ast = - parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + let ast = first_parse_error(parsed, &source_file, mode)?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -7732,11 +1823,11 @@ mod tests { ), ( "t\"x\" b\"y\"", - "Cannot mix t-string literals with string or bytes literals", + "cannot mix t-string literals with string or bytes literals", ), ( "b\"x\" t\"y\"", - "Cannot mix t-string literals with string or bytes literals", + "cannot mix t-string literals with string or bytes literals", ), // The literals ahead of the first t-string already mix, so CPython's first pass // raises from `_PyPegen_concatenate_strings` before the t-string rule is reached. @@ -7819,7 +1910,6 @@ mod tests { "(print x; '\\N'", "Missing parentheses in call to 'print'. Did you mean print(...)?", ), - ("f'{x'; '", "f-string: expecting '}'"), ("f'{x'; 0x", "f-string: expecting '}'"), ( "print x; f'{x'; 0x", @@ -7835,7 +1925,10 @@ mod tests { "f-string: valid expression required before '}'", ), ("{\\'a\\'}", "unexpected character after line continuation"), - ("\"\\\n\"(1 for c in I,\\\n\\", "'(' was never closed"), + ( + "\"\\\n\"(1 for c in I,\\\n\\", + "unexpected character after line continuation character", + ), ( r"'\N'", "(unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape", diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs index b439ddb6e..65d675563 100644 --- a/crates/stdlib/src/_tokenize.rs +++ b/crates/stdlib/src/_tokenize.rs @@ -210,22 +210,22 @@ mod _tokenize { let kind = token.kind(); let range = token.range(); - // Check for lexical indentation errors. - // Skip when source has tabs — ruff and CPython handle tab - // indentation differently (CPython uses tabsize=8), so ruff may - // report false IndentationErrors for valid mixed-tab code. - if !source.contains('\t') { - for err in errors.iter() { - if !matches!( - err.error, - ParseErrorType::Lexical(LexicalErrorType::IndentationError) - ) { - continue; - } - if err.location.start() <= range.start() && range.start() < err.location.end() { - return Err(raise_indentation_error(vm, err, source, line_index)); - } - } + // An indentation error replaces the token the lexer raised it for. + if kind == TokenKind::Unknown + && let Some(err) = errors.iter().find(|err| { + line_index.line_index(err.location.start()) + == line_index.line_index(range.start()) + && matches!( + err.error, + ParseErrorType::Lexical( + LexicalErrorType::IndentationError + | LexicalErrorType::TabError + | LexicalErrorType::TooDeepIndentation + ) + ) + }) + { + return Err(raise_indentation_error(vm, err, source, line_index)); } if kind == TokenKind::EndOfFile { @@ -480,7 +480,7 @@ mod _tokenize { exc } - /// Raise an IndentationError from a parse error. + /// Raise an IndentationError or TabError from a parse error. fn raise_indentation_error( vm: &VirtualMachine, err: &ParseError, @@ -491,10 +491,15 @@ mod _tokenize { let err_line_text = source.full_line_str(err.location.start()); let err_text = err_line_text.trim_end_matches('\n').trim_end_matches('\r'); let msg = format!("{}", err.error); - let exc = vm.new_exception_msg( - vm.ctx.exceptions.indentation_error.to_owned(), - msg.clone().into(), - ); + let exc_type = if matches!( + err.error, + ParseErrorType::Lexical(LexicalErrorType::TabError) + ) { + vm.ctx.exceptions.tab_error + } else { + vm.ctx.exceptions.indentation_error + }; + let exc = vm.new_exception_msg(exc_type.to_owned(), msg.clone().into()); let obj = exc.as_object(); let _ = obj.set_attr("lineno", vm.ctx.new_int(err_lc.line.get()), vm); let _ = obj.set_attr("offset", vm.ctx.new_int(err_text.len() as i64 + 1), vm); @@ -658,7 +663,7 @@ mod _tokenize { const fn token_kind_value(kind: TokenKind) -> u8 { match kind { TokenKind::EndOfFile => 0, - TokenKind::Name + TokenKind::Identifier | TokenKind::For | TokenKind::In | TokenKind::Pass diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index bba073d5b..241ff35ea 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -1815,6 +1815,12 @@ pub(crate) fn parse( ); rustpython_compiler::pre_parse_source_error(&source_file)?; let parsed = parser::parse_unchecked(barry_source.source(), options); + if interactive + && let Some(error) = + rustpython_compiler::single_mode_multiple_statements_error(&source_file, &parsed) + { + return Err(error); + } let type_comment_source = type_comments.then(|| TypeCommentSource::new(source, parsed.tokens())); if let Some(lines) = &type_comment_source diff --git a/crates/vm/src/stdlib/_ast/argument.rs b/crates/vm/src/stdlib/_ast/argument.rs index 0ea9725a1..687c557b4 100644 --- a/crates/vm/src/stdlib/_ast/argument.rs +++ b/crates/vm/src/stdlib/_ast/argument.rs @@ -7,7 +7,7 @@ pub(super) struct PositionalArguments { } enum PositionalArgumentsKind { - Args(Box<[ast::Expr]>), + Args(thin_vec::ThinVec), RuntimeValues(Vec>), } @@ -24,7 +24,7 @@ impl PositionalArguments { Ok(Self::from_values(TextRange::default(), values)) } - fn from_args(range: TextRange, args: Box<[ast::Expr]>) -> Self { + fn from_args(range: TextRange, args: thin_vec::ThinVec) -> Self { Self { range, kind: PositionalArgumentsKind::Args(args), @@ -42,14 +42,7 @@ impl PositionalArguments { if values.iter().any(Option::is_none) { Self::from_runtime_values(range, values) } else { - Self::from_args( - range, - values - .into_iter() - .flatten() - .collect::>() - .into_boxed_slice(), - ) + Self::from_args(range, values.into_iter().flatten().collect()) } } @@ -57,13 +50,14 @@ impl PositionalArguments { self.range } - fn into_args_and_runtime_values(self) -> (Box<[ast::Expr]>, Option>>) { + fn into_args_and_runtime_values( + self, + ) -> (thin_vec::ThinVec, Option>>) { match self.kind { PositionalArgumentsKind::Args(args) => (args, None), - PositionalArgumentsKind::RuntimeValues(values) => ( - lower_runtime_expr_list(values.clone()).into_boxed_slice(), - Some(values), - ), + PositionalArgumentsKind::RuntimeValues(values) => { + (lower_runtime_expr_list(values.clone()).into(), Some(values)) + } } } } @@ -71,7 +65,7 @@ impl PositionalArguments { impl Node for PositionalArguments { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self.kind { - PositionalArgumentsKind::Args(args) => BoxedSlice(args).ast_to_object(vm, source_file), + PositionalArgumentsKind::Args(args) => args.ast_to_object(vm, source_file), PositionalArgumentsKind::RuntimeValues(values) => values.ast_to_object(vm, source_file), } } @@ -81,8 +75,8 @@ impl Node for PositionalArguments { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - let args: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; - Ok(Self::from_args(TextRange::default(), args.0)) + let args: thin_vec::ThinVec<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self::from_args(TextRange::default(), args)) } } @@ -137,7 +131,7 @@ pub(super) fn merge_function_call_arguments( node_index: Default::default(), range, args, - keywords: key_args.keywords.into(), + keywords: key_args.keywords.into_vec().into(), runtime_args, runtime_bases: None, } @@ -173,7 +167,7 @@ pub(super) fn split_function_call_arguments( // debug_assert!(range.contains_range(keyword_arguments_range)); let keyword_arguments = KeywordArguments { range: keyword_arguments_range, - keywords: keywords.into(), + keywords: keywords.into_iter().collect::>().into_boxed_slice(), }; (positional_arguments, keyword_arguments) @@ -214,7 +208,7 @@ pub(super) fn split_class_def_args( // debug_assert!(range.contains_range(keyword_arguments_range)); let keyword_arguments = KeywordArguments { range: keyword_arguments_range, - keywords: keywords.into(), + keywords: keywords.into_iter().collect::>().into_boxed_slice(), }; (Some(positional_arguments), Some(keyword_arguments)) @@ -231,7 +225,7 @@ pub(super) fn merge_class_def_args( let (args, runtime_bases) = if let Some(positional_arguments) = positional_arguments { positional_arguments.into_args_and_runtime_values() } else { - (vec![].into_boxed_slice(), None) + (thin_vec::ThinVec::new(), None) }; let keywords = if let Some(keyword_arguments) = keyword_arguments { keyword_arguments.keywords @@ -243,7 +237,7 @@ pub(super) fn merge_class_def_args( node_index: Default::default(), range: Default::default(), // TODO args, - keywords: keywords.into(), + keywords: keywords.into_vec().into(), runtime_args: None, runtime_bases, })) diff --git a/crates/vm/src/stdlib/_ast/expression.rs b/crates/vm/src/stdlib/_ast/expression.rs index 6d5e21302..aae354e50 100644 --- a/crates/vm/src/stdlib/_ast/expression.rs +++ b/crates/vm/src/stdlib/_ast/expression.rs @@ -866,7 +866,8 @@ fn expr_dict_comp_from_object_with_range( "DictComp", )?), value: get_required_node_field(vm, source_file, object, "value", "DictComp")?, - generators: get_node_list_field(vm, source_file, object, "generators", "DictComp")?, + generators: get_node_list_field(vm, source_file, object, "generators", "DictComp")? + .into_boxed_slice(), range, }) } @@ -888,8 +889,12 @@ impl Node for ast::ExprDictComp { .unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) - .unwrap(); + dict.set_item( + "generators", + BoxedSlice(generators).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); node_add_location(&dict, range, vm, source_file); node.into() } @@ -1082,14 +1087,17 @@ fn expr_compare_from_object_with_range( object: &PyObject, range: TextRange, ) -> PyResult { + let left: ast::Expr = get_required_node_field(vm, source_file, object, "left", "Compare")?; let comparators: Vec> = get_node_list_field(vm, source_file, object, "comparators", "Compare")?; let (runtime_comparators, comparators) = runtime_expr_boxed_slice_from_values(comparators); + let mut operands = Vec::with_capacity(comparators.len() + 1); + operands.push(left); + operands.extend(comparators); Ok(ast::ExprCompare { node_index: Default::default(), - left: get_required_node_field(vm, source_file, object, "left", "Compare")?, ops: get_node_boxed_slice_field(vm, source_file, object, "ops", "Compare")?, - comparators, + operands: operands.into_boxed_slice(), range, runtime_comparators, }) @@ -1099,9 +1107,8 @@ impl Node for ast::ExprCompare { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, - left, ops, - comparators, + operands, range, runtime_comparators, } = self; @@ -1109,10 +1116,13 @@ impl Node for ast::ExprCompare { .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); + let mut operands = operands.into_vec().into_iter(); + let left = operands.next().expect("A comparison has a left operand"); dict.set_item("left", left.ast_to_object(vm, source_file), vm) .unwrap(); dict.set_item("ops", BoxedSlice(ops).ast_to_object(vm, source_file), vm) .unwrap(); + let comparators: Box<[ast::Expr]> = operands.collect(); let comparators = runtime_comparators.map_or_else( || BoxedSlice(comparators).ast_to_object(vm, source_file), |values| values.ast_to_object(vm, source_file), diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index 553b235c4..e3b1c9475 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -320,6 +320,35 @@ fn ruff_format_spec_to_joined_str( } } +fn dummy_string_literal() -> ast::StringLiteral { + ast::StringLiteral { + node_index: Default::default(), + range: Default::default(), + value: "".into(), + flags: ast::StringLiteralFlags::empty(), + } +} + +fn dummy_fstring() -> ast::FString { + ast::FString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::FStringFlags::empty(), + } +} + +fn take_fstring_part(part: ast::FStringPartMut<'_>) -> ast::FStringPart { + match part { + ast::FStringPartMut::Literal(literal) => { + ast::FStringPart::Literal(core::mem::replace(literal, dummy_string_literal())) + } + ast::FStringPartMut::FString(fstring) => { + ast::FStringPart::FString(core::mem::replace(fstring, dummy_fstring())) + } + } +} + fn ruff_fstring_element_to_ruff_fstring_part( element: ast::InterpolatedStringElement, ) -> ast::FStringPart { @@ -362,16 +391,9 @@ fn format_spec_expr_to_ruff_format_spec( runtime_joined_str: _, runtime_values: _, } = fstring; - let default_part = ast::FStringPart::FString(ast::FString { - node_index: Default::default(), - range: Default::default(), - elements: Default::default(), - flags: ast::FStringFlags::empty(), - }); let mut elements = Vec::new(); - for i in 0..value.as_slice().len() { - let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); - match part { + for part in value { + match take_fstring_part(part) { ast::FStringPart::Literal(ast::StringLiteral { range, value, @@ -718,16 +740,9 @@ pub(super) fn fstring_to_object( .ast_to_object(vm, source_file); } - let default_part = ast::FStringPart::FString(ast::FString { - node_index: Default::default(), - range: Default::default(), - elements: Default::default(), - flags: ast::FStringFlags::empty(), - }); let mut values = Vec::new(); - for i in 0..value.as_slice().len() { - let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); - match part { + for part in &mut value { + match take_fstring_part(part) { ast::FStringPart::Literal(literal) => { values.push(JoinedStrPart::Constant(Constant::new_str( rustpython_codegen::string_literal_part_value(source_file, &literal), diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs index cbdc15b53..2bd51611e 100644 --- a/crates/vm/src/stdlib/_ast/validate.rs +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -616,10 +616,11 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - validate_expr(vm, &await_expr.value, ast::ExprContext::Load) } ast::Expr::Compare(compare) => { - if compare.comparators.is_empty() { + let comparators = compare.operands.get(1..).unwrap_or(&[]); + if comparators.is_empty() { return Err(vm.new_value_error("Compare with no comparators")); } - if compare.comparators.len() != compare.ops.len() { + if comparators.len() != compare.ops.len() { return Err(vm.new_value_error( "Compare has a different number of comparators and operands", )); @@ -629,8 +630,8 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - compare.runtime_comparators.as_ref(), ast::ExprContext::Load, )?; - validate_exprs(vm, &compare.comparators, ast::ExprContext::Load, false)?; - validate_expr(vm, &compare.left, ast::ExprContext::Load) + validate_exprs(vm, comparators, ast::ExprContext::Load, false)?; + validate_expr(vm, compare.first_operand(), ast::ExprContext::Load) } ast::Expr::Call(call) => { validate_expr(vm, &call.func, ast::ExprContext::Load)?; diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 626659f19..b5f7c77c7 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -1506,9 +1506,9 @@ mod escape_warnings { } // F-string literals — tokenizer + string_parser paths ast::Expr::FString(fstring_expr) => { - for part in fstring_expr.value.as_slice() { + for part in &fstring_expr.value { match part { - ast::FStringPart::Literal(string_lit) => { + ast::FStringPartRef::Literal(string_lit) => { // Plain string part in f-string concatenation if !matches!( string_lit.flags.prefix(), @@ -1517,7 +1517,7 @@ mod escape_warnings { self.check_quoted_literal(string_lit.range, false); } } - ast::FStringPart::FString(fstring) => { + ast::FStringPartRef::FString(fstring) => { if matches!( fstring.flags.prefix(), ast::str_prefix::FStringPrefix::Raw { .. } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index c8f8d72ef..1a1696b71 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -1,7 +1,5 @@ #[cfg(feature = "parser")] -use ruff_python_ast::token::TokenKind; - -use ruff_python_parser::{InterpolatedStringErrorType, LexicalErrorType, ParseErrorType}; +use ruff_python_parser::{LexicalErrorType, ParseErrorType}; use rustpython_common::wtf8::Wtf8Buf; use rustpython_compiler_core::SourceLocation; @@ -9,7 +7,7 @@ use rustpython_compiler_core::SourceLocation; use core::ops::RangeInclusive; #[cfg(feature = "parser")] -use rustpython_compiler::{CompileError, ParseError, is_blank_python_source}; +use rustpython_compiler::is_blank_python_source; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -49,262 +47,6 @@ macro_rules! define_exception_fn { }; } -#[derive(Clone, Debug)] -struct SyntaxErrorInfo { - msg: String, - narrow_caret: bool, -} - -impl SyntaxErrorInfo { - #[must_use] - const fn new(msg: String, narrow_caret: bool) -> Self { - Self { msg, narrow_caret } - } - - #[cfg(feature = "parser")] - #[must_use] - const fn handle_expected_token(expected: TokenKind, found: TokenKind) -> &'static str { - match (expected, found) { - (TokenKind::Colon, TokenKind::Newline) => "expected ':'", - - (TokenKind::Lpar, _) => "expected '('", - - (TokenKind::Else, y) if !matches!(y, TokenKind::Colon) => { - "expected 'else' after 'if' expression" - } - - _ => "invalid syntax", - } - } - - #[cfg(feature = "parser")] - fn analyze_compile_error(&mut self, compile_error: &CompileError) { - let CompileError::Parse(ParseError { - error, location, .. - }) = compile_error - else { - return; - }; - - let msg = match error { - ParseErrorType::FStringError(InterpolatedStringErrorType::UnterminatedString) - | ParseErrorType::Lexical(LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedString, - )) => "unterminated f-string literal".into(), - - ParseErrorType::TStringError(InterpolatedStringErrorType::UnterminatedString) - | ParseErrorType::Lexical(LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedString, - )) => "unterminated t-string literal".into(), - - ParseErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - ) - | ParseErrorType::Lexical(LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - )) => "unterminated triple-quoted f-string literal".into(), - - ParseErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - ) - | ParseErrorType::Lexical(LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - )) => "unterminated triple-quoted t-string literal".into(), - - // ruff already prefixes these with `f-string: ` / `t-string: `, matching CPython. - // It quotes braces with backticks where CPython uses single quotes. - ParseErrorType::FStringError(_) - | ParseErrorType::TStringError(_) - | ParseErrorType::Lexical( - LexicalErrorType::FStringError(_) | LexicalErrorType::TStringError(_), - ) => self.msg.replace('`', "'"), - - ParseErrorType::UnexpectedExpressionToken => "invalid syntax".into(), - - ParseErrorType::ExpectedToken { expected, found } => { - Self::handle_expected_token(*expected, *found).into() - } - - ParseErrorType::InvalidStarredExpressionUsage => { - self.narrow_caret = true; - "invalid syntax".into() - } - - ParseErrorType::InvalidDeleteTarget => "invalid syntax".into(), - - ParseErrorType::Lexical(LexicalErrorType::LineContinuationError) => { - "unexpected character after line continuation character".into() - } - - ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError) => { - format!( - "unterminated string literal (detected at line {})", - location.line - ) - } - - ParseErrorType::EmptyTypeParams => "Type parameter list cannot be empty".into(), - - ParseErrorType::InvalidStarPatternUsage => { - self.narrow_caret = true; - "cannot use starred expression here".into() - } - - ParseErrorType::ExpectedKeywordParam => "named arguments must follow bare *".into(), - - ParseErrorType::EmptyImportNames => "Expected one or more names after 'import'".into(), - - ParseErrorType::UnparenthesizedGeneratorExpression => { - "Generator expression must be parenthesized".into() - } - - ParseErrorType::NonDefaultParamAfterDefaultParam => { - "parameter without a default follows parameter with a default".into() - } - - ParseErrorType::VarParameterWithDefault => { - "var-positional argument cannot have default value".into() - } - - ParseErrorType::PositionalAfterKeywordArgument => { - "positional argument follows keyword argument".into() - } - - ParseErrorType::PositionalAfterKeywordUnpacking => { - "positional argument follows keyword argument unpacking".into() - } - - ParseErrorType::InvalidArgumentUnpackingOrder => { - "iterable argument unpacking follows keyword argument unpacking".into() - } - - ParseErrorType::ParamAfterVarKeywordParam => { - "arguments cannot follow var-keyword argument".into() - } - - ParseErrorType::InvalidAnnotatedAssignmentTarget => { - "illegal target for annotation".into() - } - - ParseErrorType::Lexical(LexicalErrorType::UnrecognizedToken { .. }) - | ParseErrorType::SimpleStatementsOnSameLine - | ParseErrorType::SimpleAndCompoundStatementOnSameLine - | ParseErrorType::ExpectedExpression => "invalid syntax".into(), - - ParseErrorType::OtherError(s) if s.starts_with("Expected an identifier") => { - "invalid syntax".into() - } - - // What the parser says when it cannot continue the list it is - // recovering; each of these situations is a plain "invalid syntax". - ParseErrorType::OtherError(s) - if matches!( - s.as_str(), - "Expected a statement" - | "Expected an `elif` or `else` clause, or the end of the `if` statement." - | "Expected an `except` or `finally` clause or the end of the `try` statement." - | "The keyword is not allowed as a variable declaration name" - | "Expected an assignment target" - | "Expected a type parameter or the end of the type parameter list" - | "Expected an import name or a ')'" - | "Expected an import name" - | "Expected an expression or the end of the slice list" - | "Expected an expression or a ']'" - | "Expected an expression or a '}'" - | "Expected an expression or a ')'" - | "Expected an expression" - | "Expected a pattern or the end of the sequence pattern" - | "Expected a mapping pattern or the end of the mapping pattern" - | "Expected a pattern or a ')'" - | "Expected a delete target" - | "Expected a parameter or the end of the parameter list" - | "Expected an expression or the end of the with item list" - ) => - { - "invalid syntax".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "bytes literal cannot be mixed with non-bytes literals", - ) => - { - "cannot mix bytes and nonbytes literals".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("positional patterns cannot follow keyword patterns") => - { - "positional patterns follow keyword patterns".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("boolean 'not' expression cannot be used here") => - { - "'not' after an operator must be parenthesized".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("trailing comma not allowed") => - { - "trailing comma not allowed without surrounding parentheses".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "multiple exception types must be parenthesized when using `as`", - ) => - { - "multiple exception types must be parenthesized when using 'as'".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "position-only parameter separator not allowed as first parameter", - ) => - { - "at least one argument must precede /".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("only one '/' separator allowed") => - { - "/ may appear only once".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("'/' parameter must appear before '*' parameter") => - { - "/ must be ahead of *".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("expected `except` or `finally` after `try` block") => - { - "expected 'except' or 'finally' block".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("only one '*' parameter allowed") => - { - "* argument may appear only once".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - r#"cannot have both 'except' and 'except*' on the same 'try'"#, - ) => - { - r#"cannot have both 'except' and 'except*' on the same 'try'"#.into() - } - - _ => return, - }; - - self.msg = msg; - } -} - /// Collection of object creation helpers impl VirtualMachine { /// Create a new python object @@ -703,38 +445,6 @@ impl VirtualMachine { .expect("UnicodeEncodeError constructor") } - #[cfg(feature = "parser")] - fn source_has_mixed_tabs_and_spaces(source: Option<&str>, error_line: usize) -> bool { - source.is_some_and(|source| { - let mut has_space_indent = false; - let mut has_tab_indent = false; - for (i, line) in source.lines().enumerate() { - if i + 1 > error_line { - break; - } - let rest = line.trim_start_matches([' ', '\t']); - // Blank and comment-only lines do not participate in indent. - if rest.is_empty() || rest.starts_with('#') { - continue; - } - let indent = &line.as_bytes()[..line.len() - rest.len()]; - if indent.is_empty() { - continue; - } - if indent.contains(&b' ') && indent.contains(&b'\t') { - return true; - } - if indent.contains(&b' ') { - has_space_indent = true; - } - if indent.contains(&b'\t') { - has_tab_indent = true; - } - } - has_space_indent && has_tab_indent - }) - } - // TODO: don't take ownership should make the success path faster pub fn new_key_error(&self, obj: PyObjectRef) -> PyBaseExceptionRef { let key_error = self.ctx.exceptions.key_error.to_owned(); @@ -769,25 +479,27 @@ impl VirtualMachine { let syntax_error_type = match &error { #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { - error: - ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::IndentationError, - ) - | ruff_python_parser::ParseErrorType::UnexpectedIndentation, - location, + error: ParseErrorType::Lexical(LexicalErrorType::TabError), .. - }) => { - if Self::source_has_mixed_tabs_and_spaces(source, location.line.get()) { - self.ctx.exceptions.tab_error - } else { - self.ctx.exceptions.indentation_error - } - } + }) => self.ctx.exceptions.tab_error, + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ParseErrorType::Lexical( + LexicalErrorType::IndentationError | LexicalErrorType::TooDeepIndentation, + ) + | ParseErrorType::UnexpectedIndentation, + .. + }) => self.ctx.exceptions.indentation_error, #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::Eof, + ruff_python_parser::LexicalErrorType::Eof + | ruff_python_parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }, ), .. }) => incomplete_or_syntax(allow_incomplete), @@ -807,26 +519,31 @@ impl VirtualMachine { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::FStringError( - ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ) | ruff_python_parser::LexicalErrorType::TStringError( - ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ), ), .. }) => incomplete_or_syntax(allow_incomplete), #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { - error: - ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::UnclosedStringError, - ), + error: ruff_python_parser::ParseErrorType::ExpectedIndentedBlock { .. }, + raw_location, .. }) => { - if allow_incomplete { - incomplete_or_syntax(source.is_some_and(unclosed_string_is_incomplete)) + // The block can still follow when the error is found at whitespace, that is, at + // the end of the input. + let at_end = source.is_some_and(|source| { + source + .get(raw_location.start().to_usize()..raw_location.end().to_usize()) + .is_some_and(|text| text.chars().all(|c| c.is_ascii_whitespace())) + }); + if allow_incomplete && at_end { + self.ctx.exceptions.incomplete_input_error } else { - self.ctx.exceptions.syntax_error + self.ctx.exceptions.indentation_error } } #[cfg(feature = "parser")] @@ -835,39 +552,7 @@ impl VirtualMachine { raw_location, .. }) => { - if s.starts_with("Expected an indented block after") - || s.starts_with("expected an indented block after") - { - if allow_incomplete { - // Check that all chars in the error are whitespace, if so, the source is - // incomplete. Otherwise, we've found code that might violates - // indentation rules. - let mut is_incomplete = true; - if let Some(source) = source { - let start = raw_location.start().to_usize(); - let end = raw_location.end().to_usize(); - let mut iter = source.chars(); - iter.nth(start); - for _ in start..end { - if let Some(c) = iter.next() { - if !c.is_ascii_whitespace() { - is_incomplete = false; - } - } else { - break; - } - } - } - - if is_incomplete { - self.ctx.exceptions.incomplete_input_error - } else { - self.ctx.exceptions.indentation_error // not syntax_error - } - } else { - self.ctx.exceptions.indentation_error - } - } else if allow_incomplete + if allow_incomplete && (s == "incomplete input" || (s == "unexpected EOF while parsing" && source.is_some_and(|source| { @@ -908,52 +593,12 @@ impl VirtualMachine { source.and_then(|src| get_statement(src, error.location())) }; - let mut msg = error.to_string(); - if !msg.starts_with("Exceeds the limit ") - && !msg.starts_with("Did you mean ") - && !msg.starts_with("Invalid star expression") - && !msg.starts_with("Function parameters cannot be parenthesized") - && !msg.starts_with("Lambda expression parameters cannot be parenthesized") - && !msg.starts_with("Cannot have two type comments on def") - && !msg.starts_with("Variable annotation syntax is") - && !msg.starts_with("The '@' operator is") - && !msg.starts_with("Async functions are") - && !msg.starts_with("Async comprehensions are") - && !msg.starts_with("Async for loops are") - && !msg.starts_with("Async with statements are") - && !msg.starts_with("Exception groups are") - && !msg.starts_with("Positional-only parameters are") - && !msg.starts_with("Pattern matching is") - && !msg.starts_with("Type statement is") - && !msg.starts_with("Type parameter lists are") - && !msg.starts_with("Type parameter defaults are") - && !msg.starts_with("Assignment expressions are") - && !msg.starts_with("Await expressions are") - && !msg.starts_with("Underscores in numeric literals are") - && !msg.starts_with("Missing parentheses") - && let Some(msg) = msg.get_mut(..1) - { - msg.make_ascii_lowercase(); - } - - cfg_select! { - feature = "parser" => { - let mut syntax_error_info = SyntaxErrorInfo::new(msg, false); - syntax_error_info.analyze_compile_error(error); - } - _ => { - let syntax_error_info = SyntaxErrorInfo::new(msg, false); - } + let msg = if syntax_error_type.is(self.ctx.exceptions.incomplete_input_error) { + String::from("incomplete input") + } else { + error.to_string() }; - if syntax_error_type.is(self.ctx.exceptions.tab_error) { - syntax_error_info.msg = - String::from("inconsistent use of tabs and spaces in indentation"); - } else if syntax_error_type.is(self.ctx.exceptions.incomplete_input_error) { - syntax_error_info.msg = String::from("incomplete input"); - } - - let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info; let unterminated_triple_quoted_string = msg.starts_with("unterminated triple-quoted"); let unexpected_eof_error = msg == "unexpected EOF while parsing"; if unterminated_triple_quoted_string @@ -970,6 +615,50 @@ impl VirtualMachine { || msg.starts_with("except expressions without parentheses are") || msg.starts_with("Pattern matching is"); let line_end_binary_operator_error = msg.starts_with("The '@' operator is"); + // Tokenizer errors other than unexpected EOF end where the tokenizer stopped, which is + // reported as end offset 0 or -1, or where they start. + let tokenizer_end_offset = cfg_select! { + feature = "parser" => { + match error { + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical( + LexicalErrorType::TabError + | LexicalErrorType::TooDeepIndentation + | LexicalErrorType::LineContinuationError + | LexicalErrorType::UnclosedBracket { .. }, + ), + .. + }) => Some(0), + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::IndentationError) + | ParseErrorType::UnexpectedIndentation, + .. + }) => Some(-1), + // The span runs from the last character of the key to the end of the line. + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::ExpectedColonAfterDictionaryKey, + .. + }) => Some(0), + // Other tokenizer errors end where they start, except those reported + // over a range. + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical(lexical), + .. + }) if lexical.is_tokenizer_error() + && !matches!( + lexical, + LexicalErrorType::Eof + | LexicalErrorType::LeadingZerosInDecimalInteger + | LexicalErrorType::IncompatibleStringPrefixes { .. } + ) => + { + Some(error.python_location().1 as isize) + } + _ => None, + } + } + _ => None, + }; let unclosed_bracket_error = cfg_select! { feature = "parser" => { matches!( @@ -1011,15 +700,14 @@ impl VirtualMachine { .is_some_and(|ch| ch.is_ascii_whitespace())); let (end_lineno, end_offset) = if no_end_offset { (end_lineno, -1) + } else if let Some(end_offset) = tokenizer_end_offset { + (end_lineno, end_offset) } else if unclosed_bracket_error { // The bracket that was never closed is marked where it opened, // and the span stops there. (end_lineno, 0) } else if line_end_binary_operator_error && end_offset == offset_raw { (end_lineno, (end_offset + 1) as isize) - } else if narrow_caret { - let (l, o) = error.python_location(); - (l, (o + 1) as isize) } else { (end_lineno, end_offset as isize) }; @@ -1207,119 +895,3 @@ impl VirtualMachine { .upcast() } } - -#[cfg(feature = "parser")] -enum QuotedStringScan { - Closed(usize), - Unclosed { - triple: bool, - unescaped_newline: bool, - }, -} - -/// An unclosed string is incomplete when more input could still finish it. -/// A non-triple string that already hit an unescaped newline is a hard error. -#[cfg(feature = "parser")] -fn unclosed_string_is_incomplete(source: &str) -> bool { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => match scan_quoted_string_for_incomplete(bytes, index) { - QuotedStringScan::Closed(end) => index = end, - QuotedStringScan::Unclosed { - triple, - unescaped_newline, - } => { - // Single-quoted f/t-strings never set E_EOLS. - if !triple && interpolated_string_prefix_at(bytes, index) { - return false; - } - return triple || !unescaped_newline; - } - }, - _ => index += 1, - } - } - false -} - -#[cfg(feature = "parser")] -fn interpolated_string_prefix_at(bytes: &[u8], quote: usize) -> bool { - let Some(&prev) = quote.checked_sub(1).and_then(|index| bytes.get(index)) else { - return false; - }; - let lower = prev.to_ascii_lowercase(); - let marker_index = if matches!(lower, b'f' | b't') { - if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') { - quote - 2 - } else { - quote - 1 - } - } else if lower == b'r' - && quote >= 2 - && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't') - { - quote - 2 - } else { - return false; - }; - marker_index == 0 || !identifier_continue_before(bytes, marker_index) -} - -#[cfg(feature = "parser")] -fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { - if index == 0 { - return false; - } - if bytes[index - 1].is_ascii() { - return bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphanumeric(); - } - let mut start = index - 1; - while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 { - start -= 1; - } - ::core::str::from_utf8(&bytes[start..index]) - .ok() - .and_then(|text| text.chars().next_back()) - .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) -} - -#[cfg(feature = "parser")] -fn scan_quoted_string_for_incomplete(bytes: &[u8], quote_index: usize) -> QuotedStringScan { - let quote = bytes[quote_index]; - let triple = - bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); - let mut index = quote_index + if triple { 3 } else { 1 }; - while index < bytes.len() { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - continue; - } - if triple { - if bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - return QuotedStringScan::Closed(index + 3); - } - } else if bytes[index] == quote { - return QuotedStringScan::Closed(index + 1); - } else if bytes[index] == b'\n' { - return QuotedStringScan::Unclosed { - triple: false, - unescaped_newline: true, - }; - } - index += 1; - } - QuotedStringScan::Unclosed { - triple, - unescaped_newline: false, - } -} diff --git a/extra_tests/snippets/syntax_invalid.py b/extra_tests/snippets/syntax_invalid.py index 37a3805a2..00acf6cfb 100644 --- a/extra_tests/snippets/syntax_invalid.py +++ b/extra_tests/snippets/syntax_invalid.py @@ -77,3 +77,46 @@ src = """ from __future__ import print_function """ compile(src, 'test.py', 'exec') + + +# CPython reports the unparenthesized `except ... as` error with a range that starts at the +# first exception type and stops just before the `:` closing the clause. `end_offset` is an +# exclusive 1-based *character* column, so that end lands on the column the `:` sits on. +# +# The cases below pin the parts that are easy to get wrong: the column counts characters +# rather than UTF-8 bytes, and an explicit line join may sit anywhere between the exception +# types and the `:`. Values verified against CPython 3.14. +except_as_ranges = [ + # (clause, (lineno, offset), (end_lineno, end_offset)) + ("except A, B as e:", (3, 8), (3, 17)), + ("except A, B as e :", (3, 8), (3, 18)), + ("except A, B as e :", (3, 8), (3, 20)), + ("except A, B as e\t:", (3, 8), (3, 18)), + ("except A, B, C as blech:", (3, 8), (3, 24)), + ("except* A, B, C as e:", (3, 9), (3, 21)), + # A non-ASCII name must not push the column right by its extra UTF-8 bytes. + ("except Ä, B as e:", (3, 8), (3, 17)), + ("except 사과, B as e:", (3, 8), (3, 18)), + ("except A, B as 오:", (3, 8), (3, 17)), + # An explicit line join may precede or follow `as`, or close the clause on its own line. + ("except A, B as exc\\\n :", (3, 8), (4, 5)), + ("except A, B \\\nas exc:", (3, 8), (4, 7)), + ("except A, \\\nB as exc:", (3, 8), (4, 9)), + ("except A, B as \\\nexc:", (3, 8), (4, 4)), +] + +for clause, start, end in except_as_ranges: + src = "try:\n pass\n%s\n pass\n" % clause + with assert_raises(SyntaxError) as ae: + compile(src, "test.py", "exec") + exc = ae.exception + assert exc.msg == "multiple exception types must be parenthesized when using 'as'", ( + clause, + exc.msg, + ) + assert (exc.lineno, exc.offset) == start, (clause, (exc.lineno, exc.offset), start) + assert (exc.end_lineno, exc.end_offset) == end, ( + clause, + (exc.end_lineno, exc.end_offset), + end, + ) diff --git a/src/shell.rs b/src/shell.rs index d727003e6..a899370ac 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -60,29 +60,18 @@ fn shell_exec( error: ParseErrorType::Lexical( LexicalErrorType::Eof | LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ) + | LexicalErrorType::UnclosedStringError { + triple_quoted: true, + .. + } ), .. })) ) { return ShellExecResult::ContinueLine; } - if let VmCompileError::Compile(CompileError::Parse(ParseError { - error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError), - raw_location, - .. - })) = &err - { - let loc = raw_location.start().to_usize(); - let mut iter = source.chars(); - if let Some(quote) = iter.nth(loc) - && iter.next() == Some(quote) - && iter.next() == Some(quote) - { - return ShellExecResult::ContinueLine; - } - } // An unfinished suite is _IncompleteInputError, not IndentationError. let exc = err.into_pyexception_maybe_incomplete(vm, Some(source), !empty_line_given);