Use the Ruff fork 0.17.1 for CPython syntax error messages and the obsolete '<>' hint (#9212)

* Use the RustPython Ruff fork 0.17.1 from git and place EOF errors by kind

- The ruff crates come from the RustPython/ruff `rustpython-0.17.1`
  branch at 93967836c8 instead of the 0.16.10 crates.io release.
- `EofToken.known_location` is replaced by `EofErrorLocation`
  (Tokenizer, Token, StartingFromToken), chosen from the parse error kind.
  The keyword scan in `clause_keyword` is removed, since the parser now
  reports a forced `:` as `ExpectedForcedToken`.
- An empty import at EOF ends at the end of the line read, and an empty
  import before a newline is an empty range at the newline.

Assisted-by: Claude Code:claude-opus-5-5

* Report the obsolete `<>` from the parser outside Barry mode

- Bump the RustPython/ruff fork to 9290a88495, where the parser reports an
  adjacent `<` and `>` in a comparison as "invalid syntax.  Maybe you meant
  '!=' instead of '<>'?" over both tokens.
- Outside Barry mode, `BarrySource` no longer searches the source text for
  `<>`, so the parser's error is reported unchanged. A `<>` that starts a
  statement is reported at the `<` only, and `<<>` at the `>`.

Assisted-by: Claude Code:claude-opus-5-5

Cargo.lock

@@ -1639,15 +1639,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "getopts"
-version = "0.2.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.17"
@@ -3528,6 +3519,24 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "ruff_unicode_names2"
+version = "0.0.18"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
+dependencies = [
+ "phf 0.11.3",
+ "ruff_unicode_names2_generator",
+]
+
+[[package]]
+name = "ruff_unicode_names2_generator"
+version = "0.0.18"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
+dependencies = [
+ "phf_codegen",
+ "rand 0.10.3",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
@@ -3916,9 +3925,8 @@ dependencies = [
 
 [[package]]
 name = "rustpython-ruff_python_ast"
-version = "0.16.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b7ffeb85f9a68c461d538839c79548584768df08f2d68cb1ee693be697e0355"
+version = "0.17.1"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
 dependencies = [
  "aho-corasick",
  "arrayvec",
@@ -3938,9 +3946,8 @@ dependencies = [
 
 [[package]]
 name = "rustpython-ruff_python_parser"
-version = "0.16.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c9729e35cac02839c7a67aaccfea3eba32032f7aeacca2e82ad79e8d7b7fd95"
+version = "0.17.1"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
 dependencies = [
  "bitflags 2.13.2",
  "bstr",
@@ -3949,6 +3956,7 @@ dependencies = [
  "hashbrown 0.17.1",
  "icu_properties",
  "memchr",
+ "ruff_unicode_names2",
  "rustc-hash",
  "rustpython-ruff_python_ast",
  "rustpython-ruff_python_trivia",
@@ -3958,14 +3966,12 @@ dependencies = [
  "thin-vec",
  "unicode-ident",
  "unicode-normalization",
- "unicode_names2 1.3.0",
 ]
 
 [[package]]
 name = "rustpython-ruff_python_trivia"
-version = "0.16.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "764f43ea060f888164fbd360b0e1a36f2a35f5c03def85407bab7039fc62a645"
+version = "0.17.1"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
 dependencies = [
  "itertools 0.15.0",
  "memchr",
@@ -3978,9 +3984,8 @@ dependencies = [
 
 [[package]]
 name = "rustpython-ruff_source_file"
-version = "0.16.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfe26430c3f2eadb40524b6df7ac9a3970dd2b58f921b01ed75df0121e332fad"
+version = "0.17.1"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
 dependencies = [
  "memchr",
  "rustpython-ruff_text_size",
@@ -3988,9 +3993,8 @@ dependencies = [
 
 [[package]]
 name = "rustpython-ruff_text_size"
-version = "0.16.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5377a04f2855030600b93a05754242950ea9b070507ef9aef67965a52eeb09"
+version = "0.17.1"
+source = "git+https://github.com/RustPython/ruff.git?rev=9290a8849542134d83f77e9360e7f0b93897fd93#9290a8849542134d83f77e9360e7f0b93897fd93"
 dependencies = [
  "get-size2",
 ]
@@ -4070,7 +4074,7 @@ dependencies = [
  "icu_normalizer",
  "icu_properties",
  "rustpython-wtf8",
- "unicode_names2 4.0.0",
+ "unicode_names2",
  "writeable",
  "zlib-rs",
 ]
@@ -4875,16 +4879,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "unicode_names2"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd"
-dependencies = [
- "phf 0.11.3",
- "unicode_names2_generator 1.3.0",
-]
-
 [[package]]
 name = "unicode_names2"
 version = "4.0.0"
@@ -4892,19 +4886,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ba646312acace3c9ec35d7404b58214b6fbf17cbb0d040f35a88407c424447f"
 dependencies = [
  "phf 0.11.3",
- "unicode_names2_generator 4.0.0",
-]
-
-[[package]]
-name = "unicode_names2_generator"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e"
-dependencies = [
- "getopts",
- "log",
- "phf_codegen",
- "rand 0.8.6",
+ "unicode_names2_generator",
 ]
 
 [[package]]

Cargo.toml

@@ -202,10 +202,10 @@ rustpython-wtf8 = { path = "crates/wtf8", version = "0.6.1" }
 rustpython-doc = { path = "crates/doc", version = "0.6.1" }
 
 # Use the RustPython Ruff fork for RustPython public `_ast` metadata.
-ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.10" }
-ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.10" }
-ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.10" }
-ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.10" }
+ruff_python_parser = { package = "rustpython-ruff_python_parser", git = "https://github.com/RustPython/ruff.git", rev = "9290a8849542134d83f77e9360e7f0b93897fd93" }
+ruff_python_ast = { package = "rustpython-ruff_python_ast", git = "https://github.com/RustPython/ruff.git", rev = "9290a8849542134d83f77e9360e7f0b93897fd93" }
+ruff_text_size = { package = "rustpython-ruff_text_size", git = "https://github.com/RustPython/ruff.git", rev = "9290a8849542134d83f77e9360e7f0b93897fd93" }
+ruff_source_file = { package = "rustpython-ruff_source_file", git = "https://github.com/RustPython/ruff.git", rev = "9290a8849542134d83f77e9360e7f0b93897fd93" }
 
 der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] }
 phf = { version = "0.14.0", default-features = false, features = ["macros"]}

crates/compiler/src/lib.rs

@@ -50,13 +50,26 @@ pub struct EofToken {
     pub lineno: usize,
     /// The last line read, which the tokenizer stopped at the end of.
     pub line: String,
-    /// The error is raised at the token itself, as the generic syntax error
-    /// and a missing forced token are, so its columns are 0.
-    pub known_location: bool,
+    /// Where the error on the token is raised from.
+    pub raised_at: EofErrorLocation,
+}
+
+/// Where an error on a token made up at the end of the input is placed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EofErrorLocation {
+    /// Where the tokenizer stopped, as `_PyPegen_raise_error` places an error
+    /// on a token without a column.
+    Tokenizer,
+    /// At the token itself, as the generic syntax error and a missing forced
+    /// token are, so its columns are 0.
+    Token,
+    /// From the token to where the tokenizer stopped, as
+    /// `RAISE_SYNTAX_ERROR_STARTING_FROM` places it.
+    StartingFromToken,
 }
 
 impl EofToken {
-    fn new(source: &str, mode: Mode, known_location: bool) -> Self {
+    fn new(source: &str, mode: Mode, raised_at: EofErrorLocation) -> Self {
         let lineno = source.split_inclusive('\n').count();
         let mut line = source
             .split_inclusive('\n')
@@ -72,24 +85,27 @@ impl EofToken {
         Self {
             lineno,
             line,
-            known_location,
+            raised_at,
         }
     }
 
     /// `SyntaxError.offset`
     #[must_use]
     pub fn offset(&self) -> usize {
-        if self.known_location {
-            0
-        } else {
-            self.line.chars().count()
+        match self.raised_at {
+            EofErrorLocation::Tokenizer => self.line.chars().count(),
+            EofErrorLocation::Token | EofErrorLocation::StartingFromToken => 0,
         }
     }
 
     /// `SyntaxError.end_offset`
     #[must_use]
-    pub const fn end_offset(&self) -> isize {
-        if self.known_location { 0 } else { -1 }
+    pub fn end_offset(&self) -> isize {
+        match self.raised_at {
+            EofErrorLocation::Tokenizer => -1,
+            EofErrorLocation::Token => 0,
+            EofErrorLocation::StartingFromToken => self.line.chars().count() as isize,
+        }
     }
 }
 
@@ -137,8 +153,11 @@ impl CompileError {
 
         let eof_token = at_eof_token.then(|| {
             let source = source_file.source_text();
-            let known_location = is_raised_at_token(&diagnostic.error, source, raw_location);
-            Box::new(EofToken::new(source, mode, known_location))
+            Box::new(EofToken::new(
+                source,
+                mode,
+                eof_error_location(&diagnostic.error),
+            ))
         });
         Self::Parse(ParseError {
             error: diagnostic.error,
@@ -419,57 +438,34 @@ fn fails_at_eof_token(error: &parser::ParseError, source: &str, mode: Mode) -> b
     }
 }
 
-/// Whether the error is raised at the token the parser failed on, as the
-/// generic syntax error and a missing forced token are, rather than where the
-/// tokenizer stopped.
-fn is_raised_at_token(
-    error: &parser::ParseErrorType,
-    source: &str,
-    range: ruff_text_size::TextRange,
-) -> bool {
+/// Where the error is placed when the parser fails on a token made up at the
+/// end of the input.
+fn eof_error_location(error: &parser::ParseErrorType) -> EofErrorLocation {
     match error {
-        // Only `else`, `try`, `finally` and `def` force their `:`; the other
-        // clause headers report a missing one from an `invalid_*` rule.
+        // A `:` that is not forced is reported missing before the newline by an
+        // `invalid_*` rule.
         parser::ParseErrorType::ExpectedToken {
             expected: TokenKind::Colon,
             found: TokenKind::Newline,
-        } => matches!(
-            clause_keyword(source, range.start()),
-            Some(TokenKind::Else | TokenKind::Try | TokenKind::Finally | TokenKind::Def)
-        ),
+        } => EofErrorLocation::Tokenizer,
         parser::ParseErrorType::ExpectedToken { .. }
+        | parser::ParseErrorType::ExpectedForcedToken { .. }
         | parser::ParseErrorType::ExpectedExpression
         | parser::ParseErrorType::ExpectedIdentifier
         | parser::ParseErrorType::UnexpectedExpressionToken
+        | parser::ParseErrorType::UnexpectedTokenAfterAsync(_)
+        | parser::ParseErrorType::EmptyDeleteTargets
+        | parser::ParseErrorType::EmptyGlobalNames
+        | parser::ParseErrorType::EmptyNonlocalNames
         | parser::ParseErrorType::SimpleStatementsOnSameLine
         | parser::ParseErrorType::SimpleAndCompoundStatementOnSameLine
-        | parser::ParseErrorType::InvalidStarredExpressionUsage => true,
-        parser::ParseErrorType::OtherError(message) => message == "invalid syntax",
-        _ => false,
-    }
-}
-
-/// The keyword that starts the logical line `offset` is in, after `async`.
-fn clause_keyword(source: &str, offset: TextSize) -> Option<TokenKind> {
-    let parsed = parser::parse_unchecked(source, parser::ParseOptions::from(parser::Mode::Module));
-    let mut keyword = None;
-    let mut at_line_start = true;
-    for token in parsed
-        .tokens()
-        .iter()
-        .take_while(|token| token.start() < offset)
-    {
-        match token.kind() {
-            TokenKind::Newline | TokenKind::Indent | TokenKind::Dedent => at_line_start = true,
-            TokenKind::NonLogicalNewline | TokenKind::Comment | TokenKind::Async => {}
-            kind if at_line_start => {
-                keyword = Some(kind);
-                at_line_start = false;
-            }
-            _ => {}
+        | parser::ParseErrorType::InvalidStarredExpressionUsage => EofErrorLocation::Token,
+        parser::ParseErrorType::EmptyImportNames => EofErrorLocation::StartingFromToken,
+        parser::ParseErrorType::OtherError(message) if message == "invalid syntax" => {
+            EofErrorLocation::Token
         }
+        _ => EofErrorLocation::Tokenizer,
     }
-    keyword
 }
 
 fn eof_parse_diagnostic(
@@ -544,7 +540,12 @@ fn default_parse_diagnostic(
     error: parser::ParseError,
     source_file: &SourceFile,
 ) -> NormalizedParseDiagnostic {
-    let (loc, end_loc) = adjusted_error_locations(source_file, error.location);
+    let (loc, end_loc) = if matches!(error.error, parser::ParseErrorType::EmptyImportNames) {
+        // RAISE_SYNTAX_ERROR_STARTING_FROM the newline ends where it starts.
+        source_locations(source_file, error.location.start(), error.location.end())
+    } else {
+        adjusted_error_locations(source_file, error.location)
+    };
     NormalizedParseDiagnostic::new(error.error, loc, end_loc)
 }
 
@@ -1635,11 +1636,9 @@ impl BarrySource<'_> {
         })
     }
 
-    /// The obsolete `<>` operator the parse error points at, if any. In Barry
-    /// mode the operator was rewritten to `!=`, so the error lands on its
-    /// start; outside Barry mode ruff lexes `<` and then an unexpected `>`, so
-    /// the error lands one character in. Either way the whole operator is one
-    /// token to the tokenizer, so report it as one.
+    /// The `<>` operator the parse error points at, if any. In Barry mode the
+    /// operator was rewritten to `!=`, so the error lands on its start, and the
+    /// whole operator is one token to the tokenizer, so report it as one.
     #[must_use]
     pub fn invalid_legacy_operator(
         &self,
@@ -1660,9 +1659,9 @@ impl BarrySource<'_> {
             .is_none_or(|(_, offset)| range.start() <= TextSize::new(offset as u32))
     }
 
-    /// The diagnostic for this source, if any: an obsolete `<>` the parse
-    /// error points at, or -- in Barry mode only -- the first `!=`. A `<>`
-    /// takes precedence over a `!=` reported later in the source.
+    /// The Barry mode diagnostic for this source, if any: a `<>` the parse
+    /// error points at, or the first `!=`. A `<>` takes precedence over a `!=`
+    /// reported later in the source.
     #[must_use]
     pub fn diagnostic(
         &self,
@@ -1712,21 +1711,6 @@ pub fn barry_as_flufl_invalid_legacy_operator_error(
     )
 }
 
-/// Every `<>` in `source`, located by plain text search. Only used where the
-/// operator is not rewritten, so an occurrence inside a string or a comment
-/// costs nothing: it can never coincide with the location of a parse error.
-fn textual_legacy_not_equal(source: &str) -> Vec<ruff_text_size::TextRange> {
-    source
-        .match_indices("<>")
-        .map(|(offset, matched)| {
-            ruff_text_size::TextRange::at(
-                TextSize::new(offset as u32),
-                TextSize::new(matched.len() as u32),
-            )
-        })
-        .collect()
-}
-
 #[doc(hidden)]
 pub fn prepare_barry_as_flufl_source(
     source: &str,
@@ -1744,7 +1728,7 @@ pub fn prepare_barry_as_flufl_source(
         return BarrySource {
             source: Cow::Borrowed(source),
             not_equal: None,
-            legacy_not_equal: textual_legacy_not_equal(source),
+            legacy_not_equal: Vec::new(),
         };
     };
 
@@ -2655,7 +2639,10 @@ mod tests {
     fn obsolete_not_equal_diagnostic_spans_the_whole_operator() {
         let err = compile("2 <> 3\n", Mode::Exec, "<obsolete>", CompileOpts::default())
             .expect_err("'<>' outside Barry mode is a syntax error");
-        assert_eq!(err.to_string(), "invalid syntax");
+        assert_eq!(
+            err.to_string(),
+            "invalid syntax.  Maybe you meant '!=' instead of '<>'?"
+        );
         assert_eq!(err.python_location(), (1, 3));
         assert_eq!(err.python_end_location(), Some((1, 5)));
 
@@ -2667,13 +2654,25 @@ mod tests {
         assert_eq!(err.python_location(), (1, 4));
         assert_eq!(err.python_end_location(), Some((1, 5)));
 
-        // A `<>` that starts a statement is reported at the `<` too, where the
-        // parser stops instead of one character in.
+        // A `<>` that starts a statement is not a comparison, so only the `<`
+        // is reported.
         let err = compile("<>\n", Mode::Exec, "<obsolete>", CompileOpts::default())
             .expect_err("a bare '<>' is a syntax error");
         assert_eq!(err.to_string(), "invalid syntax");
         assert_eq!(err.python_location(), (1, 1));
-        assert_eq!(err.python_end_location(), Some((1, 3)));
+        assert_eq!(err.python_end_location(), Some((1, 2)));
+
+        // `<<>` is a left shift followed by `>`.
+        let err = compile(
+            "2 <<> 3\n",
+            Mode::Exec,
+            "<obsolete>",
+            CompileOpts::default(),
+        )
+        .expect_err("'<<>' is a syntax error");
+        assert_eq!(err.to_string(), "invalid syntax");
+        assert_eq!(err.python_location(), (1, 5));
+        assert_eq!(err.python_end_location(), Some((1, 6)));
 
         // A bracket left open earlier in the source outranks the operator.
         let err = compile(