Classify incomplete input from tokenizer EOF flags (#8703)

* Classify incomplete input from tokenizer EOF flags

Treat exec's implicit newline, single-quoted f/t-strings, blank
single input, and unclosed field braces the way E_EOF, E_EOFS,
and E_EOLS do.

Assisted-by: Grok:4.6

* address review: lambda defaults, unicode prefixes, comments

Keep lambda parameter defaults and Unicode identifiers continuable,
and report comment-only single input as SyntaxError.

Assisted-by: Grok:4.6

crates/compiler/src/lib.rs

@@ -89,7 +89,7 @@ impl CompileError {
             location,
             end_location,
             source_path: source_file.name().to_owned(),
-            is_unclosed_bracket: false,
+            is_unclosed_bracket: diagnostic.is_unclosed_bracket,
             is_unclosed_string: diagnostic.is_unclosed_string,
         })
     }
@@ -194,6 +194,7 @@ impl NormalizedParseDiagnostic {
             end_location,
         );
         diagnostic_out.is_unclosed_string = diagnostic.is_unclosed_string;
+        diagnostic_out.is_unclosed_bracket = diagnostic.is_unclosed_bracket;
         diagnostic_out
     }
 
@@ -211,6 +212,7 @@ struct CpythonDiagnostic {
     message: String,
     range: ruff_text_size::TextRange,
     is_unclosed_string: bool,
+    is_unclosed_bracket: bool,
 }
 
 impl CpythonDiagnostic {
@@ -226,6 +228,7 @@ impl CpythonDiagnostic {
                 TextSize::new(end as u32),
             ),
             is_unclosed_string: false,
+            is_unclosed_bracket: false,
         }
     }
 
@@ -233,6 +236,11 @@ impl CpythonDiagnostic {
         self.is_unclosed_string = true;
         self
     }
+
+    const fn with_unclosed_bracket(mut self) -> Self {
+        self.is_unclosed_bracket = true;
+        self
+    }
 }
 
 fn cpython_parse_diagnostic_override(
@@ -269,9 +277,11 @@ fn cpython_parse_diagnostic_override(
         &error.error,
         parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError)
     ) {
-        // Only a backslash at the end of the source is an EOF error.
+        // exec input gets an implicit trailing newline, so a final `\` is a
+        // continuation that then hits EOF (`E_EOF`). single/eval see `\` at
+        // EOF as `E_LINECONT` instead.
         let terminal_backslash = source_text.len().checked_sub(1);
-        if !matches!(mode, Mode::Eval)
+        if matches!(mode, Mode::Exec)
             && terminal_backslash == Some(error.location.start().to_usize())
         {
             let loc = source_line_end_location(source_file, error.location.start());
@@ -289,7 +299,7 @@ fn cpython_parse_diagnostic_override(
         ));
     }
 
-    source_error!(unterminated_string_error(source_text));
+    source_error!(unterminated_string_error(source_text, mode));
     source_error!(expected_indented_block_error(error, source_text));
 
     if matches!(
@@ -494,6 +504,23 @@ fn is_ascii_identifier_char(byte: u8) -> bool {
     byte == b'_' || byte.is_ascii_alphanumeric()
 }
 
+fn identifier_continue_before(bytes: &[u8], index: usize) -> bool {
+    if index == 0 {
+        return false;
+    }
+    if bytes[index - 1].is_ascii() {
+        return is_ascii_identifier_char(bytes[index - 1]);
+    }
+    let mut start = index - 1;
+    while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 {
+        start -= 1;
+    }
+    ::core::str::from_utf8(&bytes[start..index])
+        .ok()
+        .and_then(|text| text.chars().next_back())
+        .is_some_and(|ch| ch == '_' || ch.is_alphanumeric())
+}
+
 fn numeric_keyword_suffix(rest: &[u8]) -> bool {
     rest.starts_with(b"and")
         || rest.starts_with(b"else")
@@ -3894,7 +3921,7 @@ fn non_printable_character_error(source: &str) -> Option<CpythonDiagnostic> {
     None
 }
 
-fn unterminated_string_error(source: &str) -> Option<CpythonDiagnostic> {
+fn unterminated_string_error(source: &str, mode: Mode) -> Option<CpythonDiagnostic> {
     let bytes = source.as_bytes();
     let mut index = 0;
     let mut line = 1usize;
@@ -3921,6 +3948,7 @@ fn unterminated_string_error(source: &str) -> Option<CpythonDiagnostic> {
                 };
                 index += quote_size;
                 let mut has_escaped_quote = false;
+                let mut ended_with_escape = false;
                 let mut closed = false;
                 while index < bytes.len() {
                     let c = bytes[index];
@@ -3952,6 +3980,7 @@ fn unterminated_string_error(source: &str) -> Option<CpythonDiagnostic> {
                         if bytes.get(index + 1) == Some(&quote) {
                             has_escaped_quote = true;
                         }
+                        ended_with_escape = index + 1 >= bytes.len();
                         index = (index + 2).min(bytes.len());
                     } else {
                         index += 1;
@@ -3964,17 +3993,30 @@ fn unterminated_string_error(source: &str) -> Option<CpythonDiagnostic> {
                         return Some(error);
                     }
                     let detected_line = if quote_size == 3 { line } else { start_line };
+                    let interpolated = interpolated_string_prefix(bytes, start);
                     let diagnostic = CpythonDiagnostic::new(
                         unterminated_string_message(
                             detected_line,
                             quote_size == 3,
                             has_escaped_quote,
-                            interpolated_string_prefix(bytes, start),
+                            interpolated,
                         ),
                         start,
                         start,
                     );
-                    return Some(if index >= bytes.len() {
+                    // E_EOLS is only for a plain single-quoted string at EOF.
+                    // Single-quoted f/t-strings never set it. exec input appends a
+                    // newline, so a single-quoted literal becomes the newline case
+                    // unless a final `\` consumes that newline.
+                    let exec_implicit_newline =
+                        matches!(mode, Mode::Exec) && quote_size == 1 && !ended_with_escape;
+                    let eval_assignment =
+                        matches!(mode, Mode::Eval) && eval_has_assignment_before(bytes, start);
+                    let continuable = index >= bytes.len()
+                        && !(interpolated.is_some() && quote_size == 1)
+                        && !exec_implicit_newline
+                        && !eval_assignment;
+                    return Some(if continuable {
                         diagnostic.with_unclosed_string()
                     } else {
                         diagnostic
@@ -3987,6 +4029,50 @@ fn unterminated_string_error(source: &str) -> Option<CpythonDiagnostic> {
     None
 }
 
+fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool {
+    let mut index = 0;
+    let mut level = 0usize;
+    let mut in_lambda_params = false;
+    while index < end {
+        match bytes[index] {
+            b'#' => {
+                while index < end && bytes[index] != b'\n' {
+                    index += 1;
+                }
+            }
+            b'\'' | b'"' => index = skip_quoted_string(bytes, index),
+            b'(' | b'[' | b'{' => {
+                level += 1;
+                index += 1;
+            }
+            b')' | b']' | b'}' => {
+                level = level.saturating_sub(1);
+                index += 1;
+            }
+            b':' if level == 0 => {
+                in_lambda_params = false;
+                index += 1;
+            }
+            b'=' if level == 0
+                && !in_lambda_params
+                && bytes.get(index + 1) != Some(&b'=')
+                && !matches!(
+                    bytes.get(index.saturating_sub(1)),
+                    Some(b':' | b'!' | b'<' | b'>')
+                ) =>
+            {
+                return true;
+            }
+            _ if level == 0 && starts_identifier(bytes, index, b"lambda") => {
+                in_lambda_params = true;
+                index += b"lambda".len();
+            }
+            _ => index += 1,
+        }
+    }
+    false
+}
+
 fn invalid_interpolated_string_error(source: &str) -> Option<CpythonDiagnostic> {
     let bytes = source.as_bytes();
     let mut index = 0;
@@ -4161,7 +4247,7 @@ fn interpolated_string_prefix(bytes: &[u8], quote: usize) -> Option<&'static str
         return None;
     };
 
-    if prefix_start > 0 && is_ascii_identifier_char(bytes[prefix_start - 1]) {
+    if prefix_start > 0 && identifier_continue_before(bytes, prefix_start) {
         return None;
     }
 
@@ -4424,11 +4510,10 @@ fn replacement_field_comment_error(
             }
             b'#' => {
                 if !bytes[index..literal_end].contains(&b'\n') {
-                    return Some(CpythonDiagnostic::new(
-                        "'{' was never closed".to_owned(),
-                        open,
-                        open + 1,
-                    ));
+                    return Some(
+                        CpythonDiagnostic::new("'{' was never closed".to_owned(), open, open + 1)
+                            .with_unclosed_bracket(),
+                    );
                 }
                 index = skip_replacement_field_comment(bytes, index, end);
                 continue;
@@ -5482,6 +5567,7 @@ fn unclosed_replacement_field_error(
             position,
             position + 1,
         )
+        .with_unclosed_bracket()
     })
 }
 
@@ -6173,6 +6259,49 @@ fn invalid_unparenthesized_yield_after_comma_error(source: &str) -> Option<Cpyth
     None
 }
 
+/// Horizontal and vertical space the tokenizer skips, not `str::trim()`.
+const fn is_ascii_tokenizer_whitespace(c: char) -> bool {
+    matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c')
+}
+
+/// True when every line is empty or a comment after tokenizer whitespace.
+#[doc(hidden)]
+#[must_use]
+pub fn is_blank_python_source(source: &str) -> bool {
+    source.lines().all(|line| {
+        let trimmed = line.trim_matches(is_ascii_tokenizer_whitespace);
+        trimmed.is_empty() || trimmed.starts_with('#')
+    })
+}
+
+fn single_mode_blank_source_error(source_file: &SourceFile) -> Option<CompileError> {
+    let source = source_file.source_text();
+    if !is_blank_python_source(source) {
+        return None;
+    }
+    let has_indent_only_line = source.lines().any(|line| {
+        line.trim_matches(is_ascii_tokenizer_whitespace).is_empty()
+            && line.chars().any(|c| matches!(c, ' ' | '\t'))
+    });
+    if has_indent_only_line {
+        let (location, end_location) =
+            source_locations(source_file, TextSize::new(0), TextSize::new(0));
+        return Some(CompileError::Parse(ParseError {
+            error: parser::ParseErrorType::UnexpectedIndentation,
+            raw_location: ruff_text_size::TextRange::new(TextSize::new(0), TextSize::new(0)),
+            location,
+            end_location,
+            source_path: source_file.name().to_owned(),
+            is_unclosed_bracket: false,
+            is_unclosed_string: false,
+        }));
+    }
+    Some(CompileError::from_source_error(
+        source_file,
+        CpythonDiagnostic::new("invalid syntax".to_owned(), 0, 0),
+    ))
+}
+
 /// The byte-order mark is only stripped while decoding source bytes, so one
 /// that survives into the text is just a non-printable character. The tokenizer
 /// rejects it everywhere except at the very start of the text, which is where
@@ -6590,6 +6719,11 @@ fn _compile_with_syntax_warning_handler<'a>(
     }
     let parsed =
         parsed.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?;
+    if matches!(mode, Mode::Single)
+        && let Some(error) = single_mode_blank_source_error(&source_file)
+    {
+        return Err(error);
+    }
     if opts.dont_imply_dedent
         && matches!(mode, Mode::Single)
         && let Some(error) = dont_imply_dedent_source_error(&source_file)

crates/vm/src/vm/vm_new.rs

@@ -9,7 +9,7 @@ use rustpython_compiler_core::SourceLocation;
 use core::ops::RangeInclusive;
 
 #[cfg(feature = "parser")]
-use rustpython_compiler::{CompileError, ParseError};
+use rustpython_compiler::{CompileError, ParseError, is_blank_python_source};
 
 use crate::{
     AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
@@ -846,9 +846,12 @@ impl VirtualMachine {
                         self.ctx.exceptions.indentation_error
                     }
                 } else if allow_incomplete
-                    && source.is_some_and(|source| {
-                        raw_location.end().to_usize() >= source.len() && !source.ends_with('\n')
-                    })
+                    && (s == "incomplete input"
+                        || (s == "unexpected EOF while parsing"
+                            && source.is_some_and(|source| {
+                                raw_location.end().to_usize() >= source.len()
+                                    && !source.ends_with('\n')
+                            })))
                 {
                     self.ctx.exceptions.incomplete_input_error
                 } else {
@@ -1145,13 +1148,6 @@ impl VirtualMachine {
     define_exception_fn!(fn new_unbound_local_error, unbound_local_error, UnboundLocalError);
 }
 
-fn is_blank_python_source(source: &str) -> bool {
-    source.lines().all(|line| {
-        let trimmed = line.trim();
-        trimmed.is_empty() || trimmed.starts_with('#')
-    })
-}
-
 #[cfg(feature = "parser")]
 enum QuotedStringScan {
     Closed(usize),
@@ -1179,7 +1175,13 @@ fn unclosed_string_is_incomplete(source: &str) -> bool {
                 QuotedStringScan::Unclosed {
                     triple,
                     unescaped_newline,
-                } => return triple || !unescaped_newline,
+                } => {
+                    // Single-quoted f/t-strings never set E_EOLS.
+                    if !triple && interpolated_string_prefix_at(bytes, index) {
+                        return false;
+                    }
+                    return triple || !unescaped_newline;
+                }
             },
             _ => index += 1,
         }
@@ -1187,6 +1189,47 @@ fn unclosed_string_is_incomplete(source: &str) -> bool {
     false
 }
 
+#[cfg(feature = "parser")]
+fn interpolated_string_prefix_at(bytes: &[u8], quote: usize) -> bool {
+    let Some(&prev) = quote.checked_sub(1).and_then(|index| bytes.get(index)) else {
+        return false;
+    };
+    let lower = prev.to_ascii_lowercase();
+    let marker_index = if matches!(lower, b'f' | b't') {
+        if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') {
+            quote - 2
+        } else {
+            quote - 1
+        }
+    } else if lower == b'r'
+        && quote >= 2
+        && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't')
+    {
+        quote - 2
+    } else {
+        return false;
+    };
+    marker_index == 0 || !identifier_continue_before(bytes, marker_index)
+}
+
+#[cfg(feature = "parser")]
+fn identifier_continue_before(bytes: &[u8], index: usize) -> bool {
+    if index == 0 {
+        return false;
+    }
+    if bytes[index - 1].is_ascii() {
+        return bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphanumeric();
+    }
+    let mut start = index - 1;
+    while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 {
+        start -= 1;
+    }
+    ::core::str::from_utf8(&bytes[start..index])
+        .ok()
+        .and_then(|text| text.chars().next_back())
+        .is_some_and(|ch| ch == '_' || ch.is_alphanumeric())
+}
+
 #[cfg(feature = "parser")]
 fn scan_quoted_string_for_incomplete(bytes: &[u8], quote_index: usize) -> QuotedStringScan {
     let quote = bytes[quote_index];

extra_tests/snippets/builtin_compile.py

@@ -144,6 +144,81 @@ except _IncompleteInputError as exc:
 else:
     raise AssertionError("expected _IncompleteInputError")
 
+
+def _expect_incomplete(source, mode):
+    try:
+        compile(source, "<test>", mode, flags=PY_CF_ALLOW_INCOMPLETE_INPUT)
+    except _IncompleteInputError:
+        return
+    raise AssertionError(f"expected _IncompleteInputError for {source!r} {mode}")
+
+
+def _expect_syntax(source, mode):
+    try:
+        compile(source, "<test>", mode, flags=PY_CF_ALLOW_INCOMPLETE_INPUT)
+    except _IncompleteInputError:
+        raise AssertionError(f"unexpected _IncompleteInputError for {source!r} {mode}")
+    except SyntaxError:
+        return
+    raise AssertionError(f"expected SyntaxError for {source!r} {mode}")
+
+
+# eval/single treat empty input as incomplete; exec accepts an empty module.
+_expect_incomplete("", "eval")
+_expect_incomplete("", "single")
+_expect_incomplete("\n", "eval")
+_expect_incomplete("\n", "single")
+compile("", "<test>", "exec", flags=PY_CF_ALLOW_INCOMPLETE_INPUT)
+
+# A plain string still open at EOF is incomplete, except in exec, which
+# appends a newline and so sees an unescaped newline instead.
+_expect_incomplete("'abc", "eval")
+_expect_incomplete("'abc", "single")
+_expect_syntax("'abc", "exec")
+# A final `\` eats exec's implicit newline, so the string is still open at EOF.
+_expect_incomplete("a = 'a\\", "exec")
+_expect_incomplete("a = 'a\\", "single")
+_expect_syntax("a = 'a\\", "eval")
+_expect_incomplete("'''abc", "eval")
+_expect_incomplete("'''abc", "exec")
+
+# Single-quoted f/t-strings never set E_EOLS.
+_expect_syntax("f'", "eval")
+_expect_syntax("f'", "exec")
+_expect_syntax("f'", "single")
+_expect_syntax("f'{1+'", "eval")
+# An unclosed `{` in the field is EOF (`E_EOF`), so it is incomplete.
+_expect_incomplete("f'{", "eval")
+_expect_incomplete("f'{1", "eval")
+_expect_incomplete("f'''", "eval")
+_expect_incomplete("f'''", "exec")
+
+# exec turns a final `\` into a continuation then EOF; single/eval do not.
+_expect_syntax("9+ \\", "eval")
+_expect_syntax("9+ \\", "single")
+_expect_incomplete("9+ \\", "exec")
+
+# Non-tokenizer whitespace is a hard error, not blank input.
+_expect_syntax("\xa0", "eval")
+_expect_syntax("\xa0", "exec")
+_expect_syntax("\x0b", "eval")
+
+# A lambda default `=` is not an assignment statement.
+_expect_incomplete("lambda x='abc", "eval")
+# A Unicode identifier plus a quote is a plain string, not an f-string.
+_expect_incomplete("éf'", "eval")
+# Comment-only single input is invalid syntax, not an indent error.
+try:
+    compile("# a", "<test>", "single")
+except IndentationError:
+    raise AssertionError("comment-only single input should not be IndentationError")
+except SyntaxError:
+    pass
+else:
+    raise AssertionError("expected SyntaxError for comment-only single input")
+_expect_incomplete("# a", "single")
+_expect_incomplete(" #x", "single")
+
 # The source is encoded before it is parsed, so a lone surrogate has to be
 # reported rather than assumed away.
 with assert_raises(UnicodeEncodeError):