diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 197abcdc9..2f9b70a25 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -89,7 +89,7 @@ impl CompileError { location, end_location, source_path: source_file.name().to_owned(), - is_unclosed_bracket: false, + is_unclosed_bracket: diagnostic.is_unclosed_bracket, is_unclosed_string: diagnostic.is_unclosed_string, }) } @@ -194,6 +194,7 @@ impl NormalizedParseDiagnostic { end_location, ); diagnostic_out.is_unclosed_string = diagnostic.is_unclosed_string; + diagnostic_out.is_unclosed_bracket = diagnostic.is_unclosed_bracket; diagnostic_out } @@ -211,6 +212,7 @@ struct CpythonDiagnostic { message: String, range: ruff_text_size::TextRange, is_unclosed_string: bool, + is_unclosed_bracket: bool, } impl CpythonDiagnostic { @@ -226,6 +228,7 @@ impl CpythonDiagnostic { TextSize::new(end as u32), ), is_unclosed_string: false, + is_unclosed_bracket: false, } } @@ -233,6 +236,11 @@ impl CpythonDiagnostic { self.is_unclosed_string = true; self } + + const fn with_unclosed_bracket(mut self) -> Self { + self.is_unclosed_bracket = true; + self + } } fn cpython_parse_diagnostic_override( @@ -269,9 +277,11 @@ fn cpython_parse_diagnostic_override( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) ) { - // Only a backslash at the end of the source is an EOF error. + // exec input gets an implicit trailing newline, so a final `\` is a + // continuation that then hits EOF (`E_EOF`). single/eval see `\` at + // EOF as `E_LINECONT` instead. let terminal_backslash = source_text.len().checked_sub(1); - if !matches!(mode, Mode::Eval) + if matches!(mode, Mode::Exec) && terminal_backslash == Some(error.location.start().to_usize()) { let loc = source_line_end_location(source_file, error.location.start()); @@ -289,7 +299,7 @@ fn cpython_parse_diagnostic_override( )); } - source_error!(unterminated_string_error(source_text)); + source_error!(unterminated_string_error(source_text, mode)); source_error!(expected_indented_block_error(error, source_text)); if matches!( @@ -494,6 +504,23 @@ fn is_ascii_identifier_char(byte: u8) -> bool { byte == b'_' || byte.is_ascii_alphanumeric() } +fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { + if index == 0 { + return false; + } + if bytes[index - 1].is_ascii() { + return is_ascii_identifier_char(bytes[index - 1]); + } + let mut start = index - 1; + while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 { + start -= 1; + } + ::core::str::from_utf8(&bytes[start..index]) + .ok() + .and_then(|text| text.chars().next_back()) + .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) +} + fn numeric_keyword_suffix(rest: &[u8]) -> bool { rest.starts_with(b"and") || rest.starts_with(b"else") @@ -3894,7 +3921,7 @@ fn non_printable_character_error(source: &str) -> Option { None } -fn unterminated_string_error(source: &str) -> Option { +fn unterminated_string_error(source: &str, mode: Mode) -> Option { let bytes = source.as_bytes(); let mut index = 0; let mut line = 1usize; @@ -3921,6 +3948,7 @@ fn unterminated_string_error(source: &str) -> Option { }; index += quote_size; let mut has_escaped_quote = false; + let mut ended_with_escape = false; let mut closed = false; while index < bytes.len() { let c = bytes[index]; @@ -3952,6 +3980,7 @@ fn unterminated_string_error(source: &str) -> Option { if bytes.get(index + 1) == Some("e) { has_escaped_quote = true; } + ended_with_escape = index + 1 >= bytes.len(); index = (index + 2).min(bytes.len()); } else { index += 1; @@ -3964,17 +3993,30 @@ fn unterminated_string_error(source: &str) -> Option { return Some(error); } let detected_line = if quote_size == 3 { line } else { start_line }; + let interpolated = interpolated_string_prefix(bytes, start); let diagnostic = CpythonDiagnostic::new( unterminated_string_message( detected_line, quote_size == 3, has_escaped_quote, - interpolated_string_prefix(bytes, start), + interpolated, ), start, start, ); - return Some(if index >= bytes.len() { + // E_EOLS is only for a plain single-quoted string at EOF. + // Single-quoted f/t-strings never set it. exec input appends a + // newline, so a single-quoted literal becomes the newline case + // unless a final `\` consumes that newline. + let exec_implicit_newline = + matches!(mode, Mode::Exec) && quote_size == 1 && !ended_with_escape; + let eval_assignment = + matches!(mode, Mode::Eval) && eval_has_assignment_before(bytes, start); + let continuable = index >= bytes.len() + && !(interpolated.is_some() && quote_size == 1) + && !exec_implicit_newline + && !eval_assignment; + return Some(if continuable { diagnostic.with_unclosed_string() } else { diagnostic @@ -3987,6 +4029,50 @@ fn unterminated_string_error(source: &str) -> Option { None } +fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool { + let mut index = 0; + let mut level = 0usize; + let mut in_lambda_params = false; + while index < end { + match bytes[index] { + b'#' => { + while index < end && bytes[index] != b'\n' { + index += 1; + } + } + b'\'' | b'"' => index = skip_quoted_string(bytes, index), + b'(' | b'[' | b'{' => { + level += 1; + index += 1; + } + b')' | b']' | b'}' => { + level = level.saturating_sub(1); + index += 1; + } + b':' if level == 0 => { + in_lambda_params = false; + index += 1; + } + b'=' if level == 0 + && !in_lambda_params + && bytes.get(index + 1) != Some(&b'=') + && !matches!( + bytes.get(index.saturating_sub(1)), + Some(b':' | b'!' | b'<' | b'>') + ) => + { + return true; + } + _ if level == 0 && starts_identifier(bytes, index, b"lambda") => { + in_lambda_params = true; + index += b"lambda".len(); + } + _ => index += 1, + } + } + false +} + fn invalid_interpolated_string_error(source: &str) -> Option { let bytes = source.as_bytes(); let mut index = 0; @@ -4161,7 +4247,7 @@ fn interpolated_string_prefix(bytes: &[u8], quote: usize) -> Option<&'static str return None; }; - if prefix_start > 0 && is_ascii_identifier_char(bytes[prefix_start - 1]) { + if prefix_start > 0 && identifier_continue_before(bytes, prefix_start) { return None; } @@ -4424,11 +4510,10 @@ fn replacement_field_comment_error( } b'#' => { if !bytes[index..literal_end].contains(&b'\n') { - return Some(CpythonDiagnostic::new( - "'{' was never closed".to_owned(), - open, - open + 1, - )); + return Some( + CpythonDiagnostic::new("'{' was never closed".to_owned(), open, open + 1) + .with_unclosed_bracket(), + ); } index = skip_replacement_field_comment(bytes, index, end); continue; @@ -5482,6 +5567,7 @@ fn unclosed_replacement_field_error( position, position + 1, ) + .with_unclosed_bracket() }) } @@ -6173,6 +6259,49 @@ fn invalid_unparenthesized_yield_after_comma_error(source: &str) -> Option bool { + matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') +} + +/// True when every line is empty or a comment after tokenizer whitespace. +#[doc(hidden)] +#[must_use] +pub fn is_blank_python_source(source: &str) -> bool { + source.lines().all(|line| { + let trimmed = line.trim_matches(is_ascii_tokenizer_whitespace); + trimmed.is_empty() || trimmed.starts_with('#') + }) +} + +fn single_mode_blank_source_error(source_file: &SourceFile) -> Option { + let source = source_file.source_text(); + if !is_blank_python_source(source) { + return None; + } + let has_indent_only_line = source.lines().any(|line| { + line.trim_matches(is_ascii_tokenizer_whitespace).is_empty() + && line.chars().any(|c| matches!(c, ' ' | '\t')) + }); + if has_indent_only_line { + let (location, end_location) = + source_locations(source_file, TextSize::new(0), TextSize::new(0)); + return Some(CompileError::Parse(ParseError { + error: parser::ParseErrorType::UnexpectedIndentation, + raw_location: ruff_text_size::TextRange::new(TextSize::new(0), TextSize::new(0)), + location, + end_location, + source_path: source_file.name().to_owned(), + is_unclosed_bracket: false, + is_unclosed_string: false, + })); + } + Some(CompileError::from_source_error( + source_file, + CpythonDiagnostic::new("invalid syntax".to_owned(), 0, 0), + )) +} + /// The byte-order mark is only stripped while decoding source bytes, so one /// that survives into the text is just a non-printable character. The tokenizer /// rejects it everywhere except at the very start of the text, which is where @@ -6590,6 +6719,11 @@ fn _compile_with_syntax_warning_handler<'a>( } let parsed = parsed.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; + if matches!(mode, Mode::Single) + && let Some(error) = single_mode_blank_source_error(&source_file) + { + return Err(error); + } if opts.dont_imply_dedent && matches!(mode, Mode::Single) && let Some(error) = dont_imply_dedent_source_error(&source_file) diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index f2af52f1b..423fbc189 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -9,7 +9,7 @@ use rustpython_compiler_core::SourceLocation; use core::ops::RangeInclusive; #[cfg(feature = "parser")] -use rustpython_compiler::{CompileError, ParseError}; +use rustpython_compiler::{CompileError, ParseError, is_blank_python_source}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -846,9 +846,12 @@ impl VirtualMachine { self.ctx.exceptions.indentation_error } } else if allow_incomplete - && source.is_some_and(|source| { - raw_location.end().to_usize() >= source.len() && !source.ends_with('\n') - }) + && (s == "incomplete input" + || (s == "unexpected EOF while parsing" + && source.is_some_and(|source| { + raw_location.end().to_usize() >= source.len() + && !source.ends_with('\n') + }))) { self.ctx.exceptions.incomplete_input_error } else { @@ -1145,13 +1148,6 @@ impl VirtualMachine { define_exception_fn!(fn new_unbound_local_error, unbound_local_error, UnboundLocalError); } -fn is_blank_python_source(source: &str) -> bool { - source.lines().all(|line| { - let trimmed = line.trim(); - trimmed.is_empty() || trimmed.starts_with('#') - }) -} - #[cfg(feature = "parser")] enum QuotedStringScan { Closed(usize), @@ -1179,7 +1175,13 @@ fn unclosed_string_is_incomplete(source: &str) -> bool { QuotedStringScan::Unclosed { triple, unescaped_newline, - } => return triple || !unescaped_newline, + } => { + // Single-quoted f/t-strings never set E_EOLS. + if !triple && interpolated_string_prefix_at(bytes, index) { + return false; + } + return triple || !unescaped_newline; + } }, _ => index += 1, } @@ -1187,6 +1189,47 @@ fn unclosed_string_is_incomplete(source: &str) -> bool { false } +#[cfg(feature = "parser")] +fn interpolated_string_prefix_at(bytes: &[u8], quote: usize) -> bool { + let Some(&prev) = quote.checked_sub(1).and_then(|index| bytes.get(index)) else { + return false; + }; + let lower = prev.to_ascii_lowercase(); + let marker_index = if matches!(lower, b'f' | b't') { + if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') { + quote - 2 + } else { + quote - 1 + } + } else if lower == b'r' + && quote >= 2 + && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't') + { + quote - 2 + } else { + return false; + }; + marker_index == 0 || !identifier_continue_before(bytes, marker_index) +} + +#[cfg(feature = "parser")] +fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { + if index == 0 { + return false; + } + if bytes[index - 1].is_ascii() { + return bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphanumeric(); + } + let mut start = index - 1; + while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 { + start -= 1; + } + ::core::str::from_utf8(&bytes[start..index]) + .ok() + .and_then(|text| text.chars().next_back()) + .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) +} + #[cfg(feature = "parser")] fn scan_quoted_string_for_incomplete(bytes: &[u8], quote_index: usize) -> QuotedStringScan { let quote = bytes[quote_index]; diff --git a/extra_tests/snippets/builtin_compile.py b/extra_tests/snippets/builtin_compile.py index ff11bbda2..9c3cc9a5a 100644 --- a/extra_tests/snippets/builtin_compile.py +++ b/extra_tests/snippets/builtin_compile.py @@ -144,6 +144,81 @@ except _IncompleteInputError as exc: else: raise AssertionError("expected _IncompleteInputError") + +def _expect_incomplete(source, mode): + try: + compile(source, "", mode, flags=PY_CF_ALLOW_INCOMPLETE_INPUT) + except _IncompleteInputError: + return + raise AssertionError(f"expected _IncompleteInputError for {source!r} {mode}") + + +def _expect_syntax(source, mode): + try: + compile(source, "", mode, flags=PY_CF_ALLOW_INCOMPLETE_INPUT) + except _IncompleteInputError: + raise AssertionError(f"unexpected _IncompleteInputError for {source!r} {mode}") + except SyntaxError: + return + raise AssertionError(f"expected SyntaxError for {source!r} {mode}") + + +# eval/single treat empty input as incomplete; exec accepts an empty module. +_expect_incomplete("", "eval") +_expect_incomplete("", "single") +_expect_incomplete("\n", "eval") +_expect_incomplete("\n", "single") +compile("", "", "exec", flags=PY_CF_ALLOW_INCOMPLETE_INPUT) + +# A plain string still open at EOF is incomplete, except in exec, which +# appends a newline and so sees an unescaped newline instead. +_expect_incomplete("'abc", "eval") +_expect_incomplete("'abc", "single") +_expect_syntax("'abc", "exec") +# A final `\` eats exec's implicit newline, so the string is still open at EOF. +_expect_incomplete("a = 'a\\", "exec") +_expect_incomplete("a = 'a\\", "single") +_expect_syntax("a = 'a\\", "eval") +_expect_incomplete("'''abc", "eval") +_expect_incomplete("'''abc", "exec") + +# Single-quoted f/t-strings never set E_EOLS. +_expect_syntax("f'", "eval") +_expect_syntax("f'", "exec") +_expect_syntax("f'", "single") +_expect_syntax("f'{1+'", "eval") +# An unclosed `{` in the field is EOF (`E_EOF`), so it is incomplete. +_expect_incomplete("f'{", "eval") +_expect_incomplete("f'{1", "eval") +_expect_incomplete("f'''", "eval") +_expect_incomplete("f'''", "exec") + +# exec turns a final `\` into a continuation then EOF; single/eval do not. +_expect_syntax("9+ \\", "eval") +_expect_syntax("9+ \\", "single") +_expect_incomplete("9+ \\", "exec") + +# Non-tokenizer whitespace is a hard error, not blank input. +_expect_syntax("\xa0", "eval") +_expect_syntax("\xa0", "exec") +_expect_syntax("\x0b", "eval") + +# A lambda default `=` is not an assignment statement. +_expect_incomplete("lambda x='abc", "eval") +# A Unicode identifier plus a quote is a plain string, not an f-string. +_expect_incomplete("éf'", "eval") +# Comment-only single input is invalid syntax, not an indent error. +try: + compile("# a", "", "single") +except IndentationError: + raise AssertionError("comment-only single input should not be IndentationError") +except SyntaxError: + pass +else: + raise AssertionError("expected SyntaxError for comment-only single input") +_expect_incomplete("# a", "single") +_expect_incomplete(" #x", "single") + # The source is encoded before it is parsed, so a lone surrogate has to be # reported rather than assumed away. with assert_raises(UnicodeEncodeError):