diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 1c1ecb13e..1a7df5e27 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -1279,6 +1279,35 @@ fn first_parse_error( Err(error) } +/// The first logical input accepted by the single-input grammar. +#[derive(Clone, Copy)] +pub enum SingleModeStatementBoundary { + Simple { newline_index: usize, end: TextSize }, + Compound { end: TextSize }, +} + +/// Classify a logical input without changing parser diagnostics or recovery. +#[must_use] +pub fn single_mode_statement_boundary( + parsed: &parser::Parsed, +) -> Option { + let ast::Mod::Module(module) = parsed.syntax() else { + return None; + }; + let first = module.body.first()?; + if is_compound_stmt(first) { + return Some(SingleModeStatementBoundary::Compound { end: first.end() }); + } + let newline_index = parsed + .tokens() + .iter() + .position(|token| token.kind() == TokenKind::Newline)?; + Some(SingleModeStatementBoundary::Simple { + newline_index, + end: parsed.tokens()[newline_index].end(), + }) +} + /// Single mode reads only the first statement when it is a simple statement, so any code after /// its line is reported as multiple statements, ahead of errors the rest of the source has. #[must_use] @@ -1289,8 +1318,8 @@ pub fn single_mode_multiple_statements_error( let ast::Mod::Module(module) = parsed.syntax() else { return None; }; - let first = module.body.first()?; - if is_compound_stmt(first) { + let boundary = single_mode_statement_boundary(parsed)?; + if matches!(boundary, SingleModeStatementBoundary::Compound { .. }) { // The interactive grammar ends a compound statement with a NEWLINE, // so the first token of a following statement is invalid syntax. let next = module.body.get(1)?; @@ -1315,9 +1344,9 @@ pub fn single_mode_multiple_statements_error( )); } let tokens = parsed.tokens(); - let newline_index = tokens - .iter() - .position(|token| token.kind() == TokenKind::Newline)?; + let SingleModeStatementBoundary::Simple { newline_index, .. } = boundary else { + unreachable!() + }; let newline = &tokens[newline_index]; if parsed .errors() diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index baddaddd8..061ec47e7 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -1738,6 +1738,7 @@ mod escape_warnings { warnings: Vec, tokenizer_recovery: bool, has_valid_syntax: bool, + single_boundary: Option, } fn collect_parser_warnings( @@ -1745,9 +1746,11 @@ mod escape_warnings { filename: &str, mode: Option, inherited_barry_as_flufl: bool, + single_mode: bool, ) -> CollectedParserWarnings { let mut ast_tokenizer_recovery = true; let mut has_valid_syntax = false; + let mut single_boundary = None; let mut warnings = Vec::new(); collect_numeric_literal_warnings(source, &mut warnings); // The compile that follows rejects this source; parsing it here @@ -1765,6 +1768,9 @@ mod escape_warnings { let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), options); if parsed.has_valid_syntax() { has_valid_syntax = true; + if single_mode { + single_boundary = compiler::single_mode_statement_boundary(&parsed); + } let mut visitor = EscapeWarningVisitor { source, warnings: &mut warnings, @@ -1837,6 +1843,7 @@ mod escape_warnings { warnings, tokenizer_recovery: ast_tokenizer_recovery, has_valid_syntax, + single_boundary, } } @@ -1861,14 +1868,20 @@ mod escape_warnings { inherited_barry_as_flufl: bool, emitted_escapes: &[usize], ) -> Result<(), CompileWarningError> { + let is_single = matches!(mode, compiler::Mode::Single); let mode = match mode { compiler::Mode::Eval => ruff_python_parser::Mode::Expression, compiler::Mode::Exec | compiler::Mode::Single | compiler::Mode::BlockExpr => { ruff_python_parser::Mode::Module } }; - let collected = - collect_parser_warnings(source, filename, Some(mode), inherited_barry_as_flufl); + let collected = collect_parser_warnings( + source, + filename, + Some(mode), + inherited_barry_as_flufl, + is_single, + ); if !collected.has_valid_syntax || !collected.tokenizer_recovery { // Invalid source and interpolated tokenizer warnings need a // separate recovery contract. Preserve their existing behavior. @@ -1880,9 +1893,10 @@ mod escape_warnings { filename, module, collected.warnings, + collected.single_boundary, WarningEmission::Standard { emitted_escapes, - tokenizer_recovery: collected.tokenizer_recovery, + tokenizer_recovery: true, }, ) .map(|_| ()) @@ -1902,6 +1916,7 @@ mod escape_warnings { filename, module, warnings, + None, WarningEmission::Standard { emitted_escapes: &[], tokenizer_recovery: false, @@ -1954,6 +1969,7 @@ mod escape_warnings { filename, module, warnings, + None, WarningEmission::Standard { emitted_escapes, tokenizer_recovery: false, @@ -1975,13 +1991,26 @@ mod escape_warnings { CompileStart::Eval => Some(ruff_python_parser::Mode::Expression), CompileStart::FuncType => None, }; - let collected = - collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl); + let collected = collect_parser_warnings( + source, + filename, + mode, + inherited_barry_as_flufl, + matches!(start, CompileStart::Single), + ); self.emit_parser_warning_batch( source, filename, module.map(AsObject::as_object), collected.warnings, + if matches!(start, CompileStart::Single) + && collected.has_valid_syntax + && collected.tokenizer_recovery + { + collected.single_boundary + } else { + None + }, WarningEmission::Ast { // Unlike an interactive prompt, Single compile input recovers // after a parser warning becomes an error. Later numeric @@ -2010,12 +2039,14 @@ mod escape_warnings { filename, Some(ruff_python_parser::Mode::Module), inherited_barry_as_flufl, + false, ); self.emit_parser_warning_batch( source, filename, module.map(AsObject::as_object), collected.warnings, + None, WarningEmission::Interactive(emitted), ) .map(|_| ()) @@ -2027,6 +2058,7 @@ mod escape_warnings { filename: &str, module: Option<&PyObject>, warnings: Vec, + single_boundary: Option, emission: WarningEmission<'_>, ) -> Result, CompileWarningError> { let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission { @@ -2040,6 +2072,26 @@ mod escape_warnings { let mut considered_escapes = Vec::new(); let mut parser_error = None; for warning in warnings { + // A successful single input does not parse later statements. + // Keep those numeric events for recovery after an earlier + // parser warning escalates, rather than truncating the source. + if parser_error.is_none() + && let Some(boundary) = single_boundary + { + let end = match (boundary, &warning) { + (compiler::SingleModeStatementBoundary::Simple { end, .. }, _) => Some(end), + ( + compiler::SingleModeStatementBoundary::Compound { end }, + ParserWarning::Escape(_, EscapeOrigin::Parser, _), + ) => Some(end), + // Compound input requests another token and can emit + // tokenizer warnings while rejecting the next statement. + _ => None, + }; + if end.is_some_and(|end| warning.offset() >= end.to_usize()) { + continue; + } + } // CPython resumes tokenization after a parser-origin SyntaxError. // Preserve ordinary warning order and never replay prior warnings. if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) {