Support incomplete parsing (#5764)

* continue accepting REPL input for multiline strings

* Match cpython behavior for all multi-line statements (execute when complete)

* Emit _IncompleteInputError when compiling with incomplete flag

* Refine when _IncompleteInputError is emitted

* Support multiline strings emitting _IncompleteInputError

* lint

* Undo accidental change to PyTabError

* match -> if let

* Fix test_baseexception and test_codeop

* fix spelling

* fix exception name

* Skip pickle test of _IncompleteInputError

* Use py3.15's codeop implementation

* Update Lib/test/test_baseexception.py

---------

Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>

Lib/codeop.py

@@ -65,14 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol):
             try:
                 compiler(source + "\n", filename, symbol)
                 return None
+            except _IncompleteInputError as e:
+                return None
             except SyntaxError as e:
-                # XXX: RustPython; support multiline definitions in REPL
-                # See also: https://github.com/RustPython/RustPython/pull/5743
-                strerr = str(e)
-                if source.endswith(":") and "expected an indented block" in strerr:
-                    return None
-                elif "incomplete input" in str(e):
-                    return None
+                pass
                 # fallthrough
 
     return compiler(source, filename, symbol, incomplete_input=False)

Lib/test/test_baseexception.py

@@ -83,6 +83,8 @@ class ExceptionClassTests(unittest.TestCase):
         exc_set = set(e for e in exc_set if not e.startswith('_'))
         # RUSTPYTHON specific
         exc_set.discard("JitError")
+        # TODO: RUSTPYTHON; this will be officially introduced in Python 3.15
+        exc_set.discard("IncompleteInputError")
         self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set)
 
     interface_tests = ("length", "args", "str", "repr")

Lib/test/test_pickle.py

@@ -664,6 +664,9 @@ class CompatPickleTests(unittest.TestCase):
                            BaseExceptionGroup,
                            ExceptionGroup):
                     continue
+                # TODO: RUSTPYTHON: fix name mapping for _IncompleteInputError
+                if exc is _IncompleteInputError:
+                    continue
                 if exc is not OSError and issubclass(exc, OSError):
                     self.assertEqual(reverse_mapping('builtins', name),
                                      ('exceptions', 'OSError'))

compiler/src/lib.rs

@@ -25,6 +25,7 @@ pub enum CompileErrorType {
 pub struct ParseError {
     #[source]
     pub error: parser::ParseErrorType,
+    pub raw_location: ruff_text_size::TextRange,
     pub location: SourceLocation,
     pub source_path: String,
 }
@@ -48,6 +49,7 @@ impl CompileError {
         let location = source_code.source_location(error.location.start());
         Self::Parse(ParseError {
             error: error.error,
+            raw_location: error.location,
             location,
             source_path: source_code.path.to_owned(),
         })

src/shell.rs

@@ -1,7 +1,8 @@
 mod helper;
 
 use rustpython_compiler::{
-    CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType,
+    CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType,
+    parser::ParseErrorType,
 };
 use rustpython_vm::{
     AsObject, PyResult, VirtualMachine,
@@ -14,7 +15,8 @@ use rustpython_vm::{
 enum ShellExecResult {
     Ok,
     PyErr(PyBaseExceptionRef),
-    Continue,
+    ContinueBlock,
+    ContinueLine,
 }
 
 fn shell_exec(
@@ -22,11 +24,17 @@ fn shell_exec(
     source: &str,
     scope: Scope,
     empty_line_given: bool,
-    continuing: bool,
+    continuing_block: bool,
 ) -> ShellExecResult {
+    // compiling expects only UNIX style line endings, and will replace windows line endings
+    // internally. Since we might need to analyze the source to determine if an error could be
+    // resolved by future input, we need the location from the error to match the source code that
+    // was actually compiled.
+    #[cfg(windows)]
+    let source = &source.replace("\r\n", "\n");
     match vm.compile(source, compiler::Mode::Single, "<stdin>".to_owned()) {
         Ok(code) => {
-            if empty_line_given || !continuing {
+            if empty_line_given || !continuing_block {
                 // We want to execute the full code
                 match vm.run_code_obj(code, scope) {
                     Ok(_val) => ShellExecResult::Ok,
@@ -40,8 +48,32 @@ fn shell_exec(
         Err(CompileError::Parse(ParseError {
             error: ParseErrorType::Lexical(LexicalErrorType::Eof),
             ..
-        })) => ShellExecResult::Continue,
+        })) => ShellExecResult::ContinueLine,
+        Err(CompileError::Parse(ParseError {
+            error:
+                ParseErrorType::Lexical(LexicalErrorType::FStringError(
+                    FStringErrorType::UnterminatedTripleQuotedString,
+                )),
+            ..
+        })) => ShellExecResult::ContinueLine,
         Err(err) => {
+            // Check if the error is from an unclosed triple quoted string (which should always
+            // continue)
+            if let CompileError::Parse(ParseError {
+                error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError),
+                raw_location,
+                ..
+            }) = err
+            {
+                let loc = raw_location.start().to_usize();
+                let mut iter = source.chars();
+                if let Some(quote) = iter.nth(loc) {
+                    if iter.next() == Some(quote) && iter.next() == Some(quote) {
+                        return ShellExecResult::ContinueLine;
+                    }
+                }
+            };
+
             // bad_error == true if we are handling an error that should be thrown even if we are continuing
             // if its an indentation error, set to true if we are continuing and the error is on column 0,
             // since indentations errors on columns other than 0 should be ignored.
@@ -50,10 +82,12 @@ fn shell_exec(
             let bad_error = match err {
                 CompileError::Parse(ref p) => {
                     match &p.error {
-                        ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some()
+                        ParseErrorType::Lexical(LexicalErrorType::IndentationError) => {
+                            continuing_block
+                        } // && p.location.is_some()
                         ParseErrorType::OtherError(msg) => {
                             if msg.starts_with("Expected an indented block") {
-                                continuing
+                                continuing_block
                             } else {
                                 true
                             }
@@ -68,7 +102,7 @@ fn shell_exec(
             if empty_line_given || bad_error {
                 ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source)))
             } else {
-                ShellExecResult::Continue
+                ShellExecResult::ContinueBlock
             }
         }
     }
@@ -93,10 +127,19 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
         println!("No previous history.");
     }
 
-    let mut continuing = false;
+    // We might either be waiting to know if a block is complete, or waiting to know if a multiline
+    // statement is complete. In the former case, we need to ensure that we read one extra new line
+    // to know that the block is complete. In the latter, we can execute as soon as the statement is
+    // valid.
+    let mut continuing_block = false;
+    let mut continuing_line = false;
 
     loop {
-        let prompt_name = if continuing { "ps2" } else { "ps1" };
+        let prompt_name = if continuing_block || continuing_line {
+            "ps2"
+        } else {
+            "ps1"
+        };
         let prompt = vm
             .sys_module
             .get_attr(prompt_name, vm)
@@ -105,6 +148,8 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
             Ok(ref s) => s.as_str(),
             Err(_) => "",
         };
+
+        continuing_line = false;
         let result = match repl.readline(prompt) {
             ReadlineResult::Line(line) => {
                 debug!("You entered {:?}", line);
@@ -120,39 +165,44 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
                 }
                 full_input.push('\n');
 
-                match shell_exec(vm, &full_input, scope.clone(), empty_line_given, continuing) {
+                match shell_exec(
+                    vm,
+                    &full_input,
+                    scope.clone(),
+                    empty_line_given,
+                    continuing_block,
+                ) {
                     ShellExecResult::Ok => {
-                        if continuing {
+                        if continuing_block {
                             if empty_line_given {
-                                // We should be exiting continue mode
-                                continuing = false;
+                                // We should exit continue mode since the block successfully executed
+                                continuing_block = false;
                                 full_input.clear();
-                                Ok(())
-                            } else {
-                                // We should stay in continue mode
-                                continuing = true;
-                                Ok(())
                             }
                         } else {
                             // We aren't in continue mode so proceed normally
-                            continuing = false;
                             full_input.clear();
-                            Ok(())
                         }
+                        Ok(())
                     }
-                    ShellExecResult::Continue => {
-                        continuing = true;
+                    // Continue, but don't change the mode
+                    ShellExecResult::ContinueLine => {
+                        continuing_line = true;
+                        Ok(())
+                    }
+                    ShellExecResult::ContinueBlock => {
+                        continuing_block = true;
                         Ok(())
                     }
                     ShellExecResult::PyErr(err) => {
-                        continuing = false;
+                        continuing_block = false;
                         full_input.clear();
                         Err(err)
                     }
                 }
             }
             ReadlineResult::Interrupt => {
-                continuing = false;
+                continuing_block = false;
                 full_input.clear();
                 let keyboard_interrupt =
                     vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned());

vm/src/compiler.rs

@@ -49,3 +49,10 @@ impl crate::convert::ToPyException for (CompileError, Option<&str>) {
         vm.new_syntax_error(&self.0, self.1)
     }
 }
+
+#[cfg(any(feature = "parser", feature = "codegen"))]
+impl crate::convert::ToPyException for (CompileError, Option<&str>, bool) {
+    fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef {
+        vm.new_syntax_error_maybe_incomplete(&self.0, self.1, self.2)
+    }
+}

vm/src/exceptions.rs

@@ -495,6 +495,7 @@ pub struct ExceptionZoo {
     pub not_implemented_error: &'static Py<PyType>,
     pub recursion_error: &'static Py<PyType>,
     pub syntax_error: &'static Py<PyType>,
+    pub incomplete_input_error: &'static Py<PyType>,
     pub indentation_error: &'static Py<PyType>,
     pub tab_error: &'static Py<PyType>,
     pub system_error: &'static Py<PyType>,
@@ -743,6 +744,7 @@ impl ExceptionZoo {
         let recursion_error = PyRecursionError::init_builtin_type();
 
         let syntax_error = PySyntaxError::init_builtin_type();
+        let incomplete_input_error = PyIncompleteInputError::init_builtin_type();
         let indentation_error = PyIndentationError::init_builtin_type();
         let tab_error = PyTabError::init_builtin_type();
 
@@ -817,6 +819,7 @@ impl ExceptionZoo {
             not_implemented_error,
             recursion_error,
             syntax_error,
+            incomplete_input_error,
             indentation_error,
             tab_error,
             system_error,
@@ -965,6 +968,7 @@ impl ExceptionZoo {
             "end_offset" => ctx.none(),
             "text" => ctx.none(),
         });
+        extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error);
         extend_exception!(PyIndentationError, ctx, excs.indentation_error);
         extend_exception!(PyTabError, ctx, excs.tab_error);
 
@@ -1623,6 +1627,28 @@ pub(super) mod types {
         }
     }
 
+    #[pyexception(
+        name = "_IncompleteInputError",
+        base = "PySyntaxError",
+        ctx = "incomplete_input_error"
+    )]
+    #[derive(Debug)]
+    pub struct PyIncompleteInputError {}
+
+    #[pyexception]
+    impl PyIncompleteInputError {
+        #[pyslot]
+        #[pymethod(name = "__init__")]
+        pub(crate) fn slot_init(
+            zelf: PyObjectRef,
+            _args: FuncArgs,
+            vm: &VirtualMachine,
+        ) -> PyResult<()> {
+            zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?;
+            Ok(())
+        }
+    }
+
     #[pyexception(name, base = "PySyntaxError", ctx = "indentation_error", impl)]
     #[derive(Debug)]
     pub struct PyIndentationError {}

vm/src/stdlib/ast.rs

@@ -245,6 +245,7 @@ pub(crate) fn parse(
     let top = parser::parse(source, mode.into())
         .map_err(|parse_error| ParseError {
             error: parse_error.error,
+            raw_location: parse_error.location,
             location: text_range_to_source_range(&source_code, parse_error.location)
                 .start
                 .to_source_location(),
@@ -295,8 +296,8 @@ pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400;
 // The following flags match the values from Include/cpython/compile.h
 // Caveat emptor: These flags are undocumented on purpose and depending
 // on their effect outside the standard library is **unsupported**.
-const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
-const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
+pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
+pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
 
 // __future__ flags - sync with Lib/__future__.py
 // TODO: These flags aren't being used in rust code

vm/src/stdlib/builtins.rs

@@ -186,6 +186,8 @@ mod builtins {
                     return Err(vm.new_value_error("compile() unrecognized flags".to_owned()));
                 }
 
+                let allow_incomplete = !(flags & ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero();
+
                 if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() {
                     #[cfg(not(feature = "compiler"))]
                     {
@@ -207,14 +209,17 @@ mod builtins {
                                 args.filename.to_string_lossy().into_owned(),
                                 opts,
                             )
-                            .map_err(|err| (err, Some(source)).to_pyexception(vm))?;
+                            .map_err(|err| {
+                                (err, Some(source), allow_incomplete).to_pyexception(vm)
+                            })?;
                         Ok(code.into())
                     }
                 } else {
                     let mode = mode_str
                         .parse::<parser::Mode>()
                         .map_err(|err| vm.new_value_error(err.to_string()))?;
-                    ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm))
+                    ast::parse(vm, source, mode)
+                        .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))
                 }
             }
         }
@@ -1056,6 +1061,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) {
         "NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(),
         "RecursionError" => ctx.exceptions.recursion_error.to_owned(),
         "SyntaxError" =>  ctx.exceptions.syntax_error.to_owned(),
+        "_IncompleteInputError" =>  ctx.exceptions.incomplete_input_error.to_owned(),
         "IndentationError" =>  ctx.exceptions.indentation_error.to_owned(),
         "TabError" =>  ctx.exceptions.tab_error.to_owned(),
         "SystemError" => ctx.exceptions.system_error.to_owned(),

vm/src/vm/vm_new.rs

@@ -320,10 +320,11 @@ impl VirtualMachine {
     }
 
     #[cfg(any(feature = "parser", feature = "compiler"))]
-    pub fn new_syntax_error(
+    pub fn new_syntax_error_maybe_incomplete(
         &self,
         error: &crate::compiler::CompileError,
         source: Option<&str>,
+        allow_incomplete: bool,
     ) -> PyBaseExceptionRef {
         use crate::source::SourceLocation;
 
@@ -343,12 +344,102 @@ impl VirtualMachine {
                 ..
             }) => self.ctx.exceptions.indentation_error,
             #[cfg(feature = "parser")]
+            crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
+                error:
+                    ruff_python_parser::ParseErrorType::Lexical(
+                        ruff_python_parser::LexicalErrorType::Eof,
+                    ),
+                ..
+            }) => {
+                if allow_incomplete {
+                    self.ctx.exceptions.incomplete_input_error
+                } else {
+                    self.ctx.exceptions.syntax_error
+                }
+            }
+            #[cfg(feature = "parser")]
+            crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
+                error:
+                    ruff_python_parser::ParseErrorType::Lexical(
+                        ruff_python_parser::LexicalErrorType::FStringError(
+                            ruff_python_parser::FStringErrorType::UnterminatedTripleQuotedString,
+                        ),
+                    ),
+                ..
+            }) => {
+                if allow_incomplete {
+                    self.ctx.exceptions.incomplete_input_error
+                } else {
+                    self.ctx.exceptions.syntax_error
+                }
+            }
+            #[cfg(feature = "parser")]
+            crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
+                error:
+                    ruff_python_parser::ParseErrorType::Lexical(
+                        ruff_python_parser::LexicalErrorType::UnclosedStringError,
+                    ),
+                raw_location,
+                ..
+            }) => {
+                if allow_incomplete {
+                    let mut is_incomplete = false;
+
+                    if let Some(source) = source {
+                        let loc = raw_location.start().to_usize();
+                        let mut iter = source.chars();
+                        if let Some(quote) = iter.nth(loc) {
+                            if iter.next() == Some(quote) && iter.next() == Some(quote) {
+                                is_incomplete = true;
+                            }
+                        }
+                    }
+
+                    if is_incomplete {
+                        self.ctx.exceptions.incomplete_input_error
+                    } else {
+                        self.ctx.exceptions.syntax_error
+                    }
+                } else {
+                    self.ctx.exceptions.syntax_error
+                }
+            }
+            #[cfg(feature = "parser")]
             crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
                 error: ruff_python_parser::ParseErrorType::OtherError(s),
+                raw_location,
                 ..
             }) => {
                 if s.starts_with("Expected an indented block after") {
-                    self.ctx.exceptions.indentation_error
+                    if allow_incomplete {
+                        // Check that all chars in the error are whitespace, if so, the source is
+                        // incomplete. Otherwise, we've found code that might violates
+                        // indentation rules.
+                        let mut is_incomplete = true;
+                        if let Some(source) = source {
+                            let start = raw_location.start().to_usize();
+                            let end = raw_location.end().to_usize();
+                            let mut iter = source.chars();
+                            iter.nth(start);
+                            for _ in start..end {
+                                if let Some(c) = iter.next() {
+                                    if !c.is_ascii_whitespace() {
+                                        is_incomplete = false;
+                                    }
+                                } else {
+                                    break;
+                                }
+                            }
+                        }
+
+                        if is_incomplete {
+                            self.ctx.exceptions.incomplete_input_error
+                        } else {
+                            self.ctx.exceptions.indentation_error
+                        }
+                    } else {
+                        self.ctx.exceptions.indentation_error
+                    }
                 } else {
                     self.ctx.exceptions.syntax_error
                 }
@@ -410,6 +501,15 @@ impl VirtualMachine {
         syntax_error
     }
 
+    #[cfg(any(feature = "parser", feature = "compiler"))]
+    pub fn new_syntax_error(
+        &self,
+        error: &crate::compiler::CompileError,
+        source: Option<&str>,
+    ) -> PyBaseExceptionRef {
+        self.new_syntax_error_maybe_incomplete(error, source, false)
+    }
+
     pub fn new_import_error(&self, msg: String, name: PyStrRef) -> PyBaseExceptionRef {
         let import_error = self.ctx.exceptions.import_error.to_owned();
         let exc = self.new_exception_msg(import_error, msg);