Add source_path to ParseError

benches/execution.rs

@@ -51,7 +51,7 @@ pub fn benchmark_file_execution(
 pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
     group.throughput(Throughput::Bytes(contents.len() as u64));
     group.bench_function(BenchmarkId::new("rustpython", name), |b| {
-        b.iter(|| parse_program(contents).unwrap())
+        b.iter(|| parse_program(contents, name).unwrap())
     });
     group.bench_function(BenchmarkId::new("cpython", name), |b| {
         let gil = cpython::Python::acquire_gil();

compiler/codegen/src/compile.rs

@@ -2707,7 +2707,7 @@ mod tests {
             "source_path".to_owned(),
             "<module>".to_owned(),
         );
-        let ast = parser::parse_program(source).unwrap();
+        let ast = parser::parse_program(source, "<test>").unwrap();
         let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
         compiler.pop_code_object()

compiler/parser/src/error.rs

@@ -124,6 +124,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 pub struct ParseError {
     pub error: ParseErrorType,
     pub location: Location,
+    pub source_path: String,
 }
 
 #[derive(Debug, PartialEq)]
@@ -141,21 +142,28 @@ pub enum ParseErrorType {
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
-    fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
+impl ParseError {
+    pub(crate) fn from_lalrpop(
+        err: LalrpopError<Location, Tok, LexicalError>,
+        source_path: &str,
+    ) -> Self {
+        let source_path = source_path.to_owned();
         match err {
             // TODO: Are there cases where this isn't an EOF?
             LalrpopError::InvalidToken { location } => ParseError {
                 error: ParseErrorType::Eof,
                 location,
+                source_path,
             },
             LalrpopError::ExtraToken { token } => ParseError {
                 error: ParseErrorType::ExtraToken(token.1),
                 location: token.0,
+                source_path,
             },
             LalrpopError::User { error } => ParseError {
                 error: ParseErrorType::Lexical(error.error),
                 location: error.location,
+                source_path,
             },
             LalrpopError::UnrecognizedToken { token, expected } => {
                 // Hacky, but it's how CPython does it. See PyParser_AddToken,
@@ -164,11 +172,13 @@ impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
                 ParseError {
                     error: ParseErrorType::UnrecognizedToken(token.1, expected),
                     location: token.0,
+                    source_path,
                 }
             }
             LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
                 error: ParseErrorType::Eof,
                 location,
+                source_path,
             },
         }
     }

compiler/parser/src/fstring.rs

@@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> {
 
 fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
     let fstring_body = format!("({})", source);
-    parse_expression(&fstring_body)
+    parse_expression(&fstring_body, "<fstring>")
 }
 
 /// Parse an fstring from a string, located at a certain position in the sourcecode.

compiler/parser/src/lib.rs

@@ -11,7 +11,7 @@
 //! use rustpython_parser::{parser, ast};
 //!
 //! let python_source = "print('Hello world')";
-//! let python_ast = parser::parse_expression(python_source).unwrap();
+//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
 //!
 //! ```
 

compiler/parser/src/parser.rs

@@ -20,8 +20,8 @@ use crate::python;
  */
 
 /// Parse a full python program, containing usually multiple lines.
-pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
-    parse(source, Mode::Module).map(|top| match top {
+pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
+    parse(source, Mode::Module, source_path).map(|top| match top {
         ast::Mod::Module { body, .. } => body,
         _ => unreachable!(),
     })
@@ -33,7 +33,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// ```
 /// extern crate num_bigint;
 /// use rustpython_parser::{parser, ast};
-/// let expr = parser::parse_expression("1 + 2").unwrap();
+/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
 ///
 /// assert_eq!(
 ///     expr,
@@ -63,22 +63,22 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// );
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
-    parse(source, Mode::Expression).map(|top| match top {
+pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
+    parse(source, Mode::Expression, path).map(|top| match top {
         ast::Mod::Expression { body } => *body,
         _ => unreachable!(),
     })
 }
 
 // Parse a given source code
-pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
+pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer(source);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
     python::TopParser::new()
         .parse(tokenizer)
-        .map_err(ParseError::from)
+        .map_err(|e| ParseError::from_lalrpop(e, source_path))
 }
 
 #[cfg(test)]
@@ -87,56 +87,56 @@ mod tests {
 
     #[test]
     fn test_parse_empty() {
-        let parse_ast = parse_program("").unwrap();
+        let parse_ast = parse_program("", "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_string() {
         let source = String::from("'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string() {
         let source = String::from("f'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_hello() {
         let source = String::from("print('Hello world')");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_2() {
         let source = String::from("print('Hello world', 2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_kwargs() {
         let source = String::from("my_func('positional', keyword=2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_elif_else() {
         let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_lambda() {
         let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
-        let parse_ast = parse_program(source).unwrap();
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
@@ -144,7 +144,7 @@ mod tests {
     fn test_parse_tuples() {
         let source = "a, b = 4, 5";
 
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
@@ -155,48 +155,48 @@ class Foo(A, B):
   pass
  def method_with_default(self, arg='default'):
   pass";
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
     fn test_parse_dict_comprehension() {
         let source = String::from("{x1: x2 for y in z}");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_list_comprehension() {
         let source = String::from("[x for y in z]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_double_list_comprehension() {
         let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_generator_comprehension() {
         let source = String::from("(x for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_named_expression_generator_comprehension() {
         let source = String::from("(x := y + 1 for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_else_generator_comprehension() {
         let source = String::from("(x if y else y for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }

compiler/parser/src/string.rs

@@ -88,56 +88,56 @@ mod tests {
     #[test]
     fn test_parse_string_concat() {
         let source = String::from("'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_1() {
         let source = String::from("'Hello ' u'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_2() {
         let source = String::from("u'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_1() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_2() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_3() {
         let source = String::from("'Hello ' f'world{\"!\"}'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_1() {
         let source = String::from("u'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_2() {
         let source = String::from("u'Hello ' f'world' '!'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }

compiler/src/lib.rs

@@ -43,22 +43,18 @@ impl fmt::Display for CompileError {
 
 impl CompileError {
     fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
-        CompileError {
+        Self {
             error: error.error.into(),
             location: error.location,
             source_path: error.source_path,
             statement: get_statement(source, error.location),
         }
     }
-    fn from_parse(
-        error: rustpython_parser::error::ParseError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        CompileError {
+    fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
+        Self {
             error: error.error.into(),
             location: error.location,
-            source_path,
+            source_path: error.source_path,
             statement: get_statement(source, error.location),
         }
     }
@@ -83,9 +79,9 @@ pub fn compile(
         compile::Mode::Eval => parser::Mode::Expression,
         compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
     };
-    let mut ast = match parser::parse(source, parser_mode) {
+    let mut ast = match parser::parse(source, parser_mode, &source_path) {
         Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+        Err(e) => return Err(CompileError::from_parse(e, source)),
     };
     if opts.optimize > 0 {
         ast = ConstantOptimizer::new()
@@ -101,14 +97,14 @@ pub fn compile_symtable(
     mode: compile::Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
+    let parse_err = |e| CompileError::from_parse(e, source);
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = parser::parse_program(source).map_err(parse_err)?;
+            let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
             symboltable::SymbolTable::scan_program(&ast)
         }
         compile::Mode::Eval => {
-            let expr = parser::parse_expression(source).map_err(parse_err)?;
+            let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
             symboltable::SymbolTable::scan_expr(&expr)
         }
     };

examples/parse_folder.rs

@@ -84,7 +84,8 @@ fn parse_python_file(filename: &Path) -> ParsedFile {
         },
         Ok(source) => {
             let num_lines = source.to_string().lines().count();
-            let result = parser::parse_program(&source).map_err(|e| e.to_string());
+            let result = parser::parse_program(&source, &filename.to_string_lossy())
+                .map_err(|e| e.to_string());
             ParsedFile {
                 // filename: Box::new(filename.to_path_buf()),
                 // code: source.to_string(),

vm/src/stdlib/ast.rs

@@ -265,7 +265,8 @@ impl Node for ast::ConversionFlag {
 #[cfg(feature = "rustpython-parser")]
 pub(crate) fn parse(vm: &VirtualMachine, source: &str, mode: parser::Mode) -> PyResult {
     // TODO: use vm.new_syntax_error()
-    let top = parser::parse(source, mode).map_err(|err| vm.new_value_error(format!("{}", err)))?;
+    let top = parser::parse(source, mode, "<unknown>")
+        .map_err(|err| vm.new_value_error(format!("{}", err)))?;
     Ok(top.ast_to_object(vm))
 }