Replace Python parser with ruff parser (#5494)

* stage1

* compiler pass build

* introduce rustpython-compiler-source

* stage2

* fixup

* pass compile

* Fix hello world compiler test

* Fix code generation for if-elif-else statement

* Fix code generation for lambda expression

* Fix code generation for integers

* Fix code generation for fstrings

* Fix code generation for if statement

* Fix code generation for if statement

* Fix code generation for if statement

* Fix code generation for fstring

* Fix code generation for class definition

* Replace feature flags

* Initialize frozen core modules

* Allow __future__ import after module doc comment

* Disable ast module

* Commit remaining fixes for compile errors in examples

* Fix some warnings

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Split ast stdlib module into files

* Fix codegen for positional arguments with defaults

* Update ast stdlib module

* Update ast stdlib module

* Extract string and constant handling from expression.rs

* Always add required fields to AST nodes

* Compile doc strings correctly again

* Enable "ast" Cargo feature by default

* Refactor compilation of big integer literal

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Reset barebones example

* Fix some left-over warnings

* Undo accidential change

* Adapt shell to ruff parser

* Pin parser to v0.4.10

* fix clippy

* Add TODO about interactive mode

* Fix compilation of complex number expression

* Remove moved code

* Update test case to ruff v0.4.10

* Apply suggestion

Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>

* Apply suggestion

Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>

* Fix compilation of fstring expression

* Fix compilation of fstring expression

* Fix wasm compile errors

* Attach correct source locations to ast objects

* Fix some more wasm compile errors

* Consider compile mode and enable AST stdlib module again

* Fix incorrect AST source location end column

* Fix compile error if "compiler" feature is not enabled

* Fix regrtests

* Fix some test_ast tests

* Add source range to type ignore

* Fix incompatibility with Rust 2024 edition

* Fix todos by implementing missing ast conversions and deleting unused code

* Appease clippy

* Fix remaining ast tests

* Fix remaining ast tests

* Mark/fix remaining tests

* Fix more

* Hacky windows fix

---------

Co-authored-by: Kangzhi Shi <shikangzhi@gmail.com>
Co-authored-by: Jeong YunWon <jeong@youknowone.org>
Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>
Co-authored-by: Noa <coolreader18@gmail.com>

examples/dis.rs

@@ -79,7 +79,7 @@ fn display_script(
     expand_code_objects: bool,
 ) -> Result<(), Box<dyn Error>> {
     let source = fs::read_to_string(path)?;
-    let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
+    let code = compiler::compile(&source, mode, &path.to_string_lossy(), opts)?;
     println!("{}:", path.display());
     if expand_code_objects {
         println!("{}", code.display_expand_code_objects());

examples/parse_folder.rs

@@ -8,7 +8,8 @@ extern crate env_logger;
 #[macro_use]
 extern crate log;
 
-use rustpython_parser::{Parse, ast};
+use ruff_python_parser::parse_module;
+use rustpython_compiler::ast;
 use std::{
     path::{Path, PathBuf},
     time::{Duration, Instant},
@@ -68,18 +69,19 @@ fn parse_python_file(filename: &Path) -> ParsedFile {
     info!("Parsing file {:?}", filename);
     match std::fs::read_to_string(filename) {
         Err(e) => ParsedFile {
-            // filename: Box::new(filename.to_path_buf()),
-            // code: "".to_owned(),
+            filename: Box::new(filename.to_path_buf()),
+            code: "".to_owned(),
             num_lines: 0,
             result: Err(e.to_string()),
         },
         Ok(source) => {
             let num_lines = source.lines().count();
-            let result =
-                ast::Suite::parse(&source, &filename.to_string_lossy()).map_err(|e| e.to_string());
+            let result = parse_module(&source)
+                .map(|x| x.into_suite())
+                .map_err(|e| e.to_string());
             ParsedFile {
-                // filename: Box::new(filename.to_path_buf()),
-                // code: source.to_string(),
+                filename: Box::new(filename.to_path_buf()),
+                code: source.to_string(),
                 num_lines,
                 result,
             }
@@ -132,8 +134,8 @@ struct ScanResult {
 }
 
 struct ParsedFile {
-    // filename: Box<PathBuf>,
-    // code: String,
+    filename: Box<PathBuf>,
+    code: String,
     num_lines: usize,
     result: ParseResult,
 }