mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-29 11:14:38 +08:00
Code Activity

Merge pull request #1163 from RustPython/ast-module

Browse Source 
Refactor the lexer code to bring some more sanity and reduce function…
This commit is contained in:
Windel Bouwman authored and GitHub committed 2019-07-21 16:06:02 +02:00
2 parents f31b9dc0f3 e922a6a952
commit c7ffa2e8b0
8 files changed
+555 -517
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

parser/src/error.rs
+6 -7
View File
@@ -1,6 +1,5 @@
//! Define internal parse error types
//! The goal is to provide a matching and a safe error API, maksing errors from LALR
use lalrpop_util::ParseError as InnerError;
use lalrpop_util::ParseError as LalrpopError;
use crate::location::Location;
@@ -106,23 +105,23 @@ pub enum ParseErrorType {
}
/// Convert `lalrpop_util::ParseError` to our internal type
impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
match err {
// TODO: Are there cases where this isn't an EOF?
InnerError::InvalidToken { location } => ParseError {
LalrpopError::InvalidToken { location } => ParseError {
error: ParseErrorType::EOF,
location,
},
InnerError::ExtraToken { token } => ParseError {
LalrpopError::ExtraToken { token } => ParseError {
error: ParseErrorType::ExtraToken(token.1),
location: token.0,
},
InnerError::User { error } => ParseError {
LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error),
location: error.location,
},
InnerError::UnrecognizedToken { token, expected } => {
LalrpopError::UnrecognizedToken { token, expected } => {
match token {
Some(tok) => ParseError {
error: ParseErrorType::UnrecognizedToken(tok.1, expected),
parser/src/lexer.rs
+537 -488
View File
File diff suppressed because it is too large. Load diff
parser/src/parser.rs
+8 -8
View File
@@ -121,13 +121,13 @@ mod tests {
#[test]
fn test_parse_empty() {
let parse_ast = parse_program(&String::from("\n"));
let parse_ast = parse_program(&String::from(""));
assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] }))
}
#[test]
fn test_parse_print_hello() {
let source = String::from("print('Hello world')\n");
let source = String::from("print('Hello world')");
let parse_ast = parse_program(&source).unwrap();
assert_eq!(
parse_ast,
@@ -151,7 +151,7 @@ mod tests {
#[test]
fn test_parse_print_2() {
let source = String::from("print('Hello world', 2)\n");
let source = String::from("print('Hello world', 2)");
let parse_ast = parse_program(&source).unwrap();
assert_eq!(
parse_ast,
@@ -175,7 +175,7 @@ mod tests {
#[test]
fn test_parse_kwargs() {
let source = String::from("my_func('positional', keyword=2)\n");
let source = String::from("my_func('positional', keyword=2)");
let parse_ast = parse_program(&source).unwrap();
assert_eq!(
parse_ast,
@@ -202,7 +202,7 @@ mod tests {
#[test]
fn test_parse_if_elif_else() {
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30\n");
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
let parse_ast = parse_statement(&source).unwrap();
assert_eq!(
parse_ast,
@@ -226,7 +226,7 @@ mod tests {
#[test]
fn test_parse_lambda() {
let source = String::from("lambda x, y: x * y\n"); // lambda(x, y): x * y");
let source = String::from("lambda x, y: x * y"); // lambda(x, y): x * y");
let parse_ast = parse_statement(&source);
assert_eq!(
parse_ast,
@@ -265,7 +265,7 @@ mod tests {
#[test]
fn test_parse_tuples() {
let source = String::from("a, b = 4, 5\n");
let source = String::from("a, b = 4, 5");
assert_eq!(
parse_statement(&source),
@@ -292,7 +292,7 @@ mod tests {
#[test]
fn test_parse_class() {
let source = String::from(
"class Foo(A, B):\n def __init__(self):\n pass\n def method_with_default(self, arg='default'):\n pass\n",
"class Foo(A, B):\n def __init__(self):\n pass\n def method_with_default(self, arg='default'):\n pass",
);
assert_eq!(
parse_statement(&source),
parser/src/python.lalrpop
+1 -1
View File
@@ -21,7 +21,7 @@ grammar;
pub Top: ast::Top = {
StartProgram <p:Program> => ast::Top::Program(p),
StartStatement <s:Statement> => ast::Top::Statement(s),
StartExpression <e:Test> => ast::Top::Expression(e),
StartExpression <e:Test> ("\n")* => ast::Top::Expression(e),
};
Program: ast::Program = {
src/main.rs
+1 -3
View File
@@ -298,11 +298,9 @@ fn handle_exception<T>(vm: &VirtualMachine, result: PyResult<T>) {
}
}
fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult<()> {
fn run_command(vm: &VirtualMachine, source: String) -> PyResult<()> {
debug!("Running command {}", source);
// This works around https://github.com/RustPython/RustPython/issues/17
source.push('\n');
_run_string(vm, &source, "<stdin>".to_string())?;
Ok(())
}
vm/src/builtins.rs
-7
View File
@@ -108,9 +108,6 @@ fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult<PyCodeRef
Either::B(bytes) => str::from_utf8(&bytes).unwrap().to_string(),
};
// TODO: fix this newline bug:
let source = format!("{}\n", source);
let mode = get_compile_mode(vm, &args.mode.value)?;
vm.compile(&source, &mode, args.filename.value.to_string())
@@ -161,8 +158,6 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
} else if objtype::isinstance(source, &vm.ctx.str_type()) {
let mode = compile::Mode::Eval;
let source = objstr::get_value(source);
// TODO: fix this newline bug:
let source = format!("{}\n", source);
vm.compile(&source, &mode, "<string>".to_string())
.map_err(|err| vm.new_syntax_error(&err))?
} else {
@@ -190,8 +185,6 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
let code_obj = if objtype::isinstance(source, &vm.ctx.str_type()) {
let mode = compile::Mode::Exec;
let source = objstr::get_value(source);
// TODO: fix this newline bug:
let source = format!("{}\n", source);
vm.compile(&source, &mode, "<string>".to_string())
.map_err(|err| vm.new_syntax_error(&err))?
} else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) {
vm/src/eval.rs
+1 -1
View File
@@ -31,7 +31,7 @@ mod tests {
#[test]
fn test_print_42() {
let source = String::from("print('Hello world')\n");
let source = String::from("print('Hello world')");
let vm: VirtualMachine = Default::default();
let vars = vm.new_scope_with_builtins();
let _result = eval(&vm, &source, vars, "<unittest>");
wasm/lib/src/vm_class.rs
+1 -2
View File
@@ -271,13 +271,12 @@ impl WASMVirtualMachine {
})?
}
fn run(&self, mut source: String, mode: compile::Mode) -> Result<JsValue, JsValue> {
fn run(&self, source: String, mode: compile::Mode) -> Result<JsValue, JsValue> {
self.assert_valid()?;
self.with_unchecked(
|StoredVirtualMachine {
ref vm, ref scope, ..
}| {
source.push('\n');
let code = vm.compile(&source, &mode, "<wasm>".to_string());
let code = code.map_err(|err| {
let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));