mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Merge pull request #2320 from RustPython/coolreader18/split-ast-parser

Browse Source 
Split the ast from the parser, remove compiler dep on parser
This commit is contained in:
Jeong YunWon authored and GitHub committed 2020-11-08 20:49:47 +09:00
2 parents 17e0e75622 70907babe6
commit f390f6157c
38 files changed
+446 -321
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+19 -1
View File
@@ -1674,6 +1674,13 @@ dependencies = [
"rustyline",
]
[[package]]
name = "rustpython-ast"
version = "0.1.0"
dependencies = [
"num-bigint",
]
[[package]]
name = "rustpython-bytecode"
version = "0.1.2"
@@ -1710,6 +1717,16 @@ dependencies = [
[[package]]
name = "rustpython-compiler"
version = "0.1.2"
dependencies = [
"rustpython-bytecode",
"rustpython-compiler-core",
"rustpython-parser",
"thiserror",
]
[[package]]
name = "rustpython-compiler-core"
version = "0.1.2"
dependencies = [
"arrayvec",
"indexmap",
@@ -1717,6 +1734,7 @@ dependencies = [
"itertools",
"log",
"num-complex",
"rustpython-ast",
"rustpython-bytecode",
"rustpython-parser",
]
@@ -1761,6 +1779,7 @@ dependencies = [
"log",
"num-bigint",
"num-traits",
"rustpython-ast",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
@@ -1874,7 +1893,6 @@ dependencies = [
"js-sys",
"parking_lot",
"rustpython-common",
"rustpython-compiler",
"rustpython-parser",
"rustpython-vm",
"serde",
Cargo.toml
+5 -2
View File
@@ -9,7 +9,10 @@ license = "MIT"
include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
[workspace]
members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common", "jit"]
members = [
".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
"derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib",
]
[[bench]]
name = "bench"
@@ -28,7 +31,7 @@ ssl = ["rustpython-vm/ssl"]
log = "0.4"
env_logger = "0.7"
clap = "2.33"
rustpython-compiler = { path = "compiler", version = "0.1.1" }
rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
rustpython-parser = { path = "parser", version = "0.1.1" }
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true }
ast/Cargo.toml
+8
View File
@@ -0,0 +1,8 @@
[package]
name = "rustpython-ast"
version = "0.1.0"
authors = ["RustPython Team"]
edition = "2018"
[dependencies]
num-bigint = "0.3"
parser/src/ast.rs → ast/src/ast.rs
View File
File renamed without changes.
ast/src/lib.rs
+5
View File
@@ -0,0 +1,5 @@
mod ast;
mod location;
pub use ast::*;
pub use location::Location;
parser/src/location.rs → ast/src/location.rs
+27 -3
View File
@@ -16,9 +16,33 @@ impl fmt::Display for Location {
}
impl Location {
pub fn visualize(&self, line: &str, desc: &str) -> String {
// desc.to_owned()
format!("{}\n{}\n{}↑", desc, line, " ".repeat(self.column - 1))
pub fn visualize<'a>(
&self,
line: &'a str,
desc: impl fmt::Display + 'a,
) -> impl fmt::Display + 'a {
struct Visualize<'a, D: fmt::Display> {
loc: Location,
line: &'a str,
desc: D,
}
impl<D: fmt::Display> fmt::Display for Visualize<'_, D> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"{}\n{}\n{arrow:>pad$}",
self.desc,
self.line,
pad = self.loc.column,
arrow = "↑",
)
}
}
Visualize {
loc: *self,
line,
desc,
}
}
}
compiler/Cargo.toml
+3 -2
View File
@@ -1,5 +1,5 @@
[package]
name = "rustpython-compiler"
name = "rustpython-compiler-core"
version = "0.1.2"
description = "Compiler for python code into bytecode for the rustpython VM."
authors = ["RustPython Team"]
@@ -11,10 +11,11 @@ edition = "2018"
indexmap = "1.0"
itertools = "0.9"
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
rustpython-parser = { path = "../parser", version = "0.1.1" }
rustpython-ast = { path = "../ast" }
num-complex = { version = "0.3", features = ["serde"] }
log = "0.4"
arrayvec = "0.5"
[dev-dependencies]
rustpython-parser = { path = "../parser" }
insta = "1.1"
compiler/porcelain/Cargo.toml
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "rustpython-compiler"
version = "0.1.2"
description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
authors = ["RustPython Team"]
edition = "2018"
[dependencies]
thiserror = "1.0"
rustpython-compiler-core = { path = ".." }
rustpython-parser = { path = "../../parser" }
rustpython-bytecode = { path = "../../bytecode" }
compiler/porcelain/src/lib.rs
+135
View File
@@ -0,0 +1,135 @@
use rustpython_bytecode::bytecode::CodeObject;
use rustpython_compiler_core::{compile, symboltable};
use rustpython_parser::{ast::Location, parser};
use std::fmt;
pub use compile::{CompileOpts, Mode};
pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
#[derive(Debug, thiserror::Error)]
pub enum CompileErrorType {
#[error(transparent)]
Compile(#[from] rustpython_compiler_core::error::CompileErrorType),
#[error(transparent)]
Parse(#[from] rustpython_parser::error::ParseErrorType),
}
#[derive(Debug, thiserror::Error)]
pub struct CompileError {
pub error: CompileErrorType,
pub statement: Option<String>,
pub source_path: String,
pub location: Location,
}
impl fmt::Display for CompileError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let loc = self.location;
if let Some(ref stmt) = self.statement {
// visualize the error when location and statement are provided
write!(
f,
"{}",
loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
)
} else {
write!(f, "{} at {}", self.error, loc)
}
}
}
impl CompileError {
fn from_compile(error: rustpython_compiler_core::error::CompileError, source: &str) -> Self {
CompileError {
error: error.error.into(),
location: error.location,
source_path: error.source_path,
statement: get_statement(source, error.location),
}
}
fn from_parse(
error: rustpython_parser::error::ParseError,
source: &str,
source_path: String,
) -> Self {
CompileError {
error: error.error.into(),
location: error.location,
source_path,
statement: get_statement(source, error.location),
}
}
fn from_symtable(
error: symboltable::SymbolTableError,
source: &str,
source_path: String,
) -> Self {
Self::from_compile(error.into_compile_error(source_path), source)
}
}
/// Compile a given sourcecode into a bytecode object.
pub fn compile(
source: &str,
mode: compile::Mode,
source_path: String,
opts: CompileOpts,
) -> Result<CodeObject, CompileError> {
macro_rules! try_parse {
($x:expr) => {
match $x {
Ok(x) => x,
Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
}
};
}
let res = match mode {
compile::Mode::Exec => {
let ast = try_parse!(parser::parse_program(source));
compile::compile_program(ast, source_path, opts)
}
compile::Mode::Eval => {
let statement = try_parse!(parser::parse_statement(source));
compile::compile_statement_eval(statement, source_path, opts)
}
compile::Mode::Single => {
let ast = try_parse!(parser::parse_program(source));
compile::compile_program_single(ast, source_path, opts)
}
};
res.map_err(|e| CompileError::from_compile(e, source))
}
pub fn compile_symtable(
source: &str,
mode: compile::Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
macro_rules! try_parse {
($x:expr) => {
match $x {
Ok(x) => x,
Err(e) => return Err(CompileError::from_parse(e, source, source_path.to_owned())),
}
};
}
let res = match mode {
compile::Mode::Exec | compile::Mode::Single => {
let ast = try_parse!(parser::parse_program(source));
symboltable::make_symbol_table(&ast)
}
compile::Mode::Eval => {
let statement = try_parse!(parser::parse_statement(source));
symboltable::statements_to_symbol_table(&statement)
}
};
res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
}
fn get_statement(source: &str, loc: Location) -> Option<String> {
if loc.column() == 0 || loc.row() == 0 {
return None;
}
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
Some(line + "\n")
}
compiler/src/compile.rs
+13 -33
View File
@@ -12,8 +12,8 @@ use crate::symboltable::{
};
use itertools::Itertools;
use num_complex::Complex64;
use rustpython_ast as ast;
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
use rustpython_parser::{ast, parser};
type CompileResult<T> = Result<T, CompileError>;
@@ -61,31 +61,6 @@ impl CompileContext {
}
}
/// Compile a given sourcecode into a bytecode object.
pub fn compile(
source: &str,
mode: Mode,
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
let to_compile_error =
|parse_error| CompileError::from_parse_error(parse_error, source_path.clone());
match mode {
Mode::Exec => {
let ast = parser::parse_program(source).map_err(to_compile_error)?;
compile_program(ast, source_path, opts)
}
Mode::Eval => {
let statement = parser::parse_statement(source).map_err(to_compile_error)?;
compile_statement_eval(statement, source_path, opts)
}
Mode::Single => {
let ast = parser::parse_program(source).map_err(to_compile_error)?;
compile_program_single(ast, source_path, opts)
}
}
}
/// A helper function for the shared code of the different compile functions
fn with_compiler(
source_path: String,
@@ -106,8 +81,10 @@ pub fn compile_program(
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
let symbol_table = make_symbol_table(&ast)
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
let symbol_table = match make_symbol_table(&ast) {
Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| {
compiler.compile_program(&ast, symbol_table)
})
@@ -119,8 +96,10 @@ pub fn compile_statement_eval(
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
let symbol_table = statements_to_symbol_table(&statement)
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
let symbol_table = match statements_to_symbol_table(&statement) {
Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| {
compiler.compile_statement_eval(&statement, symbol_table)
})
@@ -132,8 +111,10 @@ pub fn compile_program_single(
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
let symbol_table = make_symbol_table(&ast)
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
let symbol_table = match make_symbol_table(&ast) {
Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| {
compiler.compile_program_single(&ast, symbol_table)
})
@@ -165,7 +146,6 @@ impl Compiler {
error,
location,
source_path: self.source_path.clone(),
statement: None,
}
}
compiler/src/error.rs
+31 -81
View File
@@ -1,34 +1,17 @@
use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType};
use rustpython_parser::location::Location;
use rustpython_parser::token::Tok;
use rustpython_ast::Location;
use std::error::Error;
use std::fmt;
#[derive(Debug)]
pub struct CompileError {
pub statement: Option<String>,
pub error: CompileErrorType,
pub location: Location,
pub source_path: String,
}
impl CompileError {
pub fn from_parse_error(parse_error: ParseError, source_path: String) -> Self {
Self {
statement: None,
error: CompileErrorType::Parse(parse_error.error),
location: parse_error.location,
source_path,
}
}
pub fn update_statement_info(&mut self, statement: String) {
self.statement = Some(statement);
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum CompileErrorType {
/// Invalid assignment, cannot store value in target.
Assign(&'static str),
@@ -36,8 +19,6 @@ pub enum CompileErrorType {
Delete(&'static str),
/// Expected an expression got a statement
ExpectExpr,
/// Parser error
Parse(ParseErrorType),
SyntaxError(String),
/// Multiple `*` detected
MultipleStarArgs,
@@ -57,74 +38,43 @@ pub enum CompileErrorType {
InvalidFutureFeature(String),
}
impl CompileError {
pub fn is_indentation_error(&self) -> bool {
if let CompileErrorType::Parse(parse) = &self.error {
match parse {
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
ParseErrorType::UnrecognizedToken(token, expected) => {
*token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
}
_ => false,
impl fmt::Display for CompileErrorType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
CompileErrorType::MultipleStarArgs => {
write!(f, "two starred expressions in assignment")
}
} else {
false
}
}
pub fn is_tab_error(&self) -> bool {
if let CompileErrorType::Parse(parse) = &self.error {
if let ParseErrorType::Lexical(lex) = parse {
if let LexicalErrorType::TabError = lex {
return true;
}
CompileErrorType::InvalidStarExpr => write!(f, "can't use starred expression here"),
CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
CompileErrorType::AsyncReturnValue => {
write!(f, "'return' with value inside async generator")
}
CompileErrorType::InvalidFuturePlacement => write!(
f,
"from __future__ imports must occur at the beginning of the file"
),
CompileErrorType::InvalidFutureFeature(feat) => {
write!(f, "future feature {} is not defined", feat)
}
}
false
}
}
impl Error for CompileErrorType {}
impl fmt::Display for CompileError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let error_desc = match &self.error {
CompileErrorType::Assign(target) => format!("can't assign to {}", target),
CompileErrorType::Delete(target) => format!("can't delete {}", target),
CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
CompileErrorType::Parse(err) => err.to_string(),
CompileErrorType::SyntaxError(err) => err.to_string(),
CompileErrorType::MultipleStarArgs => {
"two starred expressions in assignment".to_owned()
}
CompileErrorType::InvalidStarExpr => "can't use starred expression here".to_owned(),
CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
CompileErrorType::AsyncReturnValue => {
"'return' with value inside async generator".to_owned()
}
CompileErrorType::InvalidFuturePlacement => {
"from __future__ imports must occur at the beginning of the file".to_owned()
}
CompileErrorType::InvalidFutureFeature(feat) => {
format!("future feature {} is not defined", feat)
}
};
if let Some(statement) = &self.statement {
if self.location.column() > 0 {
if let Some(line) = statement.lines().nth(self.location.row() - 1) {
// visualize the error, when location and statement are provided
return write!(f, "{}", self.location.visualize(line, &error_desc));
}
}
}
// print line number
write!(f, "{} at {}", error_desc, self.location)
write!(f, "{} at {}", self.error, self.location)
}
}
compiler/src/mode.rs
-11
View File
@@ -1,5 +1,3 @@
use rustpython_parser::parser;
#[derive(Clone, Copy)]
pub enum Mode {
Exec,
@@ -19,15 +17,6 @@ impl std::str::FromStr for Mode {
}
}
impl Mode {
pub fn to_parser_mode(self) -> parser::Mode {
match self {
Mode::Exec | Mode::Single => parser::Mode::Program,
Mode::Eval => parser::Mode::Statement,
}
}
}
#[derive(Debug)]
pub struct ModeParseError {
_priv: (),
compiler/src/symboltable.rs
+107 -99
View File
@@ -9,12 +9,11 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
use crate::error::{CompileError, CompileErrorType};
use indexmap::map::IndexMap;
use rustpython_parser::ast;
use rustpython_parser::location::Location;
use rustpython_ast::{self as ast, Location};
use std::fmt;
pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default();
let mut builder = SymbolTableBuilder::default();
builder.prepare();
builder.scan_program(program)?;
builder.finish()
@@ -23,7 +22,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTa
pub fn statements_to_symbol_table(
statements: &[ast::Statement],
) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default();
let mut builder = SymbolTableBuilder::default();
builder.prepare();
builder.scan_statements(statements)?;
builder.finish()
@@ -59,7 +58,7 @@ impl SymbolTable {
typ,
line_number,
is_nested,
symbols: Default::default(),
symbols: IndexMap::new(),
sub_tables: vec![],
}
}
@@ -150,12 +149,11 @@ pub struct SymbolTableError {
location: Location,
}
impl CompileError {
pub fn from_symbol_table_error(error: SymbolTableError, source_path: String) -> Self {
impl SymbolTableError {
pub fn into_compile_error(self, source_path: String) -> CompileError {
CompileError {
statement: None,
error: CompileErrorType::SyntaxError(error.error),
location: error.location,
error: CompileErrorType::SyntaxError(self.error),
location: self.location,
source_path,
}
}
@@ -238,7 +236,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
});
}
} else {
@@ -247,7 +246,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
"nonlocal {} defined at place without an enclosing scope",
symbol.name
),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
});
}
}
@@ -312,7 +312,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
"assignment expression cannot rebind comprehension iteration variable {}",
symbol.name
),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
});
}
@@ -324,7 +325,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
// named expressions are forbidden in comprehensions on class scope
return Err(SymbolTableError {
error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
} );
}
SymbolTableType::Function => {
@@ -356,7 +358,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
if parent_symbol.is_iter {
return Err(SymbolTableError {
error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
});
}
@@ -405,6 +408,7 @@ struct SymbolTableBuilder {
/// was used.
/// In cpython this is stored in the AST, but I think this
/// is not logical, since it is not context free.
#[derive(Copy, Clone)]
enum ExpressionContext {
Load,
Store,
@@ -466,7 +470,7 @@ impl SymbolTableBuilder {
} else {
SymbolUsage::Parameter
};
self.register_name(&parameter.arg, usage)
self.register_name(&parameter.arg, usage, parameter.location)
}
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -478,22 +482,23 @@ impl SymbolTableBuilder {
fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
if let Some(annotation) = &parameter.annotation {
self.scan_expression(&annotation, &ExpressionContext::Load)?;
self.scan_expression(&annotation, ExpressionContext::Load)?;
}
Ok(())
}
fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
use ast::StatementType::*;
let location = statement.location;
match &statement.node {
Global { names } => {
for name in names {
self.register_name(name, SymbolUsage::Global)?;
self.register_name(name, SymbolUsage::Global, location)?;
}
}
Nonlocal { names } => {
for name in names {
self.register_name(name, SymbolUsage::Nonlocal)?;
self.register_name(name, SymbolUsage::Nonlocal, location)?;
}
}
FunctionDef {
@@ -504,12 +509,12 @@ impl SymbolTableBuilder {
returns,
..
} => {
self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned)?;
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
if let Some(expression) = returns {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
self.enter_function(name, args, statement.location.row())?;
self.enter_function(name, args, location.row())?;
self.scan_statements(body)?;
self.leave_scope();
}
@@ -520,24 +525,24 @@ impl SymbolTableBuilder {
keywords,
decorator_list,
} => {
self.enter_scope(name, SymbolTableType::Class, statement.location.row());
self.register_name("__module__", SymbolUsage::Assigned)?;
self.register_name("__qualname__", SymbolUsage::Assigned)?;
self.register_name("__doc__", SymbolUsage::Assigned)?;
self.enter_scope(name, SymbolTableType::Class, location.row());
self.register_name("__module__", SymbolUsage::Assigned, location)?;
self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
self.register_name("__doc__", SymbolUsage::Assigned, location)?;
self.scan_statements(body)?;
self.leave_scope();
self.scan_expressions(bases, &ExpressionContext::Load)?;
self.scan_expressions(bases, ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
self.scan_expression(&keyword.value, ExpressionContext::Load)?;
}
self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned)?;
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
}
Expression { expression } => {
self.scan_expression(expression, &ExpressionContext::Load)?
self.scan_expression(expression, ExpressionContext::Load)?
}
If { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(test, ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
@@ -550,15 +555,15 @@ impl SymbolTableBuilder {
orelse,
..
} => {
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(iter, &ExpressionContext::Load)?;
self.scan_expression(target, ExpressionContext::Store)?;
self.scan_expression(iter, ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
}
}
While { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(test, ExpressionContext::Load)?;
self.scan_statements(body)?;
if let Some(code) = orelse {
self.scan_statements(code)?;
@@ -571,37 +576,38 @@ impl SymbolTableBuilder {
for name in names {
if let Some(alias) = &name.alias {
// `import mymodule as myalias`
self.register_name(alias, SymbolUsage::Imported)?;
self.register_name(alias, SymbolUsage::Imported, location)?;
} else {
// `import module`
self.register_name(
name.symbol.split('.').next().unwrap(),
SymbolUsage::Imported,
location,
)?;
}
}
}
Return { value } => {
if let Some(expression) = value {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
}
Assert { test, msg } => {
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(test, ExpressionContext::Load)?;
if let Some(expression) = msg {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
}
Delete { targets } => {
self.scan_expressions(targets, &ExpressionContext::Delete)?;
self.scan_expressions(targets, ExpressionContext::Delete)?;
}
Assign { targets, value } => {
self.scan_expressions(targets, &ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?;
self.scan_expressions(targets, ExpressionContext::Store)?;
self.scan_expression(value, ExpressionContext::Load)?;
}
AugAssign { target, value, .. } => {
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?;
self.scan_expression(target, ExpressionContext::Store)?;
self.scan_expression(value, ExpressionContext::Load)?;
}
AnnAssign {
target,
@@ -610,20 +616,20 @@ impl SymbolTableBuilder {
} => {
// https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
if let ast::ExpressionType::Identifier { ref name } = target.node {
self.register_name(name, SymbolUsage::AnnotationAssigned)?;
self.register_name(name, SymbolUsage::AnnotationAssigned, location)?;
} else {
self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(target, ExpressionContext::Store)?;
}
self.scan_expression(annotation, &ExpressionContext::Load)?;
self.scan_expression(annotation, ExpressionContext::Load)?;
if let Some(value) = value {
self.scan_expression(value, &ExpressionContext::Load)?;
self.scan_expression(value, ExpressionContext::Load)?;
}
}
With { items, body, .. } => {
for item in items {
self.scan_expression(&item.context_expr, &ExpressionContext::Load)?;
self.scan_expression(&item.context_expr, ExpressionContext::Load)?;
if let Some(expression) = &item.optional_vars {
self.scan_expression(expression, &ExpressionContext::Store)?;
self.scan_expression(expression, ExpressionContext::Store)?;
}
}
self.scan_statements(body)?;
@@ -637,10 +643,10 @@ impl SymbolTableBuilder {
self.scan_statements(body)?;
for handler in handlers {
if let Some(expression) = &handler.typ {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
if let Some(name) = &handler.name {
self.register_name(name, SymbolUsage::Assigned)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
}
self.scan_statements(&handler.body)?;
}
@@ -653,10 +659,10 @@ impl SymbolTableBuilder {
}
Raise { exception, cause } => {
if let Some(expression) = exception {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
if let Some(expression) = cause {
self.scan_expression(expression, &ExpressionContext::Load)?;
self.scan_expression(expression, ExpressionContext::Load)?;
}
}
}
@@ -666,7 +672,7 @@ impl SymbolTableBuilder {
fn scan_expressions(
&mut self,
expressions: &[ast::Expression],
context: &ExpressionContext,
context: ExpressionContext,
) -> SymbolTableResult {
for expression in expressions {
self.scan_expression(expression, context)?;
@@ -677,9 +683,10 @@ impl SymbolTableBuilder {
fn scan_expression(
&mut self,
expression: &ast::Expression,
context: &ExpressionContext,
context: ExpressionContext,
) -> SymbolTableResult {
use ast::ExpressionType::*;
let location = expression.location;
match &expression.node {
Binop { a, b, .. } => {
self.scan_expression(a, context)?;
@@ -692,11 +699,11 @@ impl SymbolTableBuilder {
self.scan_expressions(vals, context)?;
}
Subscript { a, b } => {
self.scan_expression(a, &ExpressionContext::Load)?;
self.scan_expression(b, &ExpressionContext::Load)?;
self.scan_expression(a, ExpressionContext::Load)?;
self.scan_expression(b, ExpressionContext::Load)?;
}
Attribute { value, .. } => {
self.scan_expression(value, &ExpressionContext::Load)?;
self.scan_expression(value, ExpressionContext::Load)?;
}
Dict { elements } => {
for (key, value) in elements {
@@ -740,41 +747,37 @@ impl SymbolTableBuilder {
ast::ComprehensionKind::Dict { .. } => "dictcomp",
};
self.enter_scope(
scope_name,
SymbolTableType::Comprehension,
expression.location.row(),
);
self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
// Register the passed argument to the generator function as the name ".0"
self.register_name(".0", SymbolUsage::Parameter)?;
self.register_name(".0", SymbolUsage::Parameter, location)?;
match **kind {
ast::ComprehensionKind::GeneratorExpression { ref element }
| ast::ComprehensionKind::List { ref element }
| ast::ComprehensionKind::Set { ref element } => {
self.scan_expression(element, &ExpressionContext::Load)?;
self.scan_expression(element, ExpressionContext::Load)?;
}
ast::ComprehensionKind::Dict { ref key, ref value } => {
self.scan_expression(&key, &ExpressionContext::Load)?;
self.scan_expression(&value, &ExpressionContext::Load)?;
self.scan_expression(&key, ExpressionContext::Load)?;
self.scan_expression(&value, ExpressionContext::Load)?;
}
}
let mut is_first_generator = true;
for generator in generators {
self.scan_expression(&generator.target, &ExpressionContext::Iter)?;
self.scan_expression(&generator.target, ExpressionContext::Iter)?;
if is_first_generator {
is_first_generator = false;
} else {
self.scan_expression(
&generator.iter,
&ExpressionContext::IterDefinitionExp,
ExpressionContext::IterDefinitionExp,
)?;
}
for if_expr in &generator.ifs {
self.scan_expression(if_expr, &ExpressionContext::Load)?;
self.scan_expression(if_expr, ExpressionContext::Load)?;
}
}
@@ -782,25 +785,25 @@ impl SymbolTableBuilder {
// The first iterable is passed as an argument into the created function:
assert!(!generators.is_empty());
self.scan_expression(&generators[0].iter, &ExpressionContext::IterDefinitionExp)?;
self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?;
}
Call {
function,
args,
keywords,
} => {
match *context {
match context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(function, &ExpressionContext::IterDefinitionExp)?;
self.scan_expression(function, ExpressionContext::IterDefinitionExp)?;
}
_ => {
self.scan_expression(function, &ExpressionContext::Load)?;
self.scan_expression(function, ExpressionContext::Load)?;
}
}
self.scan_expressions(args, &ExpressionContext::Load)?;
self.scan_expressions(args, ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
self.scan_expression(&keyword.value, ExpressionContext::Load)?;
}
}
String { value } => {
@@ -810,48 +813,49 @@ impl SymbolTableBuilder {
// Determine the contextual usage of this symbol:
match context {
ExpressionContext::Delete => {
self.register_name(name, SymbolUsage::Used)?;
self.register_name(name, SymbolUsage::Used, location)?;
}
ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
self.register_name(name, SymbolUsage::Used)?;
self.register_name(name, SymbolUsage::Used, location)?;
}
ExpressionContext::Store => {
self.register_name(name, SymbolUsage::Assigned)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
}
ExpressionContext::Iter => {
self.register_name(name, SymbolUsage::Iter)?;
self.register_name(name, SymbolUsage::Iter, location)?;
}
}
}
Lambda { args, body } => {
self.enter_function("lambda", args, expression.location.row())?;
match *context {
match context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, &ExpressionContext::IterDefinitionExp)?;
self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
}
_ => {
self.scan_expression(body, &ExpressionContext::Load)?;
self.scan_expression(body, ExpressionContext::Load)?;
}
}
self.leave_scope();
}
IfExpression { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(body, &ExpressionContext::Load)?;
self.scan_expression(orelse, &ExpressionContext::Load)?;
self.scan_expression(test, ExpressionContext::Load)?;
self.scan_expression(body, ExpressionContext::Load)?;
self.scan_expression(orelse, ExpressionContext::Load)?;
}
NamedExpression { left, right } => {
// named expressions are not allowed in the definiton of
// comprehension iterator definitions
if let ExpressionContext::IterDefinitionExp = *context {
if let ExpressionContext::IterDefinitionExp = context {
return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
location: Default::default(),
// TODO: accurate location info, somehow
location: Location::default(),
});
}
self.scan_expression(right, &ExpressionContext::Load)?;
self.scan_expression(right, ExpressionContext::Load)?;
// special handling for assigned identifier in named expressions
// that are used in comprehensions. This required to correctly
@@ -860,15 +864,15 @@ impl SymbolTableBuilder {
if let Identifier { name } = &left.node {
let table = self.tables.last().unwrap();
if table.typ == SymbolTableType::Comprehension {
self.register_name(name, SymbolUsage::AssignedNamedExprInCompr)?;
self.register_name(name, SymbolUsage::AssignedNamedExprInCompr, location)?;
} else {
// omit one recursion. When the handling of an store changes for
// Identifiers this needs adapted - more forward safe would be
// calling scan_expression directly.
self.register_name(name, SymbolUsage::Assigned)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
}
} else {
self.scan_expression(left, &ExpressionContext::Store)?;
self.scan_expression(left, ExpressionContext::Store)?;
}
}
}
@@ -882,10 +886,10 @@ impl SymbolTableBuilder {
line_number: usize,
) -> SymbolTableResult {
// Evaluate eventual default parameters:
self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
for kw_default in &args.kw_defaults {
if let Some(expression) = kw_default {
self.scan_expression(&expression, &ExpressionContext::Load)?;
self.scan_expression(&expression, ExpressionContext::Load)?;
}
}
@@ -917,7 +921,7 @@ impl SymbolTableBuilder {
match group {
ast::StringGroup::Constant { .. } => {}
ast::StringGroup::FormattedValue { value, spec, .. } => {
self.scan_expression(value, &ExpressionContext::Load)?;
self.scan_expression(value, ExpressionContext::Load)?;
if let Some(spec) = spec {
self.scan_string_group(spec)?;
}
@@ -931,10 +935,14 @@ impl SymbolTableBuilder {
Ok(())
}
fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
fn register_name(
&mut self,
name: &str,
role: SymbolUsage,
location: Location,
) -> SymbolTableResult {
let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap();
let location = Default::default();
// Some checks for the symbol that present on this scope level:
if let Some(symbol) = table.symbols.get(name) {
derive/Cargo.toml
+1 -1
View File
@@ -15,7 +15,7 @@ syn = { version = "1.0", features = ["full"] }
syn-ext = { version = "0.2.1", features = ["full"] }
quote = "1.0"
proc-macro2 = "1.0"
rustpython-compiler = { path = "../compiler", version = "0.1.1" }
rustpython-compiler = { path = "../compiler/porcelain", version = "0.1.1" }
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
maplit = "1.0"
once_cell = "1.3.1"
derive/src/compile_bytecode.rs
+9 -7
View File
@@ -18,7 +18,7 @@ use once_cell::sync::Lazy;
use proc_macro2::{Span, TokenStream as TokenStream2};
use quote::quote;
use rustpython_bytecode::bytecode::{CodeObject, FrozenModule};
use rustpython_compiler::compile;
use rustpython_compiler as compile;
use std::collections::HashMap;
use std::env;
use std::fs;
@@ -50,12 +50,14 @@ impl CompilationSource {
module_name: String,
origin: F,
) -> Result<CodeObject, Diagnostic> {
compile::compile(source, mode, module_name, Default::default()).map_err(|err| {
Diagnostic::spans_error(
self.span,
format!("Python compile error from {}: {}", origin(), err),
)
})
compile::compile(source, mode, module_name, compile::CompileOpts::default()).map_err(
|err| {
Diagnostic::spans_error(
self.span,
format!("Python compile error from {}: {}", origin(), err),
)
},
)
}
fn compile(
examples/dis.rs
+1 -1
View File
@@ -14,7 +14,7 @@ extern crate log;
use clap::{App, Arg};
use rustpython_compiler::compile;
use rustpython_compiler as compile;
use std::error::Error;
use std::fs;
use std::path::Path;
examples/hello_embed.rs
+1 -2
View File
@@ -1,4 +1,3 @@
use rustpython_compiler as compiler;
use rustpython_vm as vm;
fn main() -> vm::pyobject::PyResult<()> {
@@ -8,7 +7,7 @@ fn main() -> vm::pyobject::PyResult<()> {
let code_obj = vm
.compile(
r#"print("Hello World!")"#,
compiler::compile::Mode::Exec,
vm::compile::Mode::Exec,
"<embedded>".to_owned(),
)
.map_err(|err| vm.new_syntax_error(&err))?;
examples/mini_repl.rs
+1 -6
View File
@@ -3,7 +3,6 @@
///! This much smaller REPL is still a useful example because it showcases inserting
///! values and functions into the Python runtime's scope, and showcases use
///! of the compilation mode "Single".
use rustpython_compiler as compiler;
use rustpython_vm as vm;
// these are needed for special memory shenanigans to let us share a variable with Python and Rust
use std::sync::atomic::{AtomicBool, Ordering};
@@ -65,11 +64,7 @@ def fib(n):
// this line also automatically prints the output
// (note that this is only the case when compile::Mode::Single is passed to vm.compile)
match vm
.compile(
&input,
compiler::compile::Mode::Single,
"<embedded>".to_owned(),
)
.compile(&input, vm::compile::Mode::Single, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err))
.and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone()))
{
parser/Cargo.toml
+1
View File
@@ -12,6 +12,7 @@ edition = "2018"
lalrpop = "0.19"
[dependencies]
rustpython-ast = { path = "../ast" }
lalrpop-util = "0.19.1"
log = "0.4.1"
num-bigint = "0.3"
parser/src/error.rs
+25 -1
View File
@@ -2,7 +2,7 @@
//! The goal is to provide a matching and a safe error API, maksing errors from LALR
use lalrpop_util::ParseError as LalrpopError;
use crate::location::Location;
use crate::ast::Location;
use crate::token::Tok;
use std::error::Error;
@@ -200,6 +200,30 @@ impl fmt::Display for ParseErrorType {
}
}
impl Error for ParseErrorType {}
impl ParseErrorType {
pub fn is_indentation_error(&self) -> bool {
match self {
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
ParseErrorType::UnrecognizedToken(token, expected) => {
*token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
}
_ => false,
}
}
pub fn is_tab_error(&self) -> bool {
matches!(self, ParseErrorType::Lexical(LexicalErrorType::TabError))
}
}
impl std::ops::Deref for ParseError {
type Target = ParseErrorType;
fn deref(&self) -> &Self::Target {
&self.error
}
}
impl Error for ParseError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
None
parser/src/fstring.rs
+1 -2
View File
@@ -2,9 +2,8 @@ use std::iter;
use std::mem;
use std::str;
use crate::ast::{ConversionFlag, StringGroup};
use crate::ast::{ConversionFlag, Location, StringGroup};
use crate::error::{FStringError, FStringErrorType};
use crate::location::Location;
use crate::parser::parse_expression;
use self::FStringErrorType::*;
parser/src/lexer.rs
+1 -1
View File
@@ -3,8 +3,8 @@
//! This means source code is translated into separate tokens.
pub use super::token::Tok;
use crate::ast::Location;
use crate::error::{LexicalError, LexicalErrorType};
use crate::location::Location;
use num_bigint::BigInt;
use num_traits::identities::Zero;
use num_traits::Num;
parser/src/lib.rs
+1 -2
View File
@@ -21,13 +21,12 @@
#[macro_use]
extern crate log;
use lalrpop_util::lalrpop_mod;
pub use rustpython_ast as ast;
pub mod ast;
pub mod error;
mod fstring;
mod function;
pub mod lexer;
pub mod location;
pub mod mode;
pub mod parser;
lalrpop_mod!(
parser/src/python.lalrpop
+1 -2
View File
@@ -10,7 +10,6 @@ use crate::fstring::parse_located_fstring;
use crate::function::{parse_args, parse_params};
use crate::error::LexicalError;
use crate::lexer;
use crate::location;
use num_bigint::BigInt;
@@ -1155,7 +1154,7 @@ Identifier: String = <s:name> => s;
// Hook external lexer:
extern {
type Location = location::Location;
type Location = ast::Location;
type Error = LexicalError;
enum lexer::Tok {
src/main.rs
+1 -1
View File
@@ -5,9 +5,9 @@ extern crate env_logger;
extern crate log;
use clap::{App, AppSettings, Arg, ArgMatches};
use rustpython_compiler::compile;
use rustpython_vm::{
builtins::PyInt,
compile,
exceptions::print_exception,
match_class,
pyobject::{BorrowValue, ItemProtocol, PyResult, TypeProtocol},
src/shell.rs
+1 -1
View File
@@ -1,9 +1,9 @@
mod helper;
use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType};
use rustpython_parser::error::{LexicalErrorType, ParseErrorType};
use rustpython_vm::readline::{Readline, ReadlineResult};
use rustpython_vm::{
compile::{self, CompileError, CompileErrorType},
exceptions::{print_exception, PyBaseExceptionRef},
pyobject::{BorrowValue, PyResult, TypeProtocol},
scope::Scope,
vm/Cargo.toml
+1 -1
View File
@@ -41,7 +41,7 @@ mt19937 = "1.0"
log = "0.4"
rustpython-derive = { path = "../derive", version = "0.1.2" }
rustpython-parser = { path = "../parser", optional = true, version = "0.1.2" }
rustpython-compiler = { path = "../compiler", optional = true, version = "0.1.2" }
rustpython-compiler = { path = "../compiler/porcelain", optional = true, version = "0.1.2" }
rustpython-bytecode = { path = "../bytecode", version = "0.1.2" }
rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
rustpython-pylib = { path = "pylib-crate", optional = true, version = "0.1.0" }
vm/src/builtins/make_module.rs
+2 -2
View File
@@ -21,6 +21,8 @@ mod decl {
use crate::builtins::pytype::PyTypeRef;
use crate::byteslike::PyBytesLike;
use crate::common::{hash::PyHash, str::to_ascii};
#[cfg(feature = "rustpython-compiler")]
use crate::compile;
use crate::exceptions::PyBaseExceptionRef;
use crate::function::{single_or_tuple_any, Args, FuncArgs, KwArgs, OptionalArg};
use crate::iterator;
@@ -36,8 +38,6 @@ mod decl {
use crate::{py_io, sysmodule};
use num_bigint::Sign;
use num_traits::{Signed, ToPrimitive, Zero};
#[cfg(feature = "rustpython-compiler")]
use rustpython_compiler::compile;
#[pyfunction]
fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult {
vm/src/eval.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use crate::compile;
use crate::pyobject::PyResult;
use crate::scope::Scope;
use crate::VirtualMachine;
use rustpython_compiler::compile;
pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult {
match vm.compile(source, compile::Mode::Eval, source_path.to_owned()) {
vm/src/import.rs
+2 -2
View File
@@ -6,13 +6,13 @@ use rand::Rng;
use crate::builtins::code::CodeObject;
use crate::builtins::traceback::{PyTraceback, PyTracebackRef};
use crate::builtins::{code, list};
#[cfg(feature = "rustpython-compiler")]
use crate::compile;
use crate::exceptions::PyBaseExceptionRef;
use crate::pyobject::{ItemProtocol, PyResult, PyValue, TryFromObject, TypeProtocol};
use crate::scope::Scope;
use crate::version::get_git_revision;
use crate::vm::{InitParameter, VirtualMachine};
#[cfg(feature = "rustpython-compiler")]
use rustpython_compiler::compile;
pub(crate) fn init_importlib(
vm: &mut VirtualMachine,
vm/src/lib.rs
+2
View File
@@ -79,6 +79,8 @@ mod vm;
pub use self::vm::{InitParameter, Interpreter, PySettings, VirtualMachine};
pub use rustpython_bytecode::*;
pub use rustpython_common as common;
#[cfg(feature = "rustpython-compiler")]
pub use rustpython_compiler as compile;
#[doc(hidden)]
pub mod __exports {
vm/src/stdlib/dis.rs
+1 -1
View File
@@ -6,9 +6,9 @@ mod decl {
use crate::builtins::dict::PyDictRef;
use crate::builtins::pystr::PyStrRef;
use crate::bytecode::CodeFlags;
use crate::compile;
use crate::pyobject::{BorrowValue, ItemProtocol, PyObjectRef, PyResult, TryFromObject};
use crate::vm::VirtualMachine;
use rustpython_compiler::compile;
#[pyfunction]
fn dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
vm/src/stdlib/symtable.rs
+10 -31
View File
@@ -6,10 +6,9 @@ mod decl {
use crate::builtins::pystr::PyStrRef;
use crate::builtins::pytype::PyTypeRef;
use crate::compile::{self, Symbol, SymbolScope, SymbolTable, SymbolTableType};
use crate::pyobject::{BorrowValue, PyRef, PyResult, PyValue, StaticType};
use crate::vm::VirtualMachine;
use rustpython_compiler::{compile, error::CompileError, symboltable};
use rustpython_parser::parser;
/// symtable. Return top level SymbolTable.
/// See docs: https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.symtable
@@ -25,35 +24,15 @@ mod decl {
.parse::<compile::Mode>()
.map_err(|err| vm.new_value_error(err.to_string()))?;
let symtable = source_to_symtable(source.borrow_value(), mode, filename.borrow_value())
.map_err(|err| vm.new_syntax_error(&err))?;
let symtable =
compile::compile_symtable(source.borrow_value(), mode, filename.borrow_value())
.map_err(|err| vm.new_syntax_error(&err))?;
let py_symbol_table = to_py_symbol_table(symtable);
Ok(py_symbol_table.into_ref(vm))
}
fn source_to_symtable(
source: &str,
mode: compile::Mode,
filename: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let from_parse_error = |e| CompileError::from_parse_error(e, filename.to_owned());
let symtable = match mode {
compile::Mode::Exec | compile::Mode::Single => {
let ast = parser::parse_program(source).map_err(from_parse_error)?;
symboltable::make_symbol_table(&ast)
}
compile::Mode::Eval => {
let statement = parser::parse_statement(source).map_err(from_parse_error)?;
symboltable::statements_to_symbol_table(&statement)
}
}
.map_err(|e| CompileError::from_symbol_table_error(e, filename.to_owned()))?;
Ok(symtable)
}
fn to_py_symbol_table(symtable: symboltable::SymbolTable) -> PySymbolTable {
fn to_py_symbol_table(symtable: SymbolTable) -> PySymbolTable {
PySymbolTable { symtable }
}
@@ -63,7 +42,7 @@ mod decl {
#[pyattr]
#[pyclass(name = "SymbolTable")]
struct PySymbolTable {
symtable: symboltable::SymbolTable,
symtable: SymbolTable,
}
impl fmt::Debug for PySymbolTable {
@@ -102,7 +81,7 @@ mod decl {
#[pymethod(name = "is_optimized")]
fn is_optimized(&self) -> bool {
self.symtable.typ == symboltable::SymbolTableType::Function
self.symtable.typ == SymbolTableType::Function
}
#[pymethod(name = "lookup")]
@@ -180,8 +159,8 @@ mod decl {
#[pyattr]
#[pyclass(name = "Symbol")]
struct PySymbol {
symbol: symboltable::Symbol,
namespaces: Vec<symboltable::SymbolTable>,
symbol: Symbol,
namespaces: Vec<SymbolTable>,
}
impl fmt::Debug for PySymbol {
@@ -226,7 +205,7 @@ mod decl {
#[pymethod(name = "is_nonlocal")]
fn is_nonlocal(&self) -> bool {
matches!(self.symbol.scope, symboltable::SymbolScope::Nonlocal)
matches!(self.symbol.scope, SymbolScope::Nonlocal)
}
#[pymethod(name = "is_referenced")]
vm/src/vm.rs
+14 -19
View File
@@ -24,6 +24,8 @@ use crate::builtins::pystr::{PyStr, PyStrRef};
use crate::builtins::pytype::PyTypeRef;
use crate::builtins::tuple::PyTuple;
use crate::common::{hash::HashSecret, lock::PyMutex, rc::PyRc};
#[cfg(feature = "rustpython-compiler")]
use crate::compile::{self, CompileError, CompileErrorType, CompileOpts};
use crate::exceptions::{self, PyBaseException, PyBaseExceptionRef};
use crate::frame::{ExecutionResult, Frame, FrameRef};
use crate::function::{FuncArgs, IntoFuncArgs};
@@ -34,11 +36,6 @@ use crate::pyobject::{
use crate::scope::Scope;
use crate::slots::PyComparisonOp;
use crate::{builtins, bytecode, frozen, import, iterator, stdlib, sysmodule};
#[cfg(feature = "rustpython-compiler")]
use rustpython_compiler::{
compile::{self, CompileOpts},
error::CompileError,
};
// use objects::ects;
@@ -659,12 +656,12 @@ impl VirtualMachine {
#[cfg(feature = "rustpython-compiler")]
pub fn new_syntax_error(&self, error: &CompileError) -> PyBaseExceptionRef {
let syntax_error_type = if error.is_indentation_error() {
self.ctx.exceptions.indentation_error.clone()
} else if error.is_tab_error() {
self.ctx.exceptions.tab_error.clone()
} else {
self.ctx.exceptions.syntax_error.clone()
let syntax_error_type = match &error.error {
CompileErrorType::Parse(p) if p.is_indentation_error() => {
self.ctx.exceptions.indentation_error.clone()
}
CompileErrorType::Parse(p) if p.is_tab_error() => self.ctx.exceptions.tab_error.clone(),
_ => self.ctx.exceptions.syntax_error.clone(),
};
let syntax_error = self.new_exception_msg(syntax_error_type, error.to_string());
let lineno = self.ctx.new_int(error.location.row());
@@ -673,10 +670,12 @@ impl VirtualMachine {
.unwrap();
self.set_attr(syntax_error.as_object(), "offset", offset)
.unwrap();
if let Some(v) = error.statement.as_ref() {
self.set_attr(syntax_error.as_object(), "text", self.new_pyobj(v))
.unwrap();
}
self.set_attr(
syntax_error.as_object(),
"text",
error.statement.clone().into_pyobject(self),
)
.unwrap();
self.set_attr(
syntax_error.as_object(),
"filename",
@@ -1231,10 +1230,6 @@ impl VirtualMachine {
) -> Result<PyCodeRef, CompileError> {
compile::compile(source, mode, source_path, opts)
.map(|code| PyCode::new(self.ctx.map_codeobj(code)).into_ref(self))
.map_err(|mut compile_error| {
compile_error.update_statement_info(source.trim_end().to_owned());
compile_error
})
}
fn call_codec_func(
wasm/lib/Cargo.toml
-1
View File
@@ -15,7 +15,6 @@ default = ["freeze-stdlib"]
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
[dependencies]
rustpython-compiler = { path = "../../compiler" }
rustpython-parser = { path = "../../parser" }
rustpython-common = { path = "../../common" }
# no threading feature for rustpython-vm -- doesn't much matter anyway, but it might be more optimized
wasm/lib/src/convert.rs
+1 -1
View File
@@ -1,9 +1,9 @@
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
use wasm_bindgen::{closure::Closure, prelude::*, JsCast};
use rustpython_compiler::error::{CompileError, CompileErrorType};
use rustpython_parser::error::ParseErrorType;
use rustpython_vm::byteslike::PyBytesLike;
use rustpython_vm::compile::{CompileError, CompileErrorType};
use rustpython_vm::exceptions::PyBaseExceptionRef;
use rustpython_vm::function::FuncArgs;
use rustpython_vm::pyobject::{
wasm/lib/src/lib.rs
+1 -1
View File
@@ -8,7 +8,7 @@ pub mod wasm_builtins;
extern crate rustpython_vm;
use js_sys::{Object, Reflect, TypeError};
use rustpython_compiler::compile::Mode;
use rustpython_vm::compile::Mode;
use std::panic;
use wasm_bindgen::prelude::*;
wasm/lib/src/vm_class.rs
+1 -2
View File
@@ -5,8 +5,8 @@ use std::rc::{Rc, Weak};
use js_sys::{Object, TypeError};
use wasm_bindgen::prelude::*;
use rustpython_compiler::compile;
use rustpython_vm::common::rc::PyRc;
use rustpython_vm::compile::{self, Mode};
use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyObjectWeak, PyValue};
use rustpython_vm::scope::{NameProtocol, Scope};
use rustpython_vm::{InitParameter, Interpreter, PySettings, VirtualMachine};
@@ -15,7 +15,6 @@ use crate::browser_module::setup_browser_module;
use crate::convert::{self, PyResultExt};
use crate::js_module;
use crate::wasm_builtins;
use rustpython_compiler::mode::Mode;
pub(crate) struct StoredVirtualMachine {
pub interp: Interpreter,